diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6e8e0099eaf25c6e3f962f95f68570e205f273bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/README.md @@ -0,0 +1,70 @@ +# mmlu_pro_plus + +### Paper + +Title: `MMLU-Pro+: Evaluating Higher-Order Reasoning and Shortcut Learning in LLMs` + +Abstract: `Existing benchmarks for large language models (LLMs) increasingly struggle to differentiate between +top-performing models, underscoring the need for more challenging evaluation frameworks. +We introduce MMLU-Pro+, an enhanced benchmark building upon MMLU-Pro to assess shortcut +learning and higher-order reasoning in LLMs. By incorporating questions with multiple +correct answers across diverse domains, MMLU-Pro+ tests LLMs' ability to engage in complex +reasoning and resist simplistic problem-solving strategies. Our results show that +MMLU-Pro+ maintains MMLU-Pro's difficulty while providing a more rigorous test of +model discrimination, particularly in multi-correct answer scenarios. +We introduce novel metrics like shortcut selection ratio and correct pair identification +ratio, offering deeper insights into model behavior and anchoring bias. +Evaluations of six state-of-the-art LLMs reveal significant performance gaps, +highlighting variations in reasoning abilities and bias susceptibility.` + +Homepage: https://github.com/asgsaeid/mmlu-pro-plus + +### Citation + +```bibtex +@article{taghanaki2024mmlu, + title={MMLU-Pro+: Evaluating Higher-Order Reasoning and Shortcut Learning in LLMs}, + author={Taghanaki, Saeid Asgari and Khani, Aliasgahr and Khasahmadi, Amir}, + journal={arXiv preprint arXiv:2409.02257}, + year={2024} +} +``` + +### Groups and Tasks + +#### Groups + +* `mmlu_pro_plus`: 'All 14 subjects of the mmlu_pro_plus dataset, evaluated following the methodology in mmlu's original implementation' + +#### Tasks + +The following tasks evaluate subjects in the mmlu_pro dataset +- `mmlu_pro_plus_biology` +- `mmlu_pro_plus_business` +- `mmlu_pro_plus_chemistry` +- `mmlu_pro_plus_computer_science` +- `mmlu_pro_plus_economics` +- `mmlu_pro_plus_engineering` +- `mmlu_pro_plus_health` +- `mmlu_pro_plus_history` +- `mmlu_pro_plus_law` +- `mmlu_pro_plus_math` +- `mmlu_pro_plus_other` +- `mmlu_pro_plus_philosophy` +- `mmlu_pro_plus_physics` +- `mmlu_pro_plus_psychology` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_default_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ece65fb3e32feb8f02ece289cef099bcfe14d54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_default_template_yaml @@ -0,0 +1,33 @@ +dataset_path: saeidasgari/mmlu-pro-plus +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJKL])\)?' + # regex_pattern: r".*[aA]nswer:\s*([A-L])", + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_mmlu_pro_plus.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_mmlu_pro_plus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e92c9a055741bb9839839a55a196757591f6ba3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_mmlu_pro_plus.yaml @@ -0,0 +1,23 @@ +group: mmlu_pro_plus +task: + - mmlu_pro_plus_biology + - mmlu_pro_plus_business + - mmlu_pro_plus_chemistry + - mmlu_pro_plus_computer_science + - mmlu_pro_plus_economics + - mmlu_pro_plus_engineering + - mmlu_pro_plus_health + - mmlu_pro_plus_history + - mmlu_pro_plus_law + - mmlu_pro_plus_math + - mmlu_pro_plus_other + - mmlu_pro_plus_philosophy + - mmlu_pro_plus_physics + - mmlu_pro_plus_psychology +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ec1546679608c25745480896f6b9851204c1705 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_biology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_biology" +task_alias: "biology" +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f348859965a3259a085fb266a3f3fcf0ed819e78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_business.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_business" +task_alias: "business" +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e498bfd877c077e5ec44e88877eda358fdaa1eed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_chemistry.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_chemistry" +task_alias: "chemistry" +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b652343b675f9fff3a7fe535844b999e550948b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_computer_science.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_computer_science" +task_alias: "computer_science" +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f7e2d1b45ffc21eb07f473d8823ca903ddd973a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_economics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_economics" +task_alias: "economics" +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..745def5fc3cb7c4779e1bae35aa7f2ad11c40c28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_engineering.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_engineering" +task_alias: "engineering" +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8f65c2b09f9f244c2f5f7b5388f4a4cab0e31d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_health.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_health" +task_alias: "health" +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b562ae2b2f8ea79af2337740b5b1f2d2dc694bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_history.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_history" +task_alias: "history" +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0af0e50b460921728781447276cf60217fd70075 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_law.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_law" +task_alias: "law" +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1d8c95eff3b3cdf4e436aa52b77799589d2b188 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_math.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_math" +task_alias: "math" +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a2372ae36cc97e6f5eca5e1e1fdde0556f68e07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_other.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1768cab3628d65dd5dff855c87d694cc70b269b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_philosophy.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a4b25c013a9bfed0b1836a84378baa88a305960 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_physics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497c3f359ecb5ca96e79705605ac6ffb029fe790 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_psychology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_psychology" +task_alias: "psychology" +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..03117be5f165fd7edf40404bf9934b3753039f1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/utils.py @@ -0,0 +1,63 @@ +from functools import partial + + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + + +def format_cot_example(example, including_answer=True): + prompt = "Question:\n" + question = example["question"] + options = example["options"] + prompt += question + "\n" + prompt += "Options:\n" + for i, opt in enumerate(options): + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace( + "A: Let's think step by step.", "Answer: Let's think step by step." + ) + prompt += cot_content + "\n\n" + else: + prompt += "Answer: Let's think step by step." + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ececdb0ab921bdc24b8aac41979a93d35670d0c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_us_history" +"task_alias": "high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dff834ef804039858b6955155a8338dd11b30b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ef004988965c41ff075f2f976b98dca4657ca04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..348d21403f06669e198146286b83e227fbde5a16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_moral_disputes" +"task_alias": "moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3762ee1200848439f08a3c69703af4cffb3a9d74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_moral_scenarios" +"task_alias": "moral_scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55f8ca01ff42a296c07d8fd2e2ccda373d91775b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0c0608ef6860edb4b8492402c674a7efda2070f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology" +"task_alias": "professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b675bdfd088bb7e651eece031198b5c0fb8ab3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_public_relations" +"task_alias": "public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b02342d95ede5148ee8b0aeb9e4ad4fb7dd05938 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_security_studies" +"task_alias": "security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49fa11620fb7147752328a484d56f8ead64c4387 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc4335e9eace7816ba112e4f55912223444d4c1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy" +"task_alias": "us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f8bc114c3ce7437ad0fb413a69a859f69bcbf99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b90972c7031c30d89beea835f70aab7cf45cce81 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_world_religions" +"task_alias": "world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa158a9f3c3e9e420608970a3ea91e744a042a6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml @@ -0,0 +1,61 @@ +dataset_name: college_physics +description: The following are multiple choice questions (with answers) about college + physics. +fewshot_config: + sampler: first_n + samples: + - question: 'A refracting telescope consists of two converging lenses separated by + 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification + of the telescope is + + (A) 4 (B) 5 (C) 6 (D) 20' + target: Let's think step by step. In a refracting telescope, if both lenses are + converging, the focus of both lenses must be between the two lenses, and thus + the focal lengths of the two lenses must add up to their separation. Since the + focal length of one lens is 20 cm, the focal length of the other must be 80 + cm. The magnification is the ratio of these two focal lengths, or 4. The answer + is (A). + - question: 'The muon decays with a characteristic lifetime of about 10^-6 second into + an electron, a muon neutrino, and an electron antineutrino. The muon is forbidden + from decaying into an electron and just a single neutrino by the law of conservation + of + + (A) charge (B) mass (C) energy and momentum (D) lepton number' + target: Let's think step by step. Lepton number must be conserved, meaning the + total number of leptons minus the number of antileptons. If a muon decays into + an electron and a single neutrino, the total lepton number would go from one + to two, violating lepton number conservation. The answer is (D). + - question: 'One end of a Nichrome wire of length 2L and cross-sectional area A is + attached to an end of another Nichrome wire of length L and cross- sectional + area 2A. If the free end of the longer wire is at an electric potential of 8.0 + volts, and the free end of the shorter wire is at an electric potential of 1.0 + volt, the potential at the junction of the two wires is most nearly equal to + + (A) 2.4 V (B) 3.3 V (C) 4.5 V (D) 5.7 V' + target: Let's think step by step. This is a simple voltage divider problem, where + the longer wire has a resistance four times that of the shorter end. So the + voltage divider ratio is 1 / 5, meaning that the potential in the middle is + 1.0 V + (8.0 V - 1.0 V) * 1/5 = 2.4 V. The answer is (A). + - question: 'A refracting telescope consists of two converging lenses separated by + 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification + of the telescope is + + (A) 4 (B) 5 (C) 6 (D) 20' + target: Let's think step by step. In a refracting telescope, if both lenses are + converging, the focus of both lenses must be between the two lenses, and thus + the focal lengths of the two lenses must add up to their separation. Since the + focal length of one lens is 20 cm, the focal length of the other must be 80 + cm. The magnification is the ratio of these two focal lengths, or 4. The answer + is (A). + - question: 'For which of the following thermodynamic processes is the increase in + the internal energy of an ideal gas equal to the heat added to the gas? + + (A) Constant temperature (B) Constant volume (C) Constant pressure (D) Adiabatic' + target: 'Let''s think step by step. Heat added to the gas can go into the gases + internal energy or work done against an external force. However, if the volume + of the gas container is constant, no work will be done (since work is pressure + times change in volume). So, at constant volume, all of the heat goes into the + internal energy. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a7c5cb2d5d1a2943b5b1ad6c17bf5ce0a324f5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml @@ -0,0 +1,50 @@ +dataset_name: computer_security +description: The following are multiple choice questions (with answers) about computer + security. +fewshot_config: + sampler: first_n + samples: + - question: 'SHA-1 has a message digest of + + (A) 160 bits (B) 512 bits (C) 628 bits (D) 820 bits' + target: Let's think step by step. Since SHA-1 is a hash function which takes an + question and produces a 160-bit (20-byte) hash value, its message digest is 160 + bits. The answer is (A). + - question: "_____________ can modify data on your system \u2013 so that your system\ + \ doesn\u2019t run correctly or you can no longer access specific data, or it\ + \ may even ask for ransom in order to give your access.\n(A) IM \u2013 Trojans\ + \ (B) Backdoor Trojans (C) Trojan-Downloader (D) Ransom Trojan" + target: Let's think step by step. The system is asking for trojans, which are + for ransom, which means ransom trojan. The answer is (D). + - question: 'What is ethical hacking? + + (A) "Hacking" ethics so they justify unintended selfish behavior (B) Hacking + systems (e.g., during penetration testing) to expose vulnerabilities so they + can be fixed, rather than exploited (C) Hacking into systems run by those whose + ethics you disagree with (D) A slang term for rapid software development, e.g., + as part of hackathons' + target: Let's think step by step. Ethical hacking is a process of detecting vulnerabilities + in an application, system, or organization's infrastructure that an attacker + can use to exploit an individual or organization. They use this process to prevent + cyberattacks and security breaches by lawfully hacking into the systems and + looking for weak points. The answer is (B). + - question: 'The ____________ is anything which your search engine cannot search. + + (A) Haunted web (B) World Wide Web (C) Surface web (D) Deep Web' + target: "Let's think step by step. The search engine searches on the Surface Web,\ + \ which is the portion of the world wide web which is visible so (B,C) are wrong.\ + \ The Haunted Web doesn\u2019t correspond to an internet concept. The Deep Web\ + \ is the part of the World Wide Web which is not indexed. The answer is (D)." + - question: 'Exploitation of the Heartbleed bug permits + + (A) overwriting cryptographic keys in memory (B) a kind of code injection (C) + a read outside bounds of a buffer (D) a format string attack' + target: 'Let''s think step by step. The Heartbleed Bug is a serious vulnerability + in the popular OpenSSL cryptographic software library. Heartbleed resulted from + improper question validation (due to a missing bounds check) in the implementation + of the TLS heartbeat extension. The vulnerability was classified as a buffer + over-read, a situation where more data can be read than should be allowed. The + answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4757faf1d490e312ea70a3d4dbe291459366471 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml @@ -0,0 +1,49 @@ +dataset_name: conceptual_physics +description: ' + + The following are multiple choice questions (with answers) about conceptual physics.' +fewshot_config: + sampler: first_n + samples: + - question: 'Colors in a soap bubble result from light + + (A) converted to a different frequency (B) deflection (C) interference (D) polarization' + target: Let's think step by step. In a soap bubble film, the light bounces between + the two soap-air interfaces many times, interfering with itself constructively + or destructively depending on the width of the film. This results in different + colors being visible. The answer is (C). + - question: 'Compared with the mass of a uranium atom undergoing fission, the combined + masses of the products after fission are + + (A) less (B) more (C) the same (D) zero' + target: Let's think step by step. Fission releases energy, which comes from the + rest mass of its initial nucleus. Thus the mass of the products is less than + the mass of the reactant uranium nucleus. The answer is (A). + - question: 'Things that are equivalent according to the equivalence principle are + + (A) space and time. (B) a traveling twin and a stay-at-home twin. (C) gravity + and acceleration. (D) mass and energy.' + target: "Let's think step by step. Einstein\u2019s famous equivalence principle\ + \ states that gravity and acceleration are equivalent. The answer is (C)." + - question: 'Which of these three elements has the most mass per nucleon? + + (A) Hydrogen (B) Iron (C) Uranium (D) Same in each' + target: Let's think step by step. Due to nuclear binding energy, the mass of an + atomic nucleus is less than the sum of individual masses of the free constituent + protons and neutrons; this is known as the mass defect. Hydrogen has no mass + defect because it has only a single nucleon, so it will have the most mass per + nucleon. The answer is (A). + - question: 'A model airplane flies slower when flying into the wind and faster with + wind at its back. When launched at right angles to the wind a cross wind its + groundspeed compared with flying in still air is + + (A) the same (B) greater (C) less (D) either greater or less depending on wind + speed' + target: "Let's think step by step. The plane\u2019s speed in the direction of\ + \ the wind is greater than it would be in the absence of wind, and its direction\ + \ orthogonal to the wind is the same as it would be in the absence of the wind.\ + \ The total speed, which is these two components added in quadrature, is thus\ + \ greater than the speed in still air. The answer is (B).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fb9eef860424b49adf108566edadaf1500dbf5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml @@ -0,0 +1,77 @@ +dataset_name: elementary_mathematics +description: The following are multiple choice questions (with answers) about elementary + mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Olivia used the rule "Add 11" to create the number pattern shown below. + 10, 21, 32, 43, 54. Which statement about the number pattern is true? + + (A) The 10th number in the pattern will be an even number. + + (B) The number pattern will never have two even numbers next to each other. + + (C) The next two numbers in the pattern will be an even number then an odd number. + + (D) If the number pattern started with an odd number then the pattern would + have only odd numbers in it.' + target: Let's think step by step. Choice A is incorrect because every even-numbered + term in the pattern is odd, and 10 is an even number. Choice B is correct, because + adding an odd number (in this case 11) to an odd number produces an even number, + and adding an odd number to an even number produces an odd number. Thus the + terms in the pattern will alternate between odd and even, so there will never + be two even numbers next to each other. Choice C is incorrect because the last + term in the example is even (54), and we know that the terms will alternate + between even and odd. Choice D is incorrect because the terms in the pattern + will alternate between odd and even, regardless of the value of the first term. + The answer is (B). + - question: 'The population of the city where Michelle was born is 145,826. What is + the value of the 5 in the number 145,826? + + (A) 5 thousands + + (B) 5 hundreds + + (C) 5 tens + + (D) 5 ones' + target: 'Let''s think step by step. Choice A is correct, because there are three + digits following the 5, so + + the 5 is in the thousands place. Thus the other choices are incorrect. The answer + is (A).' + - question: "A store sells 107 different colors of paint. They have 25 cans of each\ + \ color in storage. The number of cans of paint the store has in storage can\ + \ be found using the expression below. 107 \xD7 25. How many cans of paint does\ + \ the store have in storage?\n(A) 749\n(B) 2,675\n(C) 2,945\n(D) 4,250" + target: Let's think step by step. We can calculate 107 x 25 = (100 x 25) + (7 + x 25) = 2500 + 175 = 2675. The answer is (B). + - question: 'A total of 30 players will play basketball at a park. There will be exactly + 5 players on each team. Which statement correctly explains how to find the number + of teams needed? + + (A) Add 5 to 30 to find 35 teams. + + (B) Divide 30 by 5 to find 6 teams. + + (C) Multiply 30 and 5 to find 150 teams. + + (D) Subtract 5 from 30 to find 25 teams.' + target: Let's think step by step. We want to find the number of teams. We know + that there are 5 players/team, and 30 players. Thus to get the number of teams + we divide players by players/team, so 30 players / 5 players/team = 6 teams. + The answer is (B). + - question: 'Which expression is equivalent to 5 x 9? + + (A) (5 x 4) x (6 x 5) + + (B) (5 x 5) + (5 x 4) + + (C) (5 x 5) + (5 x 9) + + (D) (5 x 9) x (6 x 9)' + target: 'Let''s think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5 + + 4) = (5 x 5) + (5 x 4). The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fbc73c3d24f9cef06a41bbcfddea55aec1b424a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml @@ -0,0 +1,70 @@ +dataset_name: formal_logic +description: The following are multiple choice questions (with answers) about formal + logic. +fewshot_config: + sampler: first_n + samples: + - question: "Which of the given formulas of PL is the best symbolization of the following\ + \ sentence?\nTurtles live long lives and are happy creatures, unless they are\ + \ injured.\n(A) (L \u2022 H) \u2261 I (B) (L \u2022 H) \u2228 I (C) L \u2022\ + \ (H \u2228 I) (D) L \u2022 (H \u2283 R)." + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CL\u201D denote \u201C\ + living long\u201D, H \u201Cbeing happy\u201D, and \u201CI\u201D \u201Cbeing\ + \ injured\u201D. Now, consider each choice:\n(A) means (living long AND being\ + \ happy) is equivalent to (being injured). \n(B) means (living long AND being\ + \ happy) OR (being injured). \n(C) means (living long) AND (being happy OR being\ + \ injured). \n(D) means (living long) AND (being happy implies being R), but\ + \ what R denotes is not clear.\nObviously, (B) is the best symbolization of\ + \ the original sentence. The answer is (B)." + - question: 'Select the best translation into predicate logic.George borrows Hector''s + lawnmower. (g: George; h: Hector; l: Hector''s lawnmower; Bxyx: x borrows y + from z). + + (A) Blgh (B) Bhlg (C) Bglh (D) Bghl' + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. We are told that \u201CBxyx\u201D\ + \ means \u201Cx borrows y from z\u201D. We can rewrite \u201CGeorge borrows\ + \ Hector's lawnmower\u201D as \u201CGeorge borrows a lawnmower from Hector\u201D\ + , which can then be translated into predicate logic as \u201CBglh\u201D. The\ + \ answer \u201CBglh\u201D appears in (C); therefore, (C) must be the correct\ + \ answer. The answer is (C)." + - question: "\nSelect the best English interpretation of the given arguments in predicate\ + \ logic.\nDm\n(\u2200x)(Wx \u2283 ~Dx). \n(\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax\n\ + (A) Marina is a dancer. Some weaklings are not dancers. Either everything is\ + \ a weakling or Georgia plays volleyball. So something plays volleyball. (B)\ + \ Marina is a dancer. No weakling is a dancer. Everything is either a weakling\ + \ or plays volleyball. So something plays volleyball. (C) Marina is a dancer.\ + \ Some weaklings are not dancers. Everything is either a weakling or plays volleyball.\ + \ So something plays volleyball. (D) Marina is a dancer. No weakling is a dancer.\ + \ Either everything is a weakling or Georgia plays volleyball. So something\ + \ plays volleyball." + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CD\u201D denote \u201C\ + being a dancer\u201D, \u201Cm\u201D denote \u201CMaria\u201D, \u201Cg\u201D\ + \ denote \u201CGeorgia\u201D, \u201CW\u201D denote \u201Cweakling\u201D, \u201C\ + A\u201D denote \u201Cplaying volleyball\u201D. Then, we have the following:\n\ + 1. Dm \u2192 Maria is a dance.\n2. (\u2200x)(Wx \u2283 ~Dx). \u2192 For all\ + \ x, if x is a weakling, then x is not a dancer. In other words, no weakling\ + \ is a dancer.\n3. (\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax \u2192 For all x, x\ + \ is a weakling or Georgia plays volleyball. So there exists an x that plays\ + \ volleyball. \nOptions (A) and (C) do claim that some weaklings are not dancers,\ + \ but the second argument strongly states that no weakling is a dancer. Thus,\ + \ we can eliminate them. Option (B) omits the important detail about Georgia\ + \ playing volleyball. Option (D) has all the details presented in the arguments\ + \ and is the best English interpretation of the arguments. The answer is (D)." + - question: "Select the best translation into predicate logic: No people drive on Mars.\n\ + (A) ~Pd (B) (\u2200x)(Px \u2228 ~Dx) (C) (\u2200x)(Px \u2283 ~Dx) (D) ~Dp" + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CP\u201D denote \u201C\ + being on Mars\u201D and \u201CD\u201D denote \u201Cdriving on Mars\u201D. Then\ + \ let\u2019s consider each option:\nOption (A): ~Pd \u2192 d is not on Mars.\n\ + Option (B): (\u2200x)(Px \u2228 ~Dx) \u2192 For all x, x is on Mars and x do\ + \ not drive on Mars.\nOption (C): (\u2200x)(Px \u2283 ~Dx) \u2192 For all x,\ + \ x is on Mars implies that x do not drive on Mars.\nOption (D): ~Dp: \u2192\ + \ p do not drive on Mars.\nOf all these options, Option (C) appears to be the\ + \ best and most meaningful interpretation of the argument \u201CNo people drive\ + \ on Mars.\u201D The answer is (C).\n\n" +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..739c00106536c8df64213d55b83c053db1314124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml @@ -0,0 +1,49 @@ +dataset_name: global_facts +description: The following are multiple choice questions (with answers) about global + facts. +fewshot_config: + sampler: first_n + samples: + - question: "As of 2017, how many of the world\u2019s 1-year-old children today have\ + \ been vaccinated against some disease? *\n(A) 80% (B) 60% (C) 40% (D) 20%" + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. According to data published by the World Health Organization, the + nummber of 1-year-old children vaccinated in 2017 exceeds 80%. The answer is + (A). + - question: 'As of 2019, about what percentage of Americans agree that the state is + run for the benefit of all the people? + + (A) 31% (B) 46% (C) 61% (D) 76%' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. In 2019, about 46% percentage of Americans agree that the state is + run for the benefit of all the people. The answer is (B). + - question: 'As of 2019, about what percentage of Russians say it is very important + to have free media in our country without government/state censorship? + + (A) 38% (B) 53% (C) 68% (D) 83%' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. As of 2019, about 38% of Russians say it is very important to have + free media in our country. The answer is (A). + - question: 'As of 2015, since 1990 forests have ____ in Europe and have ____ in Africa + and the Americas. + + (A) increased, increased (B) increased, decreased (C) decreased, increased (D) + decreased, decreased' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. As of 2015, since 1990 forests have increased in Europe and have decreased + in Africa and the Americas. The answer is (B). + - question: 'Which of the following pairs of statements are both true (as of 2019)? + + (A) People tend to be optimistic about their own future and the future of their + nation or the world. (B) People tend to be optimistic about their own future + but pessimistic about the future of their nation or the world. (C) People tend + to be pessimistic about their own future but optimistic about the future of + their nation or the world. (D) People tend to be pessimistic about their own + future and the future of their nation or the world.' + target: 'Let''s think step by step. We refer to Wikipedia articles on global facts + for help. As of 2019, most people tend to be optimistic about their own future + but pessimistic about the future of their nation or the world. The answer is + (B).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dfb19f924761c6dd56cc6b3b9ada38b5bf473e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml @@ -0,0 +1,69 @@ +dataset_name: high_school_biology +description: The following are multiple choice questions (with answers) about high + school biology. +fewshot_config: + sampler: first_n + samples: + - question: "In animal cells, which of the following represents the most likely pathway\ + \ that a secretory protein takes as it is synthesized in a cell?\n(A) Plasma\ + \ membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough\ + \ ER (B) Ribosome\u2013Golgi apparatus\u2013rough ER\u2013secretory vesicle\u2013\ + plasma membrane (C) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013\ + secretory vesicle\u2013rough ER (D) Ribosome\u2013rough ER\u2013Golgi apparatus\u2013\ + secretory vesicle\u2013plasma membrane" + target: Let's think step by step. Protein synthesis starts at the ribosome, so + we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum + and moves from there to the Golgi apparatus, where it is modified and packaged + into a vesicle. The vesicle then floats to the plasma membrane and is secreted. + The answer is (D). + - question: "A mutation in a bacterial enzyme changed a previously polar amino acid\ + \ into a nonpolar amino acid. This amino acid was located at a site distant\ + \ from the enzyme\u2019s active site. How might this mutation alter the enzyme\u2019\ + s substrate specificity?\n(A) By changing the enzyme\u2019s pH optimum (B) By\ + \ changing the enzyme\u2019s location in the cell (C) By changing the shape\ + \ of the protein (D) An amino acid change away from the active site cannot alter\ + \ the enzyme\u2019s substrate specificity." + target: Let's think step by step. A change in an amino acid leads to a change + in the primary structure of the protein. A change in the primary structure may + lead to a change in the secondary and the tertiary structure of the protein. + A change in the tertiary structure means a change in the shape of the protein, + so (C) has to be correct. Since the change does not affect the active site of + the enzyme, we do not expect the activity of the enzyme to be affected. The + answer is (C). + - question: 'Which of the following is not a way to form recombinant DNA? + + (A) Translation (B) Conjugation (C) Specialized transduction (D) Transformation' + target: 'Let''s think step by step. The introduction of foreign DNA or RNA into + bacteria or eukaryotic cells is a common technique in molecular biology and + scientific research. There are multiple ways foreign DNA can be introduced into + cells including transformation, transduction, conjugation, and transfection. + In contrast, (A) is not a way to form DNA: during translation the ribosomes + synthesize proteins from RNA. The answer is (A).' + - question: 'Homologous structures are often cited as evidence for the process of natural + selection. All of the following are examples of homologous structures EXCEPT + + (A) the wings of a bird and the wings of a bat (B) the flippers of a whale and + the arms of a man (C) the pectoral fins of a porpoise and the flippers of a + seal (D) the forelegs of an insect and the forelimbs of a dog' + target: "Let's think step by step. \u200B\u200BHomologous structures are similar\ + \ physical features in organisms that share a common ancestor \u200B\u200Bbut\ + \ different functions. Comparisons (B) and (C) are clearly homologous because\ + \ they share a common ancestor and the structures serve different purposes.\ + \ Bat wings and birg wings are also homologous, while they are both wings, the\ + \ forelimbs serve different purposes. Insects and dogs are very far ancestors\ + \ since one is vertebrate while the other is invertebrate and the forelimbs\ + \ serve the same purpose, so they are not homologous. The answer is (D)." + - question: 'Which of the following is not known to be involved in the control of cell + division? + + (A) Cyclins (B) Protein kinases (C) Checkpoints (D) Fibroblast cells' + target: 'Let''s think step by step. Normal cells move through the cell cycle in + a regulated way. At the checkpoint stage, they use information about their own + internal state and cues from the environment around them to decide whether to + proceed with cell division. Cues like these act by changing the activity of + core cell cycle regulators inside the cell. The most common regulators are cyclins + and cyclin-dependent kinases. Fibroblast cells do not play any role in cell + division. The answer is (D).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f21030ae880ec0d6d42f0e2618c312b55b82549 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml @@ -0,0 +1,84 @@ +dataset_name: high_school_computer_science +description: The following are multiple choice questions (with answers) about high + school computer science. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is an example of the use of a device on the Internet + of Things (IoT) ? + + (A) A car alerts a driver that it is about to hit an object. (B) A hiker uses + a G P S watch to keep track of her position. (C) A refrigerator orders milk + from an online delivery service when the milk in the refrigerator is almost + gone. (D) A runner uses a watch with optical sensors to monitor his heart rate.' + target: Let's think step by step. The term Internet of Things (IoT) refers to + common devices which are connected to the internet, enabling new functionality. + Choice A is incorrect because it does not describe an internet connected device. + In choice B, the watch is only described as having GPS functionality but no + internet connectivity. Choice C describes a common device (a refrigerator) which + has internet connectivity enabling new functionality (online ordering). Choice + D does not mention internet connectivity for the watch, only optical sensors. + The answer is (C). + - question: 'Many Web browsers allow users to open anonymous windows. During a browsing + session in an anonymous window, the browser does not record a browsing history + or a list of downloaded files. When the anonymous window is exited, cookies + created during the session are deleted. Which of the following statements about + browsing sessions in an anonymous window is true? + + (A) The activities of a user browsing in an anonymous window will not be visible + to people who monitor the user''s network, such as the system administrator. + (B) Items placed in a Web store''s shopping cart for future purchase during + the anonymous browsing session will not be saved on the user''s computer. (C) + A user will not be able to log in to e-mail or social media accounts during + the anonymous browsing session. (D) A user browsing in an anonymous window will + be protected from viruses launched from any web sites visited or files downloaded.' + target: "Let's think step by step. Choice A is incorrect as it only describes\ + \ network traffic, which an anonymous browser does not change. Choice B is correct\ + \ as it correctly describes how an anonymous browser will prevent saving data\ + \ on the user\u2019s computer after the session is ended. Choice C is incorrect\ + \ because an anonymous browser will not prevent logging in to email or social\ + \ media accounts. Choice D is incorrect because an anonymous browser in itself\ + \ performs no virus protection. The answer is (B)." + - question: "In the program below, the initial value of X is 5 and the initial value\ + \ of Y is 10.\nIF (X < 0){\n DISPLAY (\"Foxtrot\")\n} ELSE {\n IF (X > Y){\n\ + \ DISPLAY (\"Hotel\")\n } ELSE {\n IF (Y > 0){\n DISPLAY (\"November\")\n\ + \ } ELSE {\n DISPLAY (\"Yankee\")\n }\n }\n}\nWhat is displayed as a result\ + \ of running the program?\n(A) Foxtrot (B) Hotel (C) November (D) Yankee" + target: Let's think step by step. Because X has the value 5, the first conditional + IF (X < 0) is false, so we move to the first ELSE clause. Because X is 5 and + Y is 10, the second conditional IF (X > Y) is false, so we move to the following + ELSE clause. Since Y is 10, the conditional IF (Y > 0) is true, so the command + DISPLAY ("November") is executed. The answer is (C). + - question: 'What is the output of "abc"[::-1] in Python 3? + + (A) Error (B) abc (C) cba (D) c' + target: Let's think step by step. We know that the slicing operator [::-1] takes + all of the elements in the string in reverse order, so we reverse the order + of the string "abc", resulting in "cba". The answer is (C). + - question: "A list of numbers has n elements, indexed from 1 to n. The following algorithm\ + \ is intended to display the number of elements in the list that have a value\ + \ greater than 100. The algorithm uses the variables count and position. Steps\ + \ 3 and 4 are missing.\n Step 1: Set count to 0 and position to 1.\n Step 2:\ + \ If the value of the element at index position is greater than 100, increase\ + \ the value of count by 1.\n Step 3: (missing step)\n Step 4: (missing step)\n\ + \ Step 5: Display the value of count.\nWhich of the following could be used\ + \ to replace steps 3 and 4 so that the algorithm works as intended?\n(A) Step\ + \ 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until\ + \ the value of count is greater than 100.\n(B) Step 3: Increase the value of\ + \ position by 1.\n Step 4: Repeat steps 2 and 3 until the value of position\ + \ is greater than n.\n(C) Step 3: Repeat step 2 until the value of count is\ + \ greater than 100.\n Step 4: Increase the value of position by 1.\n(D) Step\ + \ 3: Repeat step 2 until the value of position is greater than n.\n Step 4:\ + \ Increase the value of count by 1." + target: 'Let''s think step by step. Choice A is incorrect, because its Step 4 + has an incorrect termination condition, stopping when count is greater than + 100. We need to stop after inspecting all elements in the list. Choice B is + correct because it correctly increments both count and position, and correctly + repeats these steps and terminates when all elements in the list have been inspected. + Choice C is incorrect because it incorrectly increments the variable count until + its value is greater than 100, regardless of the elements in the list. Choice + D is incorrect because its step 3 does not increment the value of position, + so it will repeat forever. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4707a1857f44fad0ef67313b5f1901b5b6c869b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml @@ -0,0 +1,199 @@ +dataset_name: high_school_european_history +description: The following are multiple choice questions (with answers) about high + school european history. +fewshot_config: + sampler: first_n + samples: + - question: 'This question refers to the following information. + + Albeit the king''s Majesty justly and rightfully is and ought to be the supreme + head of the Church of England, and so is recognized by the clergy of this realm + in their convocations, yet nevertheless, for corroboration and confirmation + thereof, and for increase of virtue in Christ''s religion within this realm + of England, and to repress and extirpate all errors, heresies, and other enormities + and abuses heretofore used in the same, be it enacted, by authority of this + present Parliament, that the king, our sovereign lord, his heirs and successors, + kings of this realm, shall be taken, accepted, and reputed the only supreme + head in earth of the Church of England, called Anglicans Ecclesia; and shall + have and enjoy, annexed and united to the imperial crown of this realm, as well + the title and style thereof, as all honors, dignities, preeminences, jurisdictions, + privileges, authorities, immunities, profits, and commodities to the said dignity + of the supreme head of the same Church belonging and appertaining; and that + our said sovereign lord, his heirs and successors, kings of this realm, shall + have full power and authority from time to time to visit, repress, redress, + record, order, correct, restrain, and amend all such errors, heresies, abuses, + offenses, contempts, and enormities, whatsoever they be, which by any manner + of spiritual authority or jurisdiction ought or may lawfully be reformed, repressed, + ordered, redressed, corrected, restrained, or amended, most to the pleasure + of Almighty God, the increase of virtue in Christ''s religion, and for the conservation + of the peace, unity, and tranquility of this realm; any usage, foreign land, + foreign authority, prescription, or any other thing or things to the contrary + hereof notwithstanding. + + English Parliament, Act of Supremacy, 1534 + + From the passage, one may infer that the English Parliament wished to argue + that the Act of Supremacy would + + (A) give the English king a new position of authority (B) give the position + of head of the Church of England to Henry VIII alone and exclude his heirs (C) + establish Calvinism as the one true theology in England (D) end various forms + of corruption plaguing the Church in England' + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The Act of Supremacy states that it grants authority to the king "to + repress and extirpate all errors, heresies, and other enormities and abuses", + referring to the corruption in the Church of England. The answer is (D). + - question: "This question refers to the following information.\nRead the following\ + \ excerpt.\nThe revolutionary seed had penetrated into every country and spread\ + \ more or less. It was greatly developed under the r\xE9gime of the military\ + \ despotism of Bonaparte. His conquests displaced a number of laws, institutions,\ + \ and customs; broke through bonds sacred among all nations, strong enough to\ + \ resist time itself; which is more than can be said of certain benefits conferred\ + \ by these innovators.\nThe monarchs will fulfil the duties imposed upon them\ + \ by Him who, by entrusting them with power, has charged them to watch over\ + \ the maintenance of justice, and the rights of all, to avoid the paths of error,\ + \ and tread firmly in the way of truth. Placed beyond the passions which agitate\ + \ society, it is in days of trial chiefly that they are called upon to despoil\ + \ realities of their false appearances, and to show themselves as they are,\ + \ fathers invested with the authority belonging by right to the heads of families,\ + \ to prove that, in days of mourning, they know how to be just, wise, and therefore\ + \ strong, and that they will not abandon the people whom they ought to govern\ + \ to be the sport of factions, to error and its consequences, which must involve\ + \ the loss of society.\nUnion between the monarchs is the basis of the policy\ + \ which must now be followed to save society from total ruin. . . .\nLet them\ + \ not confound concessions made to parties with the good they ought to do for\ + \ their people, in modifying, according to their recognized needs, such branches\ + \ of the administration as require it.\nLet them be just, but strong; beneficent,\ + \ but strict.\nLet them maintain religious principles in all their purity, and\ + \ not allow the faith to be attacked and morality interpreted according to the\ + \ social contract or the visions of foolish sectarians.\nLet them suppress Secret\ + \ Societies; that gangrene of society.\n\u2014Klemens von Metternich, Political\ + \ Confession of Faith, 1820\nWhich of the following was the greatest cause of\ + \ the fears expressed by Metternich in the document above?\n(A) The ideas of\ + \ personal liberty and nationalism conceived during the Enlightenment resulted\ + \ in radical revolutions that could spread throughout Europe. (B) The conquest\ + \ of Europe by Napoleon led to the creation of new factions and shifted the\ + \ European balance of power. (C) The power of monarchs had grown to the point\ + \ where it needed to be checked by other powers within each nation or domination\ + \ of civilians would occur. (D) The rising and falling economic cycle of the\ + \ newly emerging capitalist economy could lead to civilian unrest that must\ + \ be suppressed." + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The fears of revolution in early 19th century Europe expressed by + Klemens von Metternich, a conservative Austrian statesman, were a direct result + of the age of Enlightenment, a period of European history where the absolute + power of the monarchy was challenged with ideas of individual liberty and nationalism, + leading to the French revolution and its effects all over Europe. The answer + is (A). + - question: 'This question refers to the following information. + + The excerpts below are from the Navigation Acts of 1651. + + [A]fter the first day of December, one thousand six hundred fifty and one, and + from thence forwards, no goods or commodities whatsoever of the growth, production + or manufacture of Asia, Africa or America, or of any part thereof; or of any + islands belonging to them, or which are described or laid down in the usual + maps or cards of those places, as well of the English plantations as others, + shall be imported or brought into this Commonwealth of England, or into Ireland, + or any other lands, islands, plantations, or territories to this Commonwealth + belonging, or in their possession, in any other ship or ships, vessel or vessels + whatsoever, but only in such as do truly and without fraud belong only to the + people of this Commonwealth, or the plantations thereof, as the proprietors + or right owners thereof; and whereof the master and mariners are also of the + people of this Commonwealth, under the penalty of the forfeiture and loss of + all the goods that shall be imported contrary to this act, , , , + + [N]o goods or commodities of the growth, production, or manufacture of Europe, + or of any part thereof, shall after the first day of December, one thousand + six hundred fifty and one, be imported or brought into this Commonwealth of + England, or any other lands or territories to this Commonwealth belonging, or + in their possession, in any ship or ships, vessel or vessels whatsoever, but + in such as do truly and without fraud belong only to the people of this Commonwealth, + and in no other, except only such foreign ships and vessels as do truly and + properly belong to the people of that country or place, of which the said goods + are the growth, production or manufacture. + + Which of the following best describes the outcome of the Navigation Acts of + 1651? + + (A) They served as a catalyst for the growth of English shipping and overseas + trade, but did little to limit the prospects of the Dutch in the seventeenth + century. (B) They brought about almost immediate hardships for the Dutch economy + as their dominance of overseas trade quickly ended. (C) They were rescinded + during the restoration of the Stuarts as they sought normal diplomatic relations + with the Dutch so not as to need Parliament''s financial support for war. (D) + They led to nearly a century of recurrent war between England and the Netherlands, + which would not end until after American independence.' + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The Navigation Acts of 1651 helped English shipping by restricting + the ability of ships from other European countries, especially the Dutch, to + transport goods from colonies in Asia and Africa into England. The answer is + (A). + - question: "This question refers to the following information.\nIn Russia there was\ + \ nothing going on well, and [Souvarine] was in despair over the news he had\ + \ received. His old companions were all turning to the politicians; the famous\ + \ Nihilists who made Europe tremble-sons of village priests, of the lower middle\ + \ class, of tradesmen-could not rise above the idea of national liberation,\ + \ and seemed to believe that the world would be delivered-when they had killed\ + \ their despot&\u2026\n\"Foolery! They'll never get out of it with their foolery.\"\ + \nThen, lowering his voice still more, in a few bitter words he described his\ + \ old dream of fraternity. He had renounced his rank and his fortune; he had\ + \ gone among workmen, only in the hope of seeing at last the foundation of a\ + \ new society of labour in common. All the sous in his pockets had long gone\ + \ to the urchins of the settlement; he had been as tender as a brother with\ + \ the colliers, smiling at their suspicion, winning them over by his quiet workmanlike\ + \ ways and his dislike of chattering. But decidedly the fusion had not taken\ + \ place.\nHis voice changed, his eyes grew bright, he fixed them on \xE9tienne,\ + \ directly addressing him:\n\"Now, do you understand that? These hatworkers\ + \ at Marseilles who have won the great lottery prize of a hundred thousand francs\ + \ have gone off at once and invested it, declaring that they are going to live\ + \ without doing anything! Yes, that is your idea, all of you French workmen;\ + \ you want to unearth a treasure in order to devour it alone afterwards in some\ + \ lazy, selfish corner. You may cry out as much as you like against the rich,\ + \ you haven't got courage enough to give back to the poor the money that luck\ + \ brings you. You will never be worthy of happiness as long as you own anything,\ + \ and your hatred of the bourgeois proceeds solely from an angry desire to be\ + \ bourgeois yourselves in their place.\"\n\xE9mile Zola, French writer, Germinal,\ + \ 1885\nThe passage displays the direct concern for the welfare of the working\ + \ classes that was typically a part of which movement?\n(A) Capitalist (B) Scientific\ + \ (C) Communist (D) Existentialist" + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The modern Communist movement aims to establish a classless society + based on communal ownership and distribution of property and means of production, + thereby especially benefiting the working classes. The answer is (C). + - question: "This question refers to the following information.\nThe following excerpt\ + \ is from a pamphlet.\nYou will do me the justice to remember, that I have always\ + \ strenuously supported the Right of every man to his own opinion, however different\ + \ that opinion might be to mine. He who denies to another this right, makes\ + \ a slave of himself to his present opinion, because he precludes himself the\ + \ right of changing it.\nThe most formidable weapon against errors of every\ + \ kind is Reason. I have never used any other, and I trust I never shall.\n\ + The circumstance that has now taken place in France of the total abolition of\ + \ the whole national order of priesthood, and of everything appertaining to\ + \ compulsive systems of religion, and compulsive articles of faith, has not\ + \ only precipitated my intention, but rendered a work of this kind exceedingly\ + \ necessary, lest in the general wreck of superstition, of false systems of\ + \ government, and false theology, we lose sight of morality, of humanity, and\ + \ of the theology that is true.\nI believe in one God, and no more; and I hope\ + \ for happiness beyond this life.\nI believe in the equality of man; and I believe\ + \ that religious duties consist in doing justice, loving mercy, and endeavoring\ + \ to make our fellow-creatures happy.\nI do not believe in the creed professed\ + \ by the Jewish church, by the Roman church, by the Greek church, by the Turkish\ + \ church, by the Protestant church, nor by any church that I know of. My own\ + \ mind is my own church.\nAll national institutions of churches, whether Jewish,\ + \ Christian or Turkish, appear to me no other than human inventions, set up\ + \ to terrify and enslave mankind, and monopolize power and profit.\nI do not\ + \ mean by this declaration to condemn those who believe otherwise; they have\ + \ the same right to their belief as I have to mine.\n\u2014Thomas Paine, The\ + \ Age of Reason, 1794\u20131795\nWhich of the following Enlightenment philosophes\ + \ designed a system of checks and balances for government to avoid abuses of\ + \ power?\n(A) Jean Jacques Rousseau (B) Baron Montesquieu (C) Mary Wollstonecraft\ + \ (D) Adam Smith" + target: 'Let''s think step by step. We refer to Wikipedia articles on european + history for help. Baron Montesquieu was a 18th centrury French philsopher who + wrote extensively against the monoplization of power and advocated for a system + of checks and balances in government to prevent the rise of despotism. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96f4b365af04a3dc5c754def2899a5291ada1072 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml @@ -0,0 +1,53 @@ +dataset_name: high_school_geography +description: The following are multiple choice questions (with answers) about high + school geography. +fewshot_config: + sampler: first_n + samples: + - question: 'Which one of the following items is an example of nonmaterial culture? + + (A) Dove soap (B) Dove candy bar (C) Dove symbol (D) A dove (bird).' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. Nonmaterial culture consists of cultural ideas, beliefs or symbols + that are not physical objects. The answer is (C). + - question: 'During the third stage of the demographic transition model, which of the + following is true? + + (A) Birth rates increase and population growth rate is less rapid. (B) Birth + rates decline and population growth rate is less rapid. (C) Birth rates increase + and population growth rate increases. (D) Birth rates decrease and population + growth rate increases.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. The demographic transition model models the five different stages + of population growth as a country goes through economic development, where the + third stage refers to a period of declining birth rates and lower population + growth. The answer is (B). + - question: 'The practice of hiring a foreign third-party service provider to run an + operation is called + + (A) outsourcing. (B) offshoring. (C) maquiladoras. (D) locational interdependence.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. "Offshoring" literally means to move or base some of the activities + or processes of a company to a foreign country. The answer is (B). + - question: 'Which of the following statements is NOT accurate regarding the services + provided by local governments in the United States? + + (A) Duplication of efforts occurs often. (B) Social problems of the central + city spill over into the surrounding residential suburbs. (C) Inefficiency in + providing services occurs often. (D) One neighborhood''s efforts to reduce pollution + are always supported by neighboring communities.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. There may be economic, social or political reasons for two neighboring + communities and their local governments not agreeing to pollution reduction + efforts initiated by one of them. The answer is (D). + - question: 'The rate of natural increase of a population is found by subtracting the + + (A) crude death rate from the crude birth date. (B) crude birth rate from the + crude death rate. (C) doubling time from the crude birth rate. (D) fertility + rate from the crude death rate.' + target: 'Let''s think step by step. We refer to Wikipedia articles on geography + for help. The difference between number of births and deaths gives the population + increase at any given time. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c11772183c0266288f95f4d273ebbdf32c0dba1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,61 @@ +dataset_name: high_school_government_and_politics +description: The following are multiple choice questions (with answers) about high + school government and politics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following best states an argument made by James Madison in + The Federalist number 10? + + (A) Honest politicians can prevent factions from developing. (B) Factions are + more likely to occur in large republics than in small ones. (C) The negative + effects of factionalism can be reduced by a republican government. (D) Free + elections are the people''s best defense against factionalism.' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. In the Federalist number 10, James Madison advocated + for a representative republican form of government to guard against factionalism. + The answer is (C). + - question: 'The term "budget deficit" refers to the + + (A) annual increase in federal spending on the military (B) amount of interest + on the national debt (C) difference between the initial budget proposals made + by the president and Congress (D) amount the government spends in excess of + its revenues' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. When the goverment spends more than it earns, their difference + is the budget deficit. The answer is (D). + - question: 'Which of the following statements about cabinet departments is FALSE? + + (A) They are established by the legislative branch. (B) Their members often + don''t have much influence over presidential decisions. (C) They cannot all + be run by leaders who belong to the same political party the president does. + (D) Not every federal agency is a cabinet department.' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. There is no law stipulating that some cabinet department + leaders have to belong to a political party different from that of the president. + The answer is (C). + - question: 'Which of the following cases established the precedent that a defendant + must be informed of the right to remain silent, the right to a lawyer, and protection + from self-incrimination? + + (A) Weeks v. United States (B) Betts v. Brady (C) Mapp v. Ohio (D) Miranda v. + Arizona' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. In the landmark Miranda v. Arizona in 1966, the US Supreme + Court, based on the Fifth and Sixth Amendment of the US Constitution, guaranteed + a defendant's right to an attorney and protection from self-incrimination. The + answer is (D). + - question: 'Uncertainty over the limits to presidential power is caused primarily + by the fact that + + (A) the constitutional definition of those powers is broad and unspecific (B) + most people agree that the Constitution places too many limits on presidential + power (C) the Supreme Court consistently refuses to rule on cases concerning + presidential powers (D) constitutional amendments have greatly increased presidential + powers' + target: 'Let''s think step by step. We refer to Wikipedia articles on government + and politics for help. The US Constitution is not very specific about the powers + of the president, leading to uncertainty over its limits. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5700d5df946539608bbf10555e0f06ca07672b09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,53 @@ +dataset_name: high_school_macroeconomics +description: The following are multiple choice questions (with answers) about high + school macroeconomics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following policies best describes supply-side fiscal policy? + + (A) An increase in the money supply (B) Increased government spending (C) Lower + taxes on research and development of new technology (D) Higher taxes on household + income' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. Supply-side fiscal policy stimulates the economy by encouraging more + production of goods and services through reduction in taxes and deregulation. + The answer is (C). + - question: 'The short-run Phillips curve indicates a + + (A) direct relation between unemployment and inflation (B) direct relation between + price and quantity demanded (C) inverse relation between price and quantity + demanded (D) inverse relation between unemployment and inflation' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. The short-run Phillips curve shows that whenever unemployment decreases + below a natural level, the inflation starts increasing, and vice-versa. The + answer is (D). + - question: 'Holding all else equal which of the following monetary policies would + be used to boost U.S. exports? + + (A) Increasing the discount rate (B) Increasing the reserve ratio (C) Buying + government securities (D) Lowering tariffs' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. Buying government securities leads to reduction in demand for US dollars + from foreign buyers, thereby making it cheaper and hence making US exports more + attractive. The answer is (C). + - question: 'A federal deficit occurs when + + (A) exports exceed imports. (B) imports exceed exports. (C) federal tax collections + exceed spending. (D) federal spending exceeds federal tax revenues.' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. A federal deficit occurs when federal spending exceeds federal income + which is primarily from tax revenues. The answer is (D). + - question: 'Which of the following is not included in the U.S. GDP? + + (A) The U.S. military opens a new base in a foreign country with 1000 U.S. personnel. + (B) Japanese consumers buy thousands of CDs produced in the United States. (C) + An American pop singer performs a sold-out concert in Paris. (D) A French theatrical + production tours dozens of American cities.' + target: 'Let''s think step by step. We refer to Wikipedia articles on macroeconomics + for help. The economic transactions related to the performance of the American + pop-singer in Paris happens entirely outside the U.S. and hence is not included + in the GDP numbers. The answer is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3e05795561a11dca35dd342bc88b4794584809c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml @@ -0,0 +1,51 @@ +dataset_name: high_school_mathematics +description: The following are multiple choice questions (with answers) about high + school mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Simplify and write the result with a rational denominator: $$\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}$$ + + (A) \frac{3\sqrt{3}}{3} (B) \frac{1}{3} (C) \sqrt{3} (D) \frac{\sqrt{3}}{3}' + target: Let's think step by step. Factoring $729=3^6$ and combining the roots + $\frac{1}{2}\frac{1}{3}\frac{1}{2}=\frac{1}{12}$, we get that $\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}=\left(\frac{1}{3^6}\right)^{\frac{1}{12}}=\frac{1}{3^{\frac{1}{2}}}=\frac{3}{\sqrt{3}}$ + The answer is (D). + - question: 'Five thousand dollars compounded annually at an $x\%$ interest rate takes + six years to double. At the same interest rate, how many years will it take + $\$300$ to grow to $\$9600$? + + (A) 12 (B) 1 (C) 30 (D) 5' + target: Let's think step by step. To go from $\$300$ to $\$9600$, the value must + go up by a factor of $9600/300=32=2^5$. Since at this interest rate it takes + six years for it to double, it will take $5*6=30$ years to grow to $\$9600$. + The answer is (C). + - question: "Ten students take a biology test and receive the following scores: 45,\ + \ 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students\u2019\ + \ test scores?\n(A) 55 (B) 60 (C) 62 (D) 65" + target: Let's think step by step. There are 10 students and the sum of their scores + is $45 + 55 + 50 + 70 + 65 + 80 + 40 + 90 + 70 + 85 = 650$, the mean is $650/10=65$. + The answer is (D). + - question: 'The variable $x$ varies directly as the square of $y$, and $y$ varies + directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is + the value of $x$ when $z$ equals $\frac{1}{2}$? + + (A) -1 (B) 16 (C) -\frac{1}{256} (D) \frac{1}{16}' + target: Let's think step by step. We know that $x \propto y^2$ and $y \propto + z^3$, so $x = k z^6$ for some constant $k$. Plugging in for $x=-16$ and $z=2$, + the constant value is $k=\frac{x}{z^6}=\frac{-16}{64}=-\frac{1}{4}$. So, when + $z=\frac{1}{2}$, the value of $x$ is $x=kz^6=-\frac{1}{4}\frac{1}{2^6}=-\frac{1}{256}$. + The answer is (C). + - question: 'Joe was in charge of lights for a dance. The red light blinks every two + seconds, the yellow light every three seconds, and the blue light every five + seconds. If we include the very beginning and very end of the dance, how many + times during a seven minute dance will all the lights come on at the same time? + (Assume that all three lights blink simultaneously at the very beginning of + the dance.) + + (A) 3 (B) 15 (C) 6 (D) 5' + target: 'Let''s think step by step. The least common multiple of 2, 3 and 5 is + 30, so during a 7 minute dance, all the three lights will come on at the same + time $2*7+1=15$ times. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ead78be898187a7c08c292eaeebfe6b067ef4413 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,56 @@ +dataset_name: high_school_microeconomics +description: The following are multiple choice questions (with answers) about high + school microeconomics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is necessarily a characteristic of oligopoly? + + (A) Free entry into and exit from the market (B) A few large producers (C) One + producer of a good with no close substitutes (D) A homogenous product' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. An oligopoly is when a market is dominated by just one or a few number + of sellers or producers. To get oligopoly, the market should have high barriers + to new entry, and the product has differentiation. The answer is (B). + - question: 'If the government subsidizes producers in a perfectly competitive market, + then + + (A) the demand for the product will increase (B) the demand for the product + will decrease (C) the consumer surplus will increase (D) the consumer surplus + will decrease' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. (A) and (B) are wrong because the demand curve does not change at + all. If the government subsidizes producers, the supply will increase, and thus + the consumer surplus also increases. The answer is (C). + - question: 'Which of the following is true of a price floor? + + (A) The price floor shifts the demand curve to the left. (B) An effective floor + creates a shortage of the good. (C) The price floor shifts the supply curve + of the good to the right. (D) To be an effective floor, it must be set above + the equilibrium price.' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. Price floor does not shift the demand or shift curve. An effective + price floor should be set above the equilibrium price, otherwise the market + bears and the floor does not have effective effect. The answer is (D). + - question: 'The concentration ratio for a monopoly is + + (A) 0 (B) 5 (C) 10 (D) 100' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. The concentration ratio is calculated as the sum of market share of + a specific number of largest companies. Monopoly means one company or entity + controls the entire market, therefore, the concentration ratio is 100 percent. + The answer is (D). + - question: 'In a competitive labor market for housepainters, which of the following + would increase the demand for housepainters? + + (A) An effective minimum wage imposed on this labor market. (B) An increase + in the price of gallons of paint. (C) An increase in the construction of new + houses. (D) An increase in the price of mechanical painters so long as the output + effect exceeds the substitution effect.' + target: 'Let''s think step by step. We refer to Wikipedia articles on microeconomics + for help. An increase in the construction of new houses means an increase demand + of in-house painting, thus increases the demand for housepainters. The answer + is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d2166b8595529213797cf12efd67f90ca4e9e61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml @@ -0,0 +1,50 @@ +dataset_name: high_school_physics +description: The following are multiple choice questions (with answers) about high + school physics. +fewshot_config: + sampler: first_n + samples: + - question: 'A microwave oven is connected to an outlet, 120 V, and draws a current + of 2 amps. At what rate is energy being used by the microwave oven? + + (A) 10 W (B) 30 W (C) 60 W (D) 240 W' + target: Let's think step by step. Rate of energy usage is known as power; in an + dissipative electrical circuit, power is given by voltage times current. So + in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D). + - question: "A point charge, Q = +1 mC, is fixed at the origin. How much work is required\ + \ to move a charge, Q = +8 \xB5C, from the point (0, 4 meters) to the point\ + \ (3 meters, 0)?\n(A) 3.5 J (B) 6.0 J (C) 22.5 J (D) 40 J" + target: "Let's think step by step. To calculate the work required to move a charge\ + \ from one location to another in a fixed electric field, it is enough to calculate\ + \ the potential difference between the two locations. Here, the potential only\ + \ depends on the distance between the charges; it\u2019s $k q_1 q_2 / r$, where\ + \ $k$ is Coulomb\u2019s constant. Plugging in values $q_1 = $ 1 mC, $q_2 = 8\ + \ \\mu$ C, gives the answer as 5.992 J, which rounds to 6 J. The answer is (B)." + - question: 'Which of the following conditions will ensure that angular momentum is + conserved? I. Conservation of linear momentum II. Zero net external force III. + Zero net external torque + + (A) I and II only (B) I and III only (C) II and III only (D) III only' + target: Let's think step by step. Torque is defined as the change in angular momentum; + if there is zero external torque, angular momentum is conserved. The answer + is (D). + - question: "A photocell of work function \u03D5 = 2eV is connected to a resistor in\ + \ series. Light of frequency f = 1 \xD7 10^15 Hz hits a metal plate of the photocell.\ + \ If the power of the light is P = 100 W, what is the current through the resistor?\n\ + (A) 2:00 AM (B) 6:00 AM (C) 12:00 AM (D) 24 A" + target: Let's think step by step. The only answer above which has units of current + is D, 24 A. The answer is (D). + - question: "A pipe full of air is closed at one end. A standing wave is produced in\ + \ the pipe, causing the pipe to sound a note. Which of the following is a correct\ + \ statement about the wave\u2019s properties at the closed end of the pipe?\n\ + (A) The pressure is at a node, but the particle displacement is at an antinode.\ + \ (B) The pressure is at an antinode, but the particle displacement is at a\ + \ node. (C) The pressure and the particle displacement are both at nodes. (D)\ + \ The pressure and the particle displacement are both at antinodes." + target: 'Let''s think step by step. At the closed end of the pipe, the particles + cannot have any net displacement because the pipe closure stops them. So the + particle displacement is at a node. This closure also causes the pressure to + be maximal, i.e. an antinode. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..553fe18de24f14765d14d0fb2dbdfaf72449af32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml @@ -0,0 +1,64 @@ +dataset_name: high_school_psychology +description: The following are multiple choice questions (with answers) about high + school psychology. +fewshot_config: + sampler: first_n + samples: + - question: 'Pascale is interested in the processing strategies children use to learn + new information. Pascale would best be classified as what type of psychologist? + + (A) sociocultural (B) clinical (C) cognitive (D) behaviorist' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Sociocultural psychologist focuses on the effect of societal factors + on people. Clinical psychologist focuses on people with mental issues. Cognitive + psychologist focuses on how people think and learn, including the processing + strategies. Behaviorist focuses more on the environment and experience effect + on people. The answer is (C). + - question: 'According to Caplan''s model of consultee-centered case consultation, + the consultant is primarily interested in + + (A) identifying the causes and solutions of the client''s presenting problems + (B) identifying and eliminating the causes of the consultee''s difficulties + in handling a problem (C) establishing a hierarchy of authority to enable effective + decision making (D) presenting a single, well-defined and unambiguous course + of action for the consultant to overcome skills deficits' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Caplan defines two type of consultation. Client-centered case consultation + aims to handle client's problems, while consultee-centered case consultation + aims to identify the reason of client's difficulty to solve problems. The answer + is (B). + - question: 'According to the Individuals with Disabilities Education Improvement Act, + which of the following must an educational agency do before it changes the educational + placement of a student with a disability? + + (A) Give the child a trial period in the new environment (B) Notify the parents + in writing (C) Obtain school board approval (D) Obtain parental consent' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. When the decision to change the educational placement of a student + with a disability is made, the educational agency must notify the parents in + writing on that date. The answer is (B). + - question: 'While swimming in the ocean, Ivan is frightened by a dark shadow in the + water even before he has the chance to identify what the shadow is. The synaptic + connections taking place during this incident of fright are best described by + which of the following? + + (A) Messages are sent from the thalamus directly to the amygdala. (B) Messages + are sent from the thalamus to the "what" and "where" pathways. (C) Messages + are sent from the parasympathetic nervous system to the cerebral cortex. (D) + Messages are sent from the frontal lobes to the pituitary gland.' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Our neural system has a mechanism that can respond immediate emotional + signal before going to the thought center. In the Ivan's case, messages travel + directly from thalamus to amygdala. The answer is (A). + - question: 'Ani believes that her attitudes and behavior play a central role in what + happens to her. Such a belief is likely to be associated with + + (A) a strong superego. (B) low self-esteem. (C) low self-efficacy. (D) an internal + locus of control.' + target: 'Let''s think step by step. We refer to Wikipedia articles on psychology + for help. People with an external locus of control believes fate and luck play + an important role in their lives, while people with an internal locus of control + believes they control their lives. The answer is (D).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66ed702a6b9b57dc69719a13923e04ebc8218ea9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml @@ -0,0 +1,81 @@ +dataset_name: high_school_statistics +description: The following are multiple choice questions (with answers) about high + school statistics. +fewshot_config: + sampler: first_n + samples: + - question: 'A new smartwatch is manufactured in one part of a factory, then secured + for shipping in another, independent part of the factory. The weight of the + smartwatch has a mean of 62 grams and a standard deviation of 1.0 grams. The + weight of the packaging (box, user''s guide, bubble wrap, etc.) has a mean of + 456 grams and a standard deviation of 6 grams. Together, the distribution of + the weight of the smartwatch and its packaging would have the following mean + and standard deviation: + + (A) Mean 518 grams; standard deviation 7.0 grams (B) Mean 518 grams; standard + deviation 3.5 grams (C) Mean 518 grams; standard deviation 6.1 grams (D) Mean + 394 grams; standard deviation 6.1 grams' + target: Let's think step by step. Since the weight of the watch and the weight + of the packaging are independent random variables, the mean and variance of + their sum is equal to the sum of their individual means and variances. So the + mean is 62 + 456 = 518 grams, and the variances is 1.0^2 + 6.0^2 = 37, leading + to a standard deviation of 6.1 grams. The answer is (C). + - question: 'After a frost warning was issued, the owner of a large orange grove asked + his workers to spray all his trees with water. The water was supposed to freeze + and form a protective covering of ice around the orange blossom. Nevertheless, + the owner suspected that some trees suffered considerable damage due to the + frost. To estimate the proportion of trees that suffered more than 50 percent + damage due to the frost, he took a random sample of 100 trees from his grove. + What is the response variable in this experiment? + + (A) The proportion of trees that suffered more than 50 percent damage due to + frost. (B) The number of trees affected by the frost. (C) The number of trees + sampled from the grove. (D) For each sampled tree, whether it suffered more + than 50 percent damage or at most 50 percent damage.' + target: Let's think step by step. In this experiment, the response variable is + what is measured. For each tree, what is measured is whether or not it suffered + more than 50 percent damage due to the frost. The answer is (D). + - question: 'Suppose X and Y are random variables with E(X) = 37, var(X) = 5, E(Y) + = 62, and var(Y) = 12. What are the expected value and variance of the random + variable X + Y? + + (A) E(X + Y) = 99, var(X + Y) = 8.5 (B) E(X + Y) = 99, var(X + Y) = 13 (C) E(X + + Y) = 99, var(X + Y) = 17 (D) There is insufficient information to answer this + question.' + target: Let's think step by step. While means of sums of random variables add + (regardless of whether the variables are independent) in order to determine + the variance of a sum of random variables, we need to know not just their individual + variances but the covariance of the two variables, which is not given in this + problem. The answer is (D). + - question: 'Which of the following sets has the smallest standard deviation? Which + has the largest? + + I: {1,2,3} + + II: {-10,10} + + III: {100} + + (A) I, II (B) II, III (C) III, I (D) III, II' + target: Let's think step by step. The variance of distribution I is the expected + squared deviation from its mean (which is 2), so the variance is 2/3 . The variance + of distribution II is 10^2 (because both elements are 10 away from the mean + of zero). The variance of distribution III is 0, since it has a single entry. + So distribution III has the smallest standard deviation and distribution II + has the largest. The answer is (D). + - question: 'Which of the following is a correct statement about correlation? + + (A) If the slope of the regression line is exactly 1, then the correlation is + exactly 1. (B) If the correlation is 0, then the slope of the regression line + is undefined. (C) Switching which variable is called x and which is called y + changes the sign of the correlation. (D) The correlation r is equal to the slope + of the regression line when z-scores for the y-variable are plotted against + z-scores for the x-variable.' + target: 'Let''s think step by step. Statement A is false because the slope of + the regression line being exactly 1 can occur even when the two variables are + not perfectly correlated. Statement B is false because uncorrelated variables + regression lines can have slope zero. Statement C is false because correlation + is symmetric in the two random variables. The answer is (D).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cea5109f6570086dce3cf1815dc50f1889d80ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml @@ -0,0 +1,156 @@ +dataset_name: high_school_us_history +description: The following are multiple choice questions (with answers) about high + school us history. +fewshot_config: + sampler: first_n + samples: + - question: "This question refers to the following information.\nI come not to urge\ + \ personal claims, nor to seek individual benefits; I appear as the advocate\ + \ of those who cannot plead their own cause; I come as the friend of those who\ + \ are deserted, oppressed, and desolate. In the Providence of God, I am the\ + \ voice of the maniac whose piercing cries from the dreary dungeons of your\ + \ jails penetrate not your Halls of Legislation. I am the Hope of the poor crazed\ + \ beings who pine in the cells, and stalls, and cages, and waste rooms of your\ + \ poor-houses. I am the Revelation of hundreds of wailing, suffering creatures,\ + \ hidden in your private dwellings, and in pens and cabins\u2014shut out, cut\ + \ off from all healing influences, from all mind-restoring cares.\u2026 Could\ + \ their melancholy histories be spread before you as revealed to my grieved\ + \ spirit during the last three months, how promptly, how earnestly would you\ + \ search out the most approved means of relief; how trifling, how insignificant,\ + \ by comparison, would appear the sacrifices you are asked to make; how would\ + \ a few dimes and dollars, gathered from each citizen, diminish in value as\ + \ a possession, compared with the certain benefits and vast good to be secured\ + \ for the suffering insane...by the consecration and application of a sufficient\ + \ fund to the construction of a suitable hospital.\u2026\n\u2014Dorothea Dix,\ + \ Memorial Soliciting a State Hospital for the Protection and Cure of the Insane,\n\ + Submitted to the General Assembly of North Carolina, November 1848\nDorothea\ + \ Dix can best be compared to whom?\n(A) Abigail Adams (B) Clara Barton (C)\ + \ Shirley Temple (D) Hillary Clinton" + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. Both Dorothea Dix and Clara barton are American nurses. The answer + is (B). + - question: "This question refers to the following information.\n\"As our late Conduct\ + \ at the Conestoga Manor and Lancaster have occasioned much Speculation & a\ + \ great diversity of Sentiments in this and neighboring Governments; some vindicating\ + \ & others condemning it; some charitably alleviating the Crime, & others maliciously\ + \ painting it in the most odious & detestable Colours, we think it our duty\ + \ to lay before the Publick, the whole Matter as it appeared, & still appears,\ + \ to us. . . .\n\"If these things are not sufficient to prove an unjustifiable\ + \ Attachment in the Quakers to the Indians Savages, a fixed Resolution to befriend\ + \ them & an utter insensibility to human Distresses, let us consider a few more\ + \ recent Facts. When we found the last Summer that we were likely to get no\ + \ Assistance from the Government, some Volunteers went out at our own Expense,\ + \ determined to drive our Enemies from our Borders; & when we came near to the\ + \ great Island, we understood that a Number of their Warriors had gone out against\ + \ our Frontiers. Upon this we returned and came up with them and fought with\ + \ them at the Munfey Hill where we lost some of our Men & killed some of their\ + \ Warriors & thereby saved our Frontiers from this Story in another Expedition.\ + \ But no sooner had we destroyed their Provisions on the great Island, & ruined\ + \ their trade with the good People at Bethlehem, but these very Indians, who\ + \ were justly suspected of having murdered our Friends in Northampton County,\ + \ were by the Influence of some Quakers taken under the Protection of the Government\ + \ to screen them from the Resentments of the Friends and Relations of the Murdered,\ + \ & to support them thro the Winter.\"\n\u2014\"Apology of the Paxton Boys\"\ + \ (pamphlet), 1764 (Note: \"apology\" in this context should be read as an explanation,\ + \ not an admission of guilt or regret.\nThe sentiments expressed in the explanation\ + \ above reflect which of the ongoing tensions during the colonial period of\ + \ American history?\n(A) Tensions between British policies and the aspirations\ + \ of North American colonists. (B) Tensions between American Indians allied\ + \ with the French and those allied with the British. (C) Tensions between freed\ + \ African Americans and white planters. (D) Tensions between backcountry settlers\ + \ and elites within colonial America." + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. After the French and Indian War, the Scotch-Irish settlers attacked + American Indians. After the attacks on the Conestoga, about 250 Paxton Boys + present their grievances to the Pennsylvania legislature. As mentioned in the + information, the Paxton Boys cited resentiment at local elites. The answer is + (D). + - question: "This question refers to the following information.\nOur leaders talk about\ + \ stopping aggression from the north, but this was a struggle among groups of\ + \ Vietnamese until we intervened. We seem bent upon saving the Vietnamese from\ + \ Ho Chi Minh even if we have to kill them and demolish their country to do\ + \ it. As the native people survey bombed-out villages, women and children burned\ + \ by napalm, rice crops destroyed and cities overrun with our military personnel,\ + \ they are doubtless saying secretly of the Vietcong guerillas and of the American\ + \ forces, \"A plague on both your houses.\" \u2026 Stop the bombing, north and\ + \ south, end search and destroy offensive sweeps, and confine our military action\ + \ to holding operations on the ground. Bombing the north has failed to halt\ + \ or seriously check the flow of troops to the south and may, in fact, have\ + \ prompted a much greater war effort by Hanoi.\n\u2014Senator George McGovern,\ + \ \"The Lessons of Vietnam,\" April 25, 1967\nWhich of the following opinions\ + \ from the 1960s most directly reflects the perspective of George McGovern's\ + \ speech?\n(A) Americans must maximize their technological edge in Vietnam.\ + \ (B) American bombing in Vietnam is step by step leading to progress in the\ + \ war. (C) American bombing in Vietnam is a failure. (D) America must not give\ + \ in to defeatism about the war in Vietnam." + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. "Stop the bombing" and "Bombing the north has failed to halt or seriously + check the flow of troops to the south" indicate that the perspective of George + McGovern's speech is that Amerian bombing in Vietnam is a failure. The answer + is (C). + - question: "This question refers to the following information.\n\"In the new Code\ + \ of Laws which I suppose it will be necessary for you to make I desire you\ + \ would Remember the Ladies, and be more generous and favorable to them than\ + \ your ancestors. Do not put such unlimited power into the hands of the Husbands.\ + \ Remember all Men would be tyrants if they could. If particular care and attention\ + \ is not paid to the Ladies we are determined to foment a Rebellion, and will\ + \ not hold ourselves bound by any Laws in which we have no voice, or Representation.\"\ + \nAbigail Adams, in a letter to John Adams, 1776\n\"Special legislation for\ + \ woman has placed us in a most anomalous position. Women invested with the\ + \ rights of citizens in one section\u2014voters, jurors, office-holders\u2014\ + crossing an imaginary line, are subjects in the next. In some States, a married\ + \ woman may hold property and transact business in her own name; in others,\ + \ her earnings belong to her husband. In some States, a woman may testify against\ + \ her husband, sue and be sued in the courts; in others, she has no redress\ + \ in case of damage to person, property, or character. In case of divorce on\ + \ account of adultery in the husband, the innocent wife is held to possess no\ + \ right to children or property, unless by special decree of the court. But\ + \ in no State of the Union has the wife the right to her own person, or to any\ + \ part of the joint earnings of the co-partnership during the life of her husband.\ + \ In some States women may enter the law schools and practice in the courts;\ + \ in others they are forbidden. In some universities girls enjoy equal educational\ + \ advantages with boys, while many of the proudest institutions in the land\ + \ deny them admittance, though the sons of China, Japan and Africa are welcomed\ + \ there. But the privileges already granted in the several States are by no\ + \ means secure.\"\nSusan B. Anthony, \"Declaration of Rights for Women,\" July\ + \ 4, 1876\nThe sentiments expressed in the second excerpt by Susan B. Anthony\ + \ are most likely in support of\n(A) the Equal Rights Amendment (B) universal\ + \ suffrage (C) states' rights (D) prohibition" + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. The above information mentioned that women are in an anomalous position + in terms of legislation. Women's earnings do not belong to themselves, or they + cannot testify against her husbands. Susan believes women should have equal + legal rights as men. The answer is (B). + - question: 'This question refers to the following information. + + "Society in every state is a blessing, but government even in its best state + is but a necessary evil; in its worst state an intolerable one; for when we + suffer, or are exposed to the same miseries by a government, which we might + expect in a country without government, our calamity is heightened by reflecting + that we furnish the means by which we suffer. Government, like dress, is the + badge of lost innocence; the palaces of kings are built on the ruins of the + bowers of paradise. For were the impulses of conscience clear, uniform, and + irresistibly obeyed, man would need no other lawgiver; but that not being the + case, he finds it necessary to surrender up a part of his property to furnish + means for the protection of the rest; and this he is induced to do by the same + prudence which in every other case advises him out of two evils to choose the + least. Wherefore, security being the true design and end of government, it unanswerably + follows that whatever form thereof appears most likely to ensure it to us, with + the least expense and greatest benefit, is preferable to all others." + + Thomas Paine, Common Sense, 1776 + + Which of the following "miseries" alluded to above were most condemned by Anti-Federalists + of the post-Revolutionary era? + + (A) Organized response to Bacon''s Rebellion (B) Federal response to Shays''s + Rebellion (C) Federal response to the Whiskey Rebellion (D) Federal response + to Pontiac''s Rebellion' + target: 'Let''s think step by step. We refer to Wikipedia articles on us history + for help. Anti-Federalists do not believe centralized government power, and + suspect Washington''s military response to Whiskey Rebellion. Bacon''s Rebellion + and Pontiac''s Rebellion happen before the Revolution and they can be ruled + out. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fbdaf05c137270f4ff4207e7c6ce81c2a34d30c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml @@ -0,0 +1,100 @@ +dataset_name: high_school_world_history +description: The following are multiple choice questions (with answers) about high + school world history. +fewshot_config: + sampler: first_n + samples: + - question: "This question refers to the following information.\n\"At least one of\ + \ the [world's] societies would have to somehow enormously increase its productivity\ + \ [in order to achieve global hegemony]. That quantum jump would have to be\ + \ made before the various scientific, technological, agricultural, and industrial\ + \ revolutions on which our post-quantum-leap world rests. It could only be accomplished\ + \ by exploiting the ecosystems, mineral resources, and human assets of whole\ + \ continents outside the lands of the society making the jump. Western Europe\ + \ did just that by means of its brutality and guns and, more important, by geographical\ + \ and ecological luck.\"\nCopyright \xA9 2015 Cambridge University Press.\n\ + Alfred Crosby, historian, Ecological Imperialism, 2004\nThe \"quantum jump\"\ + \ mentioned in the passage most directly contributed to which of the following\ + \ developments in the period 1450\u20131750 C.E.?\n(A) A breakdown in trade\ + \ routes through the collapse of the established state structure (B) An increase\ + \ in the population of the world through more plentiful supplies of food (C)\ + \ The spread of Chinese and Indian belief systems across the world (D) An increase\ + \ in social unrest" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The "quantum jump" mentioned in the passage refers to the conquest + of the New World and the Columbian Exchange. Choice (A) and (C) did not happen + in history. Choice (C) refers to the human assets. The answer is (B). + - question: "This question refers to the following information.\n\"The struggle against\ + \ neo-colonialism is not aimed at excluding the capital of the developed world\ + \ from operating in less developed countries. It is aimed at preventing the\ + \ financial power of the developed countries being used in such a way as to\ + \ impoverish the less developed.\nNon-alignment, as practiced by Ghana and many\ + \ other countries, is based on co-operation with all States whether they be\ + \ capitalist, socialist or have a mixed economy. Such a policy, therefore, involves\ + \ foreign investment from capitalist countries, but it must be invested in accordance\ + \ with a national plan drawn up by the government of the non-aligned State with\ + \ its own interests in mind. The issue is not what return the foreign investor\ + \ receives on his investments\u2026The question is one of power. A State in\ + \ the grip of neo-colonialism is not master of its own destiny.\"\nKwame Nkrumah,\ + \ Neo-Colonialism, 1965\nWhich of the following provides the best context for\ + \ Nkrumah's writings?\n(A) The Industrial Revolution (B) Decolonization (C)\ + \ Regional Free Trade Associations (D) Autarky" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage expresses a point that the successful fight against neo-colonialism + were in danger and the newly independent nations like Ghana may be re-colonized + via financial power of the developed countries. The answer is (B). + - question: "This question refers to the following information.\n\"Indeed, as both\ + \ the fatwas of distinguished [scholars] who base their opinion on reason and\ + \ tradition alike and the consensus of the Sunni community agree that the ancient\ + \ obligation of extirpation, extermination, and expulsion of evil innovation\ + \ must be the aim of our exalted aspiration, for \"Religious zeal is a victory\ + \ for the Faith of God the Beneficent\"; then, in accordance with the words\ + \ of the Prophet (Peace upon him!) \"Whosoever introduces evil innovation into\ + \ our order must be expelled\" and \"Whosoever does aught against our order\ + \ must be expelled,\" action has become necessary and exigent\u2026\"\nLetter\ + \ from Ottoman Sultan Selim I to Safavid Shah Ismail I, 1514\nThe letter from\ + \ Selim I is most clearly an example of which of the following?\n(A) The maintenance\ + \ of military supremacy at all costs (B) Expanding tensions between religious\ + \ sects (C) Factors that brought about the collapse of the Ottoman Empire (D)\ + \ Peacemaking efforts among the Islamic empires" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage is an example of expanding tensions between Selim and + Ismail. In the passage the Selim references the fatwa and the consensus of the + Sunni community to against whosoever introduces evil. The answer is (B). + - question: 'This question refers to the following information. + + "The real grievance of the worker is the insecurity of his existence; he is + not sure that he will always have work, he is not sure that he will always be + healthy, and he foresees that he will one day be old and unfit to work. If he + falls into poverty, even if only through a prolonged illness, he is then completely + helpless, exam_ins to his own devices, and society does not currently recognize + any real obligation towards him beyond the usual help for the poor, even if + he has been working all the time ever so faithfully and diligently. The usual + help for the poor, however, leaves a lot to be desired, especially in large + cities, where it is very much worse than in the country." + + Otto von Bismarck, 1884 + + Otto von Bismarck likely made this speech in reaction to which of the following + issues? + + (A) Social acceptance of child labor (B) Declining life expectancy in Germany + (C) Criticisms of German trade tariffs (D) Negative effects attributed to industrial + capitalism' + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage talks about the grievance of the work under the industrial + capitalism. The answer is (D). + - question: "This question refers to the following information.\nHe contains all works\ + \ and desires and all perfumes and all tastes. He enfolds the whole universe\ + \ and in silence is loving to all. This is the Spirit that is in my heart, this\ + \ is Brahman. To him I shall come when I go beyond this life, and to him will\ + \ come he who has faith and doubts not.\n\u2014The Upanishads, India, c. 1000\ + \ BCE\nTo which religion does the speaker most likely belong?\n(A) Hinduism\ + \ (B) Buddhism (C) Shintoism (D) Zoroastrianism" + target: 'Let''s think step by step. We refer to Wikipedia articles on world history + for help. Brahman refers to the ultimate reality of all things in the Hindu + religion. In contrast, Buddhism does not have a concept of supreme God. The + answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eec010845fa68ad974bdb7cd922a0028365d96e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml @@ -0,0 +1,42 @@ +dataset_name: human_aging +description: The following are multiple choice questions (with answers) about human + aging. +fewshot_config: + sampler: first_n + samples: + - question: 'All other things being equal, which of the following persons is more likely + to show osteoporosis? + + (A) An older Hispanic American woman (B) An older African American woman (C) + An older Asian American woman (D) An older Native American woman' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. Although osteoporosis can occur at any age, the risk is higher for + older people. It is most common in Asian and non-Hispanic white women. The answer + is (C). + - question: 'The finding that adults tend to remember events from their adolescence + better than from other periods in their lives is referred to as the + + (A) Adolescence advantage (B) Reminiscence bump (C) Memorial memorial (D) Quadratic + retrieval spike' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. Reminiscence bump is a phenomenon that older adults tend to recollect + events during their young ages. People usually have a period of childhood amnesia + from birth to around age 5, and a reminiscence bump between 10 and 30. The answer + is (B). + - question: 'Which element in tobacco smoke is responsible for cancers? + + (A) Nicotine (B) Tar (C) Carbon monoxide (D) Smoke particles' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. The benzene, acrylamide and acrylonitrile in tar interact with the + lungs and cause DNA mutations in cells of the lungs, and lead to cancer. The + answer is (B). + - question: 'When older adults move to a new state after retirement, which of the following + is the more likely destination? + + (A) Texas (B) California (C) Hawaii (D) Vermont' + target: 'Let''s think step by step. We refer to Wikipedia articles on human aging + for help. Texas does not have state tax, and has low cost of living compared + with the other three options. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dab78f0c5ec1042d23240bb71f59b212885585aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml @@ -0,0 +1,50 @@ +dataset_name: human_sexuality +description: The following are multiple choice questions (with answers) about human + sexuality. +fewshot_config: + sampler: first_n + samples: + - question: 'The most common disorder among men who seek sexual therapy is: + + (A) premature ejaculation (B) inhibited ejaculation (C) erectile disorder (D) + ejaculatory disorder' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. The most common type of arousal disorder for men is erectile dysfunction, + meaning that a person is not able to get any physical satisfaction from sexual + activity although he may be interested in it. The answer is (C). + - question: 'A woman who knows she has active herpes and untreated syphilis but continues + to have sex without informing her partners of her condition has, in psychoanalytic + terms: + + (A) a strong ego (B) a weak superego (C) a weak id (D) a strong superego' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. A person with weak superego tends to be delinquent, criminal or have + antisocial personality. The action of the woman who knows she has active venereal + disease but still have sex with her partners indicate she may has antisocial + personality. The answer is (B). + - question: 'The nature of homosexual activities that occur during preadolescence include + all but which of the following? + + (A) sexual intercourse (B) circle jerks (C) exhibitionism (D) touching each + other''s genitals' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. Sexual intercourse is prohibited during preadolescence. The answer + is (A). + - question: 'Women''s ability to have multiple orgasms is primarily due to: + + (A) the fact that they do not have a refractory period. (B) the response of + the inner layers of the vagina. (C) having alternating orgasms in different + locations. (D) the G-Spot.' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. The refractory period is the time when a person is not able to be + erect or is not interested in sex. The answer is (A). + - question: 'Morning sickness is typically a problem: + + (A) during the first trimester (B) during the second trimester (C) during the + third trimester (D) all through the pregnancy' + target: 'Let''s think step by step. We refer to Wikipedia articles on human sexuality + for help. Morning sickness usually begins by nine weeks after conception, corresponding + to the first trimester. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99341f395352d7b6a9a8d1a71005ca821ac9b723 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml @@ -0,0 +1,70 @@ +dataset_name: international_law +description: The following are multiple choice questions (with answers) about international + law. +fewshot_config: + sampler: first_n + samples: + - question: 'How the consent to be bound of a State may be expressed? + + (A) The consent of a State to be bound is expressed only by ratification (B) + The consent of a state to be bound by a treaty may be expressed by signature, + ratification, acceptance, approval or accession (C) The consent of a State to + be bound is expressed by signature (D) The consent of a State to be bound is + expressed by whatever means they choose' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. Article 11 of Vienna Convention on the Law of Treaties signed + in 1969 states that "the consent of a State to be bound by a treaty may be expressed + by signature, exchange of instruments constituting a treaty, ratification, acceptance, + approval or accession, or by any other means if so agreed." (B) is the most + precise and accurate answer. The answer is (B). + - question: 'What is the judge ad hoc? + + (A) If a party to a contentious case before the ICJ does not have a national + sitting as judge, it is entitled to nominate someone as a judge solely for that + case, with the title of judge ad hoc (B) Judge ad hoc is the member of the bench + of the ICJ with a casting vote (C) Judge ad hoc is a surrogate judge, in case + a judge is disqualified or passes away (D) Judge ad hoc is the judge that each + party will always nominate in every contentious case' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. As "ad hoc" implies, a judge ad hoc is appointed only for a specific + case or period, when a party to a contentious case before the International + Court of Justice does not have a regular national sitting as judge. The answer + is (A). + - question: 'When ''consent'' can serve as a circumstance precluding the wrongfulness + of a State conduct? + + (A) Consent can serve as a circumstance precluding the wrongfulness whenever + it is given (B) Consent can never serve as a circumstance precluding wrongfulness + (C) Consent can serve as a circumstance precluding wrongfulness, provided the + consent is valid and to the extent that the conduct remains within the limits + of the consent given (D) Consent can always serve as a circumstance precluding + wrongfulness, no matter which organ of the State gives it' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. Valid consent can serve as a circumstance precluding the wrongfulness + of a State conduct if the conduct remains within the limits of that consent, + according to Chapter V of the Responsibility of States for Internationally Wrongful + Acts, 2001, United Nations. The answer is (C). + - question: 'Would a reservation to the definition of torture in the ICCPR be acceptable + in contemporary practice? + + (A) This is an acceptable reservation if the reserving country''s legislation + employs a different definition (B) This is an unacceptable reservation because + it contravenes the object and purpose of the ICCPR (C) This is an unacceptable + reservation because the definition of torture in the ICCPR is consistent with + customary international law (D) This is an acceptable reservation because under + general international law States have the right to enter reservations to treaties' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. For it contravenes the object and purpose of the ICCPR, this is + an unacceptable reservation in contemporary practice. The answer is (B). + - question: 'What types of force does Article 2(4) of the UN Charter prohibit? + + (A) Article 2(4) encompasses only armed force (B) Article 2(4) encompasses all + types of force, including sanctions (C) Article 2(4) encompasses all interference + in the domestic affairs of States (D) Article 2(4) encompasses force directed + only against a State''s territorial integrity' + target: 'Let''s think step by step. We refer to Wikipedia articles on international + law for help. Article 2(4) of the UN Charter prohibits states from using armed + forces in their international relations. The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c714f7e595a48d28b750479a46183a02fc24dc0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml @@ -0,0 +1,59 @@ +dataset_name: jurisprudence +description: The following are multiple choice questions (with answers) about jurisprudence. +fewshot_config: + sampler: first_n + samples: + - question: 'Iverson Jewelers wrote a letter to Miller, ''We have received an exceptionally + fine self winding Rolox watch which we will sell to you at a very favorable + price.'' + + (A) The letter is an offer to sell (B) A valid offer cannot be made by letter. + (C) The letter contains a valid offer which will terminate within a reasonable + time. (D) The letter lacks one of the essential elements of an offer.' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. An offer shows the intent to enter into a mutually-beneficial contract + with specific terms. An offer can be made by a letter. While this letter indicates + the willingness to sell, the lack of specific terms, such as transaction price + and offer expiration date, makes it an incomplete offer. The answer is (D). + - question: 'Functions of the law include all but which of the following? + + (A) maximizing individual freedom (B) providing a basis for compromise (C) keeping + the peace (D) promoting the principles of the free enterprise system' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Laws are fundamentally about helping resolve disputes between individuals, + and therefore essential for maximizing individual freedom, providing a basis + for compromise, and keeping the peace. The answer is (D). + - question: 'The ________ School of jurisprudence postulates that the law is based + on what is "correct." + + (A) Natural Law (B) Analytical (C) Historical (D) Sociological' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Natural Law School of jurisprudence focuses on the laws of nature, + and states that the law should be based on ethics, morals, and what is "correct". + Analytical deals with the law as it already exists, Historical postulates that + the law was found and not made, and Sociological studies how the law and society + impact each other. The answer is (A). + - question: 'Which word best summarizes Weber''s explanation of the development of + formally rational law? + + (A) Authority. (B) Charisma. (C) Co-operation. (D) Capitalism.' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Weber explained the development of formal rationality in laws as how + the modern society moved from tradition to rationality, where people decide + actions based less on how they were culturally done and more on expected utilities. + How rational individuals optimize efficiency of accomplishing tasks for higher + rewards is a core principle of Capitalism. The answer is (D). + - question: 'Which position does Rawls claim is the least likely to be adopted by the + POP (people in the original position)? + + (A) The POP would choose equality above liberty. (B) The POP would opt for the + ''maximin'' strategy. (C) The POP would opt for the ''difference principle''. + (D) The POP would reject the ''system of natural liberty.''' + target: 'Let''s think step by step. We refer to Wikipedia articles on jurisprudence + for help. The POP would opt for the ''maximin'' strategy, opt for the ''difference + principle'', and reject the ''system of natural liberty'', but the POP would + not choose equality above liberty, since the POP assume both equal and free + citizens. The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd72cab8cf8ea0553a633810e59c0ba4059c2176 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml @@ -0,0 +1,61 @@ +dataset_name: logical_fallacies +description: The following are multiple choice questions (with answers) about logical + fallacies. +fewshot_config: + sampler: first_n + samples: + - question: 'When an arguer causes confusion during refutation because of real or feigned + lack of an ability to engage in refutation, that arguer may have committed the + fallacy of + + (A) poor sportsmanship (B) appeal to compassion (C) argument against the person + (D) ignorance of refutation' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. Ignorance of refutation, one of Aristotle's original list of logical + fallacies in his Organon, is when someone causes confusion in an argument through + real or feigned inability to engage in refutation, in order to win the argument. + The answer is (D). + - question: 'The complex question fallacy consists of + + (A) arguing something is inferior just because it doesn''t do something it was + never intended to do. (B) including more than one claim in the proposition and + treating proof for one claim as proof for all the claims. (C) drawing a conclusion + before examining the evidence, and only considering evidence that supports that + conclusion. (D) asking a question that includes either an unproven assumption + or more than one question, thus making a straightforward yes or no answer meaningless.' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. The complex question fallacy is when someone makes a single yes or + no answer to a question meaningless, by including either an unproven assumption + or many questions. The latter is also known as the many questions fallacy. The + answer is (D). + - question: 'Arguing that what is true of the parts must be true of the whole is the + fallacy of... + + (A) Division (B) Composition (C) Appeal to the person (D) Appeal to ignorance' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. Fallacy of composition occurs when someone argues what is true of + the parts must be true of the whole. The answer is (B). + - question: 'Which of the following is true of a valid categorical syllogism? + + (A) The minor premise must deny the antecedent (B) The major premise must affirm + the consequent (C) The middle term must be used in at least one premise in a + universal or unqualified sense (D) All of the above' + target: 'Let''s think step by step. We refer to Wikipedia articles on logical + fallacies for help. A valid categorical syllogism must satisfy several conditions: + (1) the syllogism must have exactly three terms (2) every term of the syllogism + must be used twice exactly, (3) a term may be used only once in any premise, + and (4) the middle term must be used in at least one premise in a universal + or unqualified sense, etc. Only (C) is true. The answer is (C).' + - question: 'If someone attacks the character of an opposing arguer, instead of responding + to that opponent''s arguments, the first person has probably committed which + of the following fallacies? + + (A) tu quoque (B) horse laugh (C) argument against the person (D) ignoratio + elenchi' + target: 'Let''s think step by step. We refer to Wikipedia articles on logical + fallacies for help. The argument against the person fallacy occurs when someone + irrelevantly attacks the character of an opposing arguer, instead of addressing + that opponent''s arguments. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33622ac4e7291eb380b6e58382e4fe84052a5bdc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml @@ -0,0 +1,74 @@ +dataset_name: machine_learning +description: The following are multiple choice questions (with answers) about machine + learning. +fewshot_config: + sampler: first_n + samples: + - question: 'Which image data augmentation is most common for natural images? + + (A) random crop and horizontal flip (B) random crop and vertical flip (C) posterization + (D) dithering' + target: Let's think step by step. Data augmentation is used to increase the diversity + of images in the training dataset. It is important that natural images are kept + natural after being augmented. Vertical flips of images are not natural, so + (B) is false. Posterization makes the image look like a poster and and dithering + increases color depth. None of these two preserve the natural property. The + only natural data augmentation technique is (A). The answer is (A). + - question: "Traditionally, when we have a real-valued question attribute during decision-tree\ + \ learning we consider a binary split according to whether the attribute is\ + \ above or below some threshold. Pat suggests that instead we should just have\ + \ a multiway split with one branch for each of the distinct values of the attribute.\ + \ From the list below choose the single biggest problem with Pat\u2019s suggestion:\n\ + (A) It is too computationally expensive. (B) It would probably result in a decision\ + \ tree that scores badly on the training set and a testset. (C) It would probably\ + \ result in a decision tree that scores well on the training set but badly on\ + \ a testset. (D) It would probably result in a decision tree that scores well\ + \ on a testset but badly on a training set." + target: "Let's think step by step. Because the question is real valued, it is unlikely\ + \ that the same values appear both at training and test time. This means that\ + \ while such a decision tree could yield good performance on the training data,\ + \ when evaluated on the test data it will perform badly because the decision\ + \ tree won\u2019t know what to do with numbers that did not appear in the training\ + \ data. The answer is (C)." + - question: "You are reviewing papers for the World\u2019s Fanciest Machine Learning\ + \ Conference, and you see submissions with the following claims. Which ones\ + \ would you consider accepting?\n(A) My method achieves a training error lower\ + \ than all previous methods! (B) My method achieves a test error lower than\ + \ all previous methods! (Footnote: When regularisation parameter \u03BB is chosen\ + \ so as to minimise test error.) (C) My method achieves a test error lower than\ + \ all previous methods! (Footnote: When regularisation parameter \u03BB is chosen\ + \ so as to minimise cross-validaton error.) (D) My method achieves a cross-validation\ + \ error lower than all previous methods! (Footnote: When regularisation parameter\ + \ \u03BB is chosen so as to minimise cross-validaton error.)" + target: "Let's think step by step. In machine learning, we train with some data\ + \ and fixed hyperparameters and the training error can be arbitrarily low, so\ + \ (A) can\u2019t be right. Then, one compares different hyperparameters by selecting\ + \ the model with the lowest cross-validation error, this means that (B) and\ + \ (D) are not the right procedure. The only relevant number after these is the\ + \ test error and thus (C) is the right answer. The answer is (C)." + - question: 'A 6-sided die is rolled 15 times and the results are: side 1 comes up + 0 times; side 2: 1 time; side 3: 2 times; side 4: 3 times; side 5: 4 times; + side 6: 5 times. Based on these results, what is the probability of side 3 coming + up when using Add-1 Smoothing? + + (A) 2.0/15 (B) 1.0/7 (C) 3.0/16 (D) 1.0/5' + target: 'Let''s think step by step. Add-1 smoothing adds the value of one to the + different counts and then normalizes the probabilities accordingly. The counts + after adding one will be: side 1 comes up 1 time; side 2: 2 times; side 3: 3 + times; side 4: 4 times; side 5: 5 times; side 6: 6 times. The number of sum + one die rolls will be 21, so the probability of drawing a three is 3/21 = 1/7. + The answer is (B).' + - question: 'To achieve an 0/1 loss estimate that is less than 1 percent of the true + 0/1 loss (with probability 95%), according to Hoeffding''s inequality the IID + test set must have how many examples? + + (A) around 10 examples (B) around 100 examples (C) between 100 and 500 examples + (D) more than 1000 examples' + target: "Let's think step by step. By the Hoeffding\u2019s inequality, we expect\ + \ that with 95% probability the in-sample and out-of-sample errors differ by\ + \ epsilon when we have N samples if 2 exp(-2 epsilon^2 N)<0.05, this implies\ + \ that N > -1/(2*epsilon**2) log ( 0.05/2 )= log (40)*5000. Since log(40)>1,\ + \ we have that one needs more than 1000 examples. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87d9ba8c9aa31733a5849695213d97deff9c2ded --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml @@ -0,0 +1,46 @@ +dataset_name: management +description: The following are multiple choice questions (with answers) about management. +fewshot_config: + sampler: first_n + samples: + - question: 'How can organisational structures that are characterised by democratic + and inclusive styles of management be described? + + (A) Hierarchical (B) Bureaucratic (C) Flat (D) Functional' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. Flat organizational structures are characterized by democratic and + inclusive styles of management, and have few (if any) levels of management between + the workers and managers. The answer is (C). + - question: 'Hygiene factors are associated with which writer? + + (A) Frederick Hertzberg (B) D.C. McClelland (C) Abraham Maslow (D) Douglas McGregor' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. Hygiene factors include compensation, company policies, supervision, + interpersonal relations, and work environments. Hertzberg lists them as factors + that cannot motivate employees but can minimize job dissatisfaction. The answer + is (A). + - question: 'What characteristic is not a key feature of the ''open systems'' model + of management? + + (A) Morale (B) Innovation (C) Growth resource (D) Adaptation' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. The key characteristics of an open system in management include innovation, + growth resource, and adaption, but do not include morale. The answer is (A). + - question: 'Which element of the cultural web forms regalia? + + (A) Symbols (B) Rituals and routines (C) Power structures (D) Control systems' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. The cultural web is a tool for mapping an organization's culture, + where symbols form the regalia that visually expresses the values that the organization + holds as important. The answer is (A). + - question: 'What are the two main dimensions of the Ohio Studies into leadership? + + (A) Starting position and end position (B) Initial environment and changed environment + (C) Organisational structure and conditioning (D) Initiating structure and considerations' + target: 'Let''s think step by step. We refer to Wikipedia articles on management + for help. The Ohio State Leadership Studies conducted in the 1940s identified + initiating structure and consideration as the two main dimensions of leader + behavior. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_management diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..182eb52ec509c34c225a176774be653d747e120e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml @@ -0,0 +1,56 @@ +dataset_name: marketing +description: The following are multiple choice questions (with answers) about marketing. +fewshot_config: + sampler: first_n + samples: + - question: 'Although the content and quality can be as controlled as direct mail, + response rates of this medium are lower because of the lack of a personal address + mechanism. This media format is known as: + + (A) Care lines. (B) Direct mail. (C) Inserts. (D) Door to door.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Door to door marketing delivers non-addressed items within all buildings + within a geographic area. While it can control the content and quality as well + as direct mail marketing, its response rate is lower because of the lack of + a personal address mechanism. The answer is (D). + - question: 'In an organization, the group of people tasked with buying decisions is + referred to as the _______________. + + (A) Outsourcing unit. (B) Procurement centre. (C) Chief executive unit. (D) + Decision-making unit.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. In an organization, the group of the people tasked with buying decision + is referred to as the decision-making unit. The answer is (D). + - question: 'The single group within society that is most vulnerable to reference group + influence is: + + (A) The older consumer who feels somewhat left out of things. (B) The married + women, many of whom feel a need for stability in their lives. (C) New immigrants + who really want to assimilate into their new culture. (D) Children, who base + most of their buying decisions on outside influences.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Children, who mostly based their buying decisions on outside influences, + are the single group within society that is more vulnerable to reference group + influence. The answer is (D). + - question: 'Which of the following is an assumption in Maslow''s hierarchy of needs? + + (A) Needs are dependent on culture and also on social class. (B) Lower-level + needs must be at least partially satisfied before higher needs can affect behaviour. + (C) Needs are not prioritized or arranged in any particular order. (D) Satisfied + needs are motivators, and new needs emerge when current needs remain unmet.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Maslow's hierarchy of needs, from the bottom upwards, are physiological + (food and clothing), safety, love and belonging needs, esteem, and self-actualization. + Lower-level needs must be at least partially satisfied before higher ones can + affect behavior. The answer is (B). + - question: '_____________ is a natural outcome when combining demographic and geographic + variables. + + (A) Geodemographics (B) Product differentiation. (C) ANSOFF matrix. (D) Brand + management.' + target: 'Let''s think step by step. We refer to Wikipedia articles on marketing + for help. Geodemographics is a natural outcome when combining demographic and + geographic variables. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757010bfed0f08f882995eed787d4f68e0f8121c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml @@ -0,0 +1,51 @@ +dataset_name: medical_genetics +description: The following are multiple choice questions (with answers) about medical + genetics. +fewshot_config: + sampler: first_n + samples: + - question: 'The stage of meiosis in which chromosomes pair and cross over is: + + (A) prophase I (B) metaphase I (C) prophase II (D) metaphase II' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Prophase I is the stage of meiosis where homologous chromosomes pair + with each other and exchange genetic material. The answer is (A). + - question: 'DNA ligase is + + (A) an enzyme that joins fragments in normal DNA replication (B) an enzyme of + bacterial origin which cuts DNA at defined base sequences (C) an enzyme that + facilitates transcription of specific genes (D) an enzyme which limits the level + to which a particular nutrient reaches' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. DNA ligase is a type of enzyme (EC 6.5.1.1) responsible for joining + DNA strands together by catalyzing a phosphodiester bond. The answer is (A). + - question: 'Which of the following conditions does not show multifactorial inheritance? + + (A) Pyloric stenosis (B) Schizophrenia (C) Spina bifida (neural tube defects) + (D) Marfan syndrome' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Multifactorial inheritance is when more than a single factor is responsible + for causing a given trait or health problem. Genes cannot be the only factor. + Marfan syndrome, on the other hand, requires only one abnormal copy of the of + the Marfan gene, from one parent, to inherit the trait. The answer is (D). + - question: 'A gene showing codominance + + (A) has both alleles independently expressed in the heterozygote (B) has one + allele dominant to the other (C) has alleles tightly linked on the same chromosome + (D) has alleles expressed at the same time in development' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Codominance, as it relates to genetics, refers to a type of genetic + inheritance where the phenotype of both the parents is easily observed in the + offspring. A heterozygote is an individual having two different alleles of a + gene. The answer is (A). + - question: 'Large triplet repeat expansions can be detected by: + + (A) polymerase chain reaction. (B) single strand conformational polymorphism + analysis. (C) Southern blotting. (D) Western blotting.' + target: 'Let''s think step by step. We refer to Wikipedia articles on medical + genetics for help. A Southern blot is a method in molecular biology for detecting + specific DNA sequences in a sample. Large triplet repeat expansions are usually + detected with this method. The answer is (C).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fe892eb06f522df6e99606013d26e0df1517cf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml @@ -0,0 +1,43 @@ +dataset_name: miscellaneous +description: The following are multiple choice questions (with answers) about miscellaneous. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of these songs was a Top 10 hit for the rock band The Police? + + (A) ''Radio Ga-Ga'' (B) ''Ob-la-di Ob-la-da'' (C) ''De Do Do Do De Da Da Da'' + (D) ''In-a-Gadda-Da-Vida''' + target: Let's think step by step. We refer to Wikipedia for help. Radio Ga-Ga + is by Queen. Ob-la-di Ob-la-da is by The Beatles. And In-a-Gadda-Da-Vida is + by Iron Butterfly. Leaving 'De Do Do Do De Da Da Da' as the only song by The + Police, and also a Top 10 hit. The answer is (C). + - question: 'What place is named in the title of the 1979 live album by rock legends + Cheap Trick? + + (A) Budapest (B) Budokan (C) Bhutan (D) Britain' + target: Let's think step by step. We refer to Wikipedia for help. Nippon Budokan + is an indoor arena in Tokyo, Japan renowned for hosting rock music concerts + including Cheap Trick in 1978. 'Cheap Trick at Budokan' became the name of their + album. The answer is (B). + - question: 'What is produced during photosynthesis? + + (A) hydrogen (B) nylon (C) oxygen (D) light' + target: Let's think step by step. We refer to Wikipedia for help. Photosynthesis + is the process in which green plants use the green pigment chlorophyll to synthesize + foods with water and carbon dioxide. Oxygen is the byproduct of this process. + The answer is (C). + - question: 'Who is the shortest man to ever win an NBA slam dunk competition? + + (A) Anthony ''Spud'' Webb (B) Michael ''Air'' Jordan (C) Tyrone ''Muggsy'' Bogues + (D) Julius ''Dr J'' Erving' + target: Let's think step by step. We refer to Wikipedia for help. In 1986, Spud + Webb, standing only 5'7" became the shortest NBA player in history to win an + official slam dunk contest. The answer is (A). + - question: 'How many axles does a standard automobile have? + + (A) one (B) two (C) four (D) eight' + target: 'Let''s think step by step. We refer to Wikipedia for help. Most cars + have two axles to rotate the wheels.. The answer is (B).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45a92e075582d6c0e2eb11c0310f87fc5debb4bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml @@ -0,0 +1,64 @@ +dataset_name: moral_disputes +description: The following are multiple choice questions (with answers) about moral + disputes. +fewshot_config: + sampler: first_n + samples: + - question: 'Baron admits that the versions of the ticking bomb hypothetical she discusses + are "stunningly stupid," but she claims this is actually evidence of + + (A) the stupidity of most traditional philosophical examples. (B) a general + lack of intelligence among people with advanced degrees. (C) the wrongness of + torture. (D) the readiness on the part of many intelligent people to see torture + as the best solution to deal with terrorism.' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. The ticking bomb hypothetical poses a problem where many people will + die to an exploding bomb, if the hypothetical terrorist does not disclose how + to defuse it. Baron sees this hypothetical as silly, but its prevalence does + suggest intelligent people, particularly utilitarians, see torture as justifiable + to save the lives in this scenario. The answer is (D). + - question: 'A fertilized ovum is also known as + + (A) a zygote. (B) an embryo. (C) a viability. (D) a blastocyst.' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. Once a single sperm penetrates the layers of an egg to form a new + cell, that cell is called a zygote. The answer is (A). + - question: 'Pence compares six different cases of reproduction, from natural twinning + to SCNT. What conclusion does he draw from this comparison? + + (A) SCNT is not a different kind of reproduction because there are no morally + relevant differences between it and other permissible means of reproduction. + (B) Because there is a low risk of harm for natural twinning, there will be + a low risk of harm for SCNT. (C) both A and B (D) neither A nor B' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. SCNT, known as 'somatic cell nuclear transfer', is a technique in + which the nucleus of a somatic (body) cell is transferred to the cytoplasm of + an enucleated egg. This is used in farm animal cloning. Pence may compare this + to other processes of reproduction, and see little moral separation. The answer + is (A). + - question: 'Which of the following is an intrinsic good, according to Singer? + + (A) being in the upper class (B) the ability to learn and solve problems (C) + being more beautiful than most of the population (D) being shorter than average' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. Bioethicist Peter Singer sees intrinsic value as innate values conferred + by oneself, for oneself. Innanimate objects can be beautiful, short, or have + some valuable criteria, but capabilities are intrinsically good. The answer + is (B). + - question: 'According to Metz, what is wrong with consequentialist arguments against + capital punishment based on African values? + + (A) It is unclear as of yet whether or not capital punishment deters harm to + the community. (B) It is unclear as of yet whether or not capital punishment + deters harm to any individuals. (C) Consequentialism is not supported by African + values. (D) Even though consequentialism is supported by African values, no + consequentialist arguments framed in terms of African values have been offered.' + target: 'Let''s think step by step. We refer to Wikipedia articles on moral disputes + for help. Thaddeus Metz is a humanities research professor focusing on ethical + philosophy in South Africa. Metz has written the death penalty is unjustified + as it treats individuals as incapable of communal relations. It is unclear that + capital punishment is to the benefit of, or a deterrent of harm to the community. + The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4a0231930d62fe73a1cb84b58420baf0182365c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml @@ -0,0 +1,62 @@ +dataset_name: moral_scenarios +description: The following are multiple choice questions (with answers) about moral + scenarios. +fewshot_config: + sampler: first_n + samples: + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I took his wallet after he harassed me at the + bar. Scenario 2 | I made love to my newborn baby boy. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Stealing from someone is not considered an ethical response to harrassment. + And having sex with a minor is egregiously unethical. The answer is (A). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I asked for a cigarette from the guy outside + the store. Scenario 2 | When the security guard left his post, I pocketed a + rare coin. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Asking for something is not morally wrong. Stealing is morally wrong. + The answer is (C). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | During the house''s demolition, I pushed him + into a wall. Scenario 2 | When I choose a babysitter for my twin nieces, I don''t + care about her racial background. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Physical violence is morally wrong. Being open to hiring people from + different racial backgrounds is an expected moral stance. The answer is (B). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I bought an old pick-up truck for work. Scenario + 2 | I looked up at the gorgeous waterfall and threw a rock at it. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Neither of these are wrong as no one is wronged by buing a pick-up + truck or throwing a rock into a stream. The answer is (D). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I really loved my newborn baby boy. Scenario + 2 | The new girl is embarrassed to live in a trailer park, so I invite the class + to her house. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: 'Let''s think step by step. We refer to Wikipedia articles on moral scenarios + for help. Loving someone is not wrong. However, exposing something that someone + is embarrassed about could be considered quite mean. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66498dc564350f893e7bd45078528b5750bce0ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml @@ -0,0 +1,63 @@ +dataset_name: nutrition +description: The following are multiple choice questions (with answers) about nutrition. +fewshot_config: + sampler: first_n + samples: + - question: 'What is the first-line drug for patients with type 2 diabetes and obesity, + as of 2020? + + (A) Acarbose (B) Metformin (C) Sulphonylureas (D) Insulin' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Metformin (Fortamet, Glumetza, or others) is usually the first medication + prescribed for type 2 diabetes, as well as obesity. It works by lowering glucose + production in the liver and improving the body's sensitivity to insulin. The + answer is (B). + - question: 'Which of the following statements is correct (according to knowledge in + 2020)? + + (A) Consumers with phenylketonuria must avoid the consumption of the sweetener + aspartame (B) Consumers with phenylketonuria must avoid the consumption of the + sweetener saccharin (C) Consumers with phenylketonuria must avoid the consumption + of the sweetener sucralose (D) Consumers with phenylketonuria must avoid the + consumption of the sweetener acesulfame K' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. People with phenylketonuria (PKU) cannot break down the amino acid + phenylalanine. As it builds up in the blood and brain it can lead to brain damage. + People with PKU should avoid foods that are converted to phenylalanine in the + body, such as aspartame. The answer is (A). + - question: 'Which of the following statements about iodine is correct, as of 2020? + + (A) 50% of adults consume iodine at levels below the RNI (B) Dairy products + are a poor source of iodine (C) The iodine content of organic milk is generally + lower that the level in non-organic milk (D) UK dietary reference values recommend + an increase in iodine intake in pregnancy' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Organic milk usually has less iodine content than non-organic milk. + The answer is (C). + - question: 'Which of the following is the most plausible explanation for the protective + effect of dietary fibre against cancer of the colon, as of 2020? + + (A) Propionic acid, formed during colonic fibre fermentation inhibits liver + fatty acid synthesis (B) Butyric acid, formed during colonic fibre fermentation + stimulates "silencing" of the SLC5A8 tumour suppressor gene (C) None of these + options are correct (D) Butyric acid, formed during colonic fibre fermentation + stimulates anti-oxidant defences in the colon' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Dietary fibre is inversely proportional to the risk of colorectal + cancer. This is presumed because butyric acid (BA) stimulates antioxidants which + help protect the colon from cancerous tumors. The answer is (D). + - question: 'In a cohort study, the risk ratio of developing diabetes was 0.86 when + comparing consumers of tea (the exposed) to those who did not drink tea (the + unexposed). Which one statement is correct (according to knowledge in 2020)? + + (A) The tea drinkers have lower risk of developing diabetes. (B) The tea drinkers + have higher risk of developing diabetes. (C) Based on the information given + we cannot tell if the observed difference in disease risk is the result of chance. + (D) The risk ratio is close to the value one, so there is no difference in disease + risk between the two groups.' + target: 'Let''s think step by step. We refer to Wikipedia articles on nutrition + for help. The risk ratio is not sufficiently reduced that it could not be explained + by random chance given the studies sample size. The answer is (C).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5116be01d6a463f23f9b7422de1b0b48bbdb55e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml @@ -0,0 +1,44 @@ +dataset_name: philosophy +description: The following are multiple choice questions (with answers) about philosophy. +fewshot_config: + sampler: first_n + samples: + - question: 'The study of reality in the broadest sense, an inquiry into the elemental + nature of the universe and the things in it, is known as _____. + + (A) metaphysics (B) epistemology (C) quantum physics (D) axiology' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Among the options, only metaphysics studies the nature of reality + and existence. The answer is (A). + - question: "According to Moore\u2019s \u201Cideal utilitarianism,\u201D the right\ + \ action is the one that brings about the greatest amount of:\n(A) pleasure.\ + \ (B) happiness. (C) good. (D) virtue." + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Moore's "ideal utilitarianism" states that one's actions should maximize + intrinsic goods. The answer is (C). + - question: 'Before Tolstoy''s Christian conversion, what was his perspective on the + meaning of life? + + (A) optimist (B) satisfied (C) nominally religious (D) pessimist' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Before his conversion, Tolstoy feels that life was uncertain, which + is a pessimist's point of view. The answer is (D). + - question: 'According to d''Holbach, people always act according to _____. + + (A) free choices (B) dictates of the soul (C) necessary natural laws (D) undetermined + will' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. d'Holbach believes that people act according to necessary laws, and + it proves nothing about people's free will. The answer is (C). + - question: 'Psychological egoism is: + + (A) an ethical theory about how we ought to behave. (B) a generalization concerning + the way people tend to behave. (C) a claim about human nature and the ways people + are capable of behaving. (D) none of the above.' + target: 'Let''s think step by step. We refer to Wikipedia articles on philosophy + for help. Psychological egoism suggests that one behaves based on what makes + one feels good, hence it is a claim about human nature and how humans are capable + of behaving. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f9e5d81664445497a13a9adf7ca818ed6d2c7ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml @@ -0,0 +1,59 @@ +dataset_name: prehistory +description: The following are multiple choice questions (with answers) about prehistory. +fewshot_config: + sampler: first_n + samples: + - question: 'What is the approximate mean cranial capacity of Homo erectus? + + (A) under 650 cc (B) about 800 cc (C) just under 1000 cc (D) 1200 cc' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. The average cranium capacity of Homo erectus is less than 1000 cubic + cm. The answer is (C). + - question: 'According to Timothy Pauketat, the evidence for social stratification + and political power at Cahokia suggests: + + (A) a center of Mississippian civilization with conditions similar to the rise + of early states. (B) the limitations of authority in a Native American society + of egalitarian foragers. (C) a simple chiefdom or perhaps a complex chiefdom + had evolved by A.D. 1500. (D) a center of Mississippian civilization with conditions + similar to societies on the Northwest Coast of North America.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Timothy Pauketat is known for his research on Cahokia, the center + of the Mississippian culture, where he found similar conditions to the rise + of early states. The answer is (A). + - question: 'Recent research on hominid species dating from the Middle Pliocene indicates + there was (as of 2020): + + (A) a great amount of species diversity, or a single species that exhibited + a lot of diversity. (B) very little species diversity during this period and + very few hominids. (C) decreased species diversity due to a prolonged ice age + followed by a severe drought. (D) decreased species diversity but increased + numbers of hammerstones and flakes, indicating stone tool manufacture.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Recent research has recognized multiple hominid species from the Middle + Pliocene, meaning that there is a great amount of species diversity or diversity + in a single species. The answer is (A). + - question: 'Researchers now believe that the decline of the Maya was caused chiefly + by: + + (A) a cataclysm of some kind, such as an earthquake, volcano, or tsunami. (B) + ecological degradation resulting from slash-and-burn farming techniques. (C) + endless wars between neighboring Mayan city-states. (D) practices of interbreeding + that led to a steep rise in congenital disorders.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Researchers believe that the Maya collapse was mainly caused by over-exploitation + of natural resources like the slash-and-burn farming techniques. The answer + is (B). + - question: 'The great Mayan king Pacal built temples in the city of Palenque in order + to: + + (A) satisfy the powerful Mayan astronomer priests. (B) display his generosity + to the common people, since they were allowed to live in the temples. (C) frighten + away enemies, in particular the Spaniards. (D) legitimize his kingship, since + his father was not royal.' + target: 'Let''s think step by step. We refer to Wikipedia articles on prehistory + for help. Pacal built the temples as the funerary monument to legitimize his + kingship. The answer is (D).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d18fc22626b952d18491b849cabc720706c17c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml @@ -0,0 +1,63 @@ +dataset_name: professional_accounting +description: The following are multiple choice questions (with answers) about professional + accounting. +fewshot_config: + sampler: first_n + samples: + - question: "An auditor traces the serial numbers on equipment to a nonissuer\u2019\ + s subledger. Which of the following management assertions is supported by this\ + \ test?\n(A) Valuation and allocation (B) Completeness (C) Rights and obligations\ + \ (D) Presentation and disclosure" + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The completeness assertion is tested by tracing supporting documents + to the record entries. The answer is (B). + - question: 'One hundred years ago, your great-great-grandmother invested $100 at 5% + yearly interest. What is the investment worth today? + + (A) $13,000 (B) $600 (C) $15,000 (D) $28,000' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. A $100 investment at 5% yearly interest is worth 100*(1.05)^100=13150 + after 100 years, which is around $13,000. The answer is (A). + - question: 'On January 1, year 1, Alpha Co. signed an annual maintenance agreement + with a software provider for $15,000 and the maintenance period begins on March + 1, year 1. Alpha also incurred $5,000 of costs on January 1, year 1, related + to software modification requests that will increase the functionality of the + software. Alpha depreciates and amortizes its computer and software assets over + five years using the straight-line method. What amount is the total expense + that Alpha should recognize related to the maintenance agreement and the software + modifications for the year ended December 31, year 1? + + (A) $5,000 (B) $13,500 (C) $16,000 (D) $20,000' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The maintenance period begins on March 1, so only 10 months of expenses + should be recognized, which is $15,000/12*10=$12,500. The software modification + cost is amortized over 5 years, so each year is $5,000/5=$1,000. So the total + expense is $12,500+$1,000=$13,500. The answer is (B). + - question: 'Krete is an unmarried taxpayer with income exclusively from wages. By + December 31, year 1, Krete''s employer has withheld $16,000 in federal income + taxes and Krete has made no estimated tax payments. On April 15, year 2, Krete + timely filed for an extension request to file her individual tax return, and + paid $300 of additional taxes. Krete''s year 1 tax liability was $16,500 when + she timely filed her return on April 30, year 2, and paid the remaining tax + liability balance. What amount would be subject to the penalty for underpayment + of estimated taxes? + + (A) $0 (B) $500 (C) $1,650 (D) $16,500' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The tax due after withholding is $16,500-$16,000=$500, which is less + than $1000, hence there is no underpayment penalty of estimated taxes. The answer + is (A). + - question: 'Box a nongovernmental not-for-profit organization had the following transactions + during the year: Proceeds from sale of investments $80000 Purchase of property + plant and equipment $10000 Proceeds from long-term debt $100000 Loss on sale + of investment $5000 What amount should be reported as net cash provided by financing + activities in Box''s statement of cash flows? + + (A) $70,000 (B) $75,000 (C) $80,000 (D) 100000' + target: 'Let''s think step by step. We refer to Wikipedia articles on accounting + for help. Among the four transactions, only Proceeds from long-term debt belongs + to the financing activities section of cashflow, hence the amount reported should + be $100000. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307f8940bc445305fdbf00e89910cd5237a41312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml @@ -0,0 +1,122 @@ +dataset_name: professional_law +description: The following are multiple choice questions (with answers) about professional + law. +fewshot_config: + sampler: first_n + samples: + - question: 'A son owed a creditor $5,000. The son''s father contacted the creditor + and told him that he wanted to pay the son''s debt. The father signed a document + that stated the father would pay the son''s debt at a rate of $500 a month for + 10 months. The creditor made no written or oral commitment to forbear to sue + the son to collect the $5,000 debt, and the father made no oral or written request + for any such forbearance. For the next five months, the father made and the + creditor accepted the $500 monthly payments as agreed. During that period, the + creditor, in fact, did forbear to take any legal action against the son. However, + the father then informed the creditor that he would make no further payments + on the debt. Which of the following is the most persuasive argument that the + father is liable to the creditor under the terms of their agreement? + + (A) The father''s promise and the creditor''s reliance thereon, if proved, gave + rise to a valid claim by the creditor against the father based on the doctrine + of promissory estoppel. (B) Because it was foreseeable that the father''s promise + would induce the creditor to forbear taking any action against the son, such + forbearance was, as a matter of law, a bargained-for consideration for the father''s + promise. (C) The father''s five payments to the creditor totaling $2,500 manifested + a serious intent on the father''s part to be contractually bound, and such manifestation + is generally recognized as an effective substitute for consideration. (D) By + assuming the antecedent debt obligation that the son owed to the creditor, the + father became a surety whose promise to the creditor was enforceable, since + it was in writing and supported by adequate consideration. ' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The doctrine of promissory estoppel stops a person from going back on a promise + in contract law, hence option (A) should be the most persuasive argument. The + answer is (A). + - question: 'A state has recently enacted a statute prohibiting the disposal of any + nuclear wastes within the state. This law does not contravene or conflict with + any federal statutes. A man operates a company in the state that is engaged + in the disposal of nuclear wastes. Subsequent to the passage of the state statute, + the man, not yet aware of the new law, entered into contracts with many out-of-state + firms to dispose of their nuclear wastes in the state. On account of this new + law, however, the man will be unable to perform these contracts. Assume that + the man has standing to challenge this state law. Which of the following presents + his strongest constitutional grounds to challenge the state law prohibiting + the disposal of nuclear wastes within the state? + + (A) The commerce clause. (B) The equal protection clause of the Fourteenth Amendment. + (C) The privileges and immunities clause of Article IV, Section 2. (D) The contract + clause.' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The commerce clause states that Congress shall have the power to regulate commerce + with foreign Nations, and among the several States, and with the Indian Tribes. + The statute affects inter-state commerce which puts it into question. Hence + the man's strongest argument should be the commerce clause. The answer is (A). + - question: 'On October 1, 1980, a developer, owner of several hundred acres in a rural + county, drafted a general development plan for the area. The duly recorded plan + imposed elaborate limitations and restrictions upon the land in the plan, which + was to be developed as a residential district. The restrictions were to extend + to all persons acquiring any of the lots and to their heirs, assigns, and lessees. + It was further provided that all subsequent owners would be charged with due + notice of the restrictions. Among those restrictions in the general plan were + the following:(22) A franchise right is created in a strip of land 10 feet in + width along the rear of each lot for the use of public utility companies with + right of ingress and egress. (23) No house or structure of any kind shall be + built on the aforementioned strip of land running through the said blocks. In + 2000, a retiree purchased one of the lots, built a house, and erected a fence + in the rear of his property within the restricted area. In 2004, a teacher purchased + a lot adjacent to the retiree''s property and built a new house. Two years later, + a librarian purchased the lot that adjoined the teacher''s property. The three + deeds to those properties each contained references to the deed book where the + general plan was recorded. In 2008, the librarian began the construction of + a seven-foot post-and-rail fence along the line dividing his lot with the teacher''s, + and along the center of the area subject to the franchise right. Although the + teacher objected to its construction, the fence was completed. If the teacher + seeks a mandatory injunction to compel removal of the librarian''s fence, the + court will most likely + + (A) grant relief, because the fence was in violation of the easement restriction. + (B) grant relief, because the encroachment of the fence violated the restriction + in the original plan. (C) deny relief, because the teacher failed to enforce + the restriction against the retiree. (D) deny relief, because the fence would + not be construed as "a structure" within the terms of the restriction. ' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The restrictions in the original plan say no house or structure of any kind + shall be built on the aforementioned strip of land running through the said + blocks. Hence the court will most likely grant relief because the fence violated + the restriction in the original plan. The answer is (B). + - question: 'Judge took judicial notice of some facts at the beginning of the trial. + Which of the following is not an appropriate kind of fact for judicial notice? + + (A) Indisputable facts. (B) Facts that have been asserted by individual political + organizations. (C) Facts recognized to be true by common knowledge. (D) Facts + capable of scientific verification.' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + Among the options, facts that have been asserted by individual political organizations + is not an appropriate kind of fact for judicial notice. The answer is (B). + - question: 'A state legislature has recently enacted a statute making it a misdemeanor + to curse or revile or use obscene or opprobrious language toward or in reference + to a police officer perfonning his duties. A student at a state university organized + a demonstration on campus to protest the war. The rally was attended by a group + of 50 students who shouted anti-war messages at cars passing by. To show his + contempt for the United States, the student sewed the American flag to the rear + of his jeans. When a police officer saw the flag sown on the student''s jeans, + he approached and told him to remove the flag or he would be placed under arrest. + The student became angered and shouted at the police officer, "Listen, you bastard, + I''ll wear this rag anywhere I please. " The student was subsequently placed + under arrest and charged with violating the state statute. The student subsequently + brings suit in state court challenging the constitutionality of the statute. + The strongest constitutional argument for the student is that + + (A) the statute is void for vagueness under the Fourteenth Amendment''s due + process clause. (B) the statute is invalid because it violates the petitioner''s + freedom of speech under the First Amendment. (C) the statute is an abridgment + of freedom of speech under the First Amendment because less restrictive means + are available for achieving the same purpose. (D) the statute is overbroad and + consequently invalid under the First and Fourteenth Amendments.' + target: 'Let''s think step by step. We refer to Wikipedia articles on law for + help. The Fourteenth Amendment further supports the First Amendment by establishing + a due process clause. Hence the strongest argument should be the statute is + overbroad and consequently invalid under the First and Fourteenth Amendments. + The answer is (D).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a5b0d995ae7d0897eaea3a6cfd0719562cdc3c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml @@ -0,0 +1,82 @@ +dataset_name: professional_medicine +description: The following are multiple choice questions (with answers) about professional + medicine. +fewshot_config: + sampler: first_n + samples: + - question: "A 22-year-old male marathon runner presents to the office with the complaint\ + \ of right-sided rib pain when he runs long distances. Physical examination\ + \ reveals normal heart and lung findings and an exhalation dysfunction at ribs\_\ + 4-5 on the right. Which of the following muscles or muscle groups will be most\ + \ useful in correcting this dysfunction utilizing a direct method?\n(A) anterior\ + \ scalene (B) latissimus dorsi (C) pectoralis minor (D) quadratus lumborum" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. Among the options, only pectoralis minor muscle origins from the outer + surfaces of the 3rd to 5th ribs. The answer is (C). + - question: "A 36-year-old male presents to the office with a\_3-week\_history of low\ + \ back pain. He denies any recent trauma but says that he climbs in and out\ + \ of his truck numerous times a day for his job. Examination of the patient\ + \ in the prone position reveals a deep sacral sulcus on the left, a posterior\ + \ inferior lateral angle on the right, and a lumbosacral junction that springs\ + \ freely on compression. The most likely diagnosis is\n(A) left-on-left sacral\ + \ torsion (B) left-on-right sacral torsion (C) right unilateral sacral flexion\ + \ (D) right-on-right sacral torsion" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The deep sulcus on the left, a posterior ILA on the right, with a negative + spring test suggests a right-on-right sacral torsion. All other options have + a deep sulcus on the right. The answer is (D). + - question: "A 44-year-old man comes to the office because of a 3-day history of sore\ + \ throat, nonproductive cough, runny nose, and frontal headache. He says the\ + \ headache is worse in the morning and ibuprofen does provide some relief. He\ + \ has not had shortness of breath. Medical history is unremarkable. He takes\ + \ no medications other than the ibuprofen for pain. Vital signs are temperature\ + \ 37.4\xB0C (99.4\xB0F), pulse 88/min, respirations 18/min, and blood pressure\ + \ 120/84 mm Hg. Examination of the nares shows erythematous mucous membranes.\ + \ Examination of the throat shows erythema and follicular lymphoid hyperplasia\ + \ on the posterior oropharynx. There is no palpable cervical adenopathy. Lungs\ + \ are clear to auscultation. Which of the following is the most likely cause\ + \ of this patient's symptoms?\n(A) Allergic rhinitis (B) Epstein-Barr virus\ + \ (C) Mycoplasma pneumonia (D) Rhinovirus" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The symptoms, especially the headache, suggest that the most likely cause + is Rhinovirus. Epstein-Barr virus will cause swollen lymph nodes but there is + no palpable cervical adenopathy. Lungs are clear to auscultation suggests it's + not Mycoplasma pneumonia. The answer is (D). + - question: 'A previously healthy 32-year-old woman comes to the physician 8 months + after her husband was killed in a car crash. Since that time, she has had a + decreased appetite and difficulty falling asleep. She states that she is often + sad and cries frequently. She has been rechecking the door lock five times before + leaving her house and has to count exactly five pieces of toilet paper before + she uses it. She says that she has always been a perfectionist but these urges + and rituals are new. Pharmacotherapy should be targeted to which of the following + neurotransmitters? + + (A) Dopamine (B) Glutamate (C) Norepinephrine (D) Serotonin' + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The patient feels sad and among the options, only Dopamine and Serotonin + can help increase positive emotions. Serotonin also affects digestion and metabolism, + which can help the patient's decreased appetite and sleep difficulty. The answer + is (D). + - question: "A 42-year-old man comes to the office for preoperative evaluation prior\ + \ to undergoing adrenalectomy scheduled in 2 weeks. One month ago, he received\ + \ care in the emergency department for pain over his right flank following a\ + \ motor vehicle collision. At that time, blood pressure was 160/100 mm Hg and\ + \ CT scan of the abdomen showed an incidental 10-cm left adrenal mass. Results\ + \ of laboratory studies, including complete blood count, serum electrolyte concentrations,\ + \ and liver function tests, were within the reference ranges. The patient otherwise\ + \ had been healthy and had never been told that he had elevated blood pressure.\ + \ He takes no medications. A follow-up visit in the office 2 weeks ago disclosed\ + \ elevated urinary normetanephrine and metanephrine and plasma aldosterone concentrations.\ + \ The patient was referred to a surgeon, who recommended the adrenalectomy.\ + \ Today, vital signs are temperature 36.6\xB0C (97.9\xB0F), pulse 100/min, respirations\ + \ 14/min, and blood pressure 170/95 mm Hg. Physical examination discloses no\ + \ significant findings. Initial preoperative preparation should include treatment\ + \ with which of the following?\n(A) Labetalol (B) A loading dose of potassium\ + \ chloride (C) Nifedipine (D) Phenoxybenzamine" + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. The symptoms and the adrenal mass suggested pheochromocytoma, and + the blood pressure indicates hypertension. Phenoxybenzamine is used to treat + hypertension caused by pheochromocytoma. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60b5da683ff87d207304b894a5138a6c439a1c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml @@ -0,0 +1,62 @@ +dataset_name: professional_psychology +description: The following are multiple choice questions (with answers) about professional + psychology. +fewshot_config: + sampler: first_n + samples: + - question: 'In the construction of a multiple regression equation for purposes of + prediction, the optimal combination of measures is one in which the predictors + + (A) are uncorrelated with each other but are moderately correlated with the + criterion (B) have low correlations with each other and low correlations with + the criterion (C) are highly intercorrelated with each other and moderately + correlated with the criterion (D) have low correlations with the criterion bur + are moderately correlated with each other' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. The basis of multiple regression is to assess the relationship between + one continuous variable and a set of independent variables. So the predictors + should be uncorrelated with each other but are moderately correlated with the + criterion. The answer is (A). + - question: 'There are three ways to measure the Central Tendency: the Mean, the Median + and the Mode. From your knowledge about them, what is the mode? + + (A) less sensitive to extreme scores than the mean (B) more useful for skewed + distributions (C) sensitive to extreme values and highly skewed distributions + (D) the most frequently occurring number' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. The definition of mode is the most frequently occurring number. The + answer is (D). + - question: "Carl Jung believed that a client's transference:\n(A) is a fantasy that\ + \ distracts the client from reality. (B) represents \u201Cmixed feelings\u201D\ + \ toward the therapist. (C) \"is a form of \"\"acting out.\"\"\" (D) reflects\ + \ the client\u2019s personal and collective unconscious." + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Transference is a phenomenon that a person's feelings are unconsciously + redirected, so it reflects the client's personal and collective unconscious. + The answer is (D). + - question: "In terms of Hofstede\u2019s (1980) five cultural dimensions, the United\ + \ States scores at the top of the scale on:\n(A) individualism. (B) individualism\ + \ and power distance. (C) power distance and masculinity. (D) uncertainty avoidance." + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. US scores highest on individualism among the five cultural dimensions. + The answer is (A). + - question: 'One of your therapy clients asks your advice about a good weight- reduction + program. You have investigated the programs in the community and are enrolled + in the one you consider the best. This program offers a $50 bonus to its patrons + for each new person they bring into the program. Under these circumstances, + your most appropriate response would be to + + (A) tell your client the pros and cons of each program you know about except + for the one in which you are enrolled (B) recommend to your client the program + in which you are enrolled and explain the $50 bonus you will receive (C) recommend + to your client the program in which you are enrolled and offer to have the $50 + bonus credited to your client''s account in the program (D) tell your client + the pros and cons of each program you know about, but do not claim the $50 bonus + if your client enrolls in your program' + target: 'Let''s think step by step. We refer to Wikipedia articles on psychology + for help. Based on the circumstances, you should tell your client about the + pros and cons of each program, but it would be inappropriate to receive the + bonus, so you should not claim the $50 bonus. The answer is (D).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe384b1e2b7d19c216f8344d5c249f2c16dc723b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml @@ -0,0 +1,55 @@ +dataset_name: public_relations +description: The following are multiple choice questions (with answers) about public + relations. +fewshot_config: + sampler: first_n + samples: + - question: 'Earth Hour was a campaign launched by which organization? + + (A) Greenpeace (B) The UN (C) Oxfam (D) World Wildlife Fund' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. Earth Hour is a worldwide movement oragnized launched by the World + Wildlife Fund. The answer is (D). + - question: 'In issues management, what is the most proactive approach to addressing + negative or misleading information posted online about your organization? + + (A) Buy domain names that could be used by opposition groups. (B) Post anonymous + comments on blogs to combat this information. (C) Prepare a news release that + discredits the inaccurate information. (D) Make policy changes to address complaints + highlighted on these sites.' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. In issues management, the most proactive approach to addressing negative + or misleading information posted online is to make policy changes to address + complaints highlighted on those sites. The answer is (D). + - question: 'At which stage in the planning process would a situation analysis be carried + out? + + (A) Defining the program (B) Planning the program (C) Taking action and implementing + ideas (D) Evaluation of the program' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. Situation analyses are typically carried out during the planning process + stage of defining the program. The answer is (A). + - question: 'Which of these statements is true of the Vatican in 2010 at the time of + the accusations of child abuse cover-ups? + + (A) There was a coordinated media response. (B) Consistent messages were communicated. + (C) Criticisms were taken as attacks on the Catholic Church. (D) The credibility + of the Vatican was upheld.' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. In 2010 when there were accusations of child abuse cover-ups, the + Vatican took those criticisms as attacks on the Catholic Church. The answer + is (C). + - question: 'What should a public relations media practitioner do if she does not know + the answer to a reporter''s question? + + (A) Give the reporter other information she is certain is correct. (B) Say that + the information is ''off the record'' and will be disseminated later. (C) Say + ''I don''t know'' and promise to provide the information later. (D) Say ''no + comment,'' rather than appear uninformed.' + target: 'Let''s think step by step. We refer to Wikipedia articles on public relations + for help. If a public relations media practitioner does not know the answer + to a reporter''s question, they should say ''I don''t know'' and offer to provide + the information later. The answer is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b37e35b3bd4fefa0ca040f0d59ff2fcae156fb45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml @@ -0,0 +1,104 @@ +dataset_name: security_studies +description: The following are multiple choice questions (with answers) about security + studies. +fewshot_config: + sampler: first_n + samples: + - question: 'What are the frameworks of analysis within which terrorism has been considered + (as of 2020)? + + (A) Competition between larger nations has resulted in some countries actively + supporting terrorist groups to undermine the strength of rival states. Terrorist + networks are extended patronage clubs maintained and paid for by their donor + states and are conceptualised as being like state actors, to be dealt with using + military force. (B) Globalization has enabled the internationalization of terrorist + activities by opening up their operational space, although coordination is still + managed from a geographical base. This suggests that terrorist groups are nationally + structured which means that terrorism cannot be considered in terms of a war + to be defeated militarily without having serious implications on the indigenous + population. (C) Terrorism can be viewed as a problem to be resolved by military + means (war on terrorism), by normal police techniques (terrorism as crime), + or as a medical problem with underlying causes and symptoms (terrorism as disease). + (D) Terrorism is viewed as a criminal problem. The criminalization of terrorism + has two important implications. Firstly, it suggests that terrorism can be eradicated + - terrorists can be caught and brought to trial by normal judicial proceedings + thereby removing the threat from society - and secondly, it suggests that preventative + crime techniques are applicable to prevent its development.' + target: "Let's think step by step. We refer to Wikipedia articles on security\ + \ studies for help. (A) is wrong because it is not competition between larger\ + \ nations that causes terrorism. \n(B) is wrong because globalization is not\ + \ the cause of terrorism.\n(C) is correct because the US undertook the war on\ + \ terrorism. \n(D) is wrong because preventative crime techniques will likely\ + \ not end terrorism. The answer is (C)." + - question: 'Which of the following is the best lens through which to investigate the + role of child soldiers? + + (A) Child soldiers are victims of combat that need re-education and rehabilitation. + (B) Children and their mothers are not active subjects in warfare and are best + considered as subjects in the private sphere. (C) Children are most often innocent + bystanders in war and are best used as signifiers of peace. (D) Children have + political subjecthood that is missed when they are considered as passive victims + of warfare.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. Child soliders as a political topic can be missed when they are considered + passive victims of warfare. The answer is (D). + - question: 'How can we best describe the relationship between the state-centric approach + and the concept of human security? + + (A) There are such wide divisions within the human security framework regarding + the nature of threats and referent objects that no widely applicable comparisons + between state-centric approaches and human security can be drawn. (B) By adopting + the framework of human security, the limitations of the realist state-centric + approach become evident. Whilst human security defines the referent object as + the person or population, state-centric approaches prioritise the security of + the state, de-prioritizing the pursuit of human security. (C) The state-centric + approach to security is a faction of human security, usually defined within + the broad school of human security. By being state-centric this approach prioritises + the individual as the referent object in security studies. (D) Both the state-centric + and human-centric approaches to security are mutually exclusive and offer a + sufficient analytic framework with which to understand the international security + system. It is therefore the role of security analysts to determine which of + these substantial concepts is correct, and which should be discarded.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. Human security focuses on a person or population whereas state-centric + approaches focus on the state while deprioritizing human security. The answer + is (B). + - question: 'In order to become securitized, a threat must be presented in which of + these ways? + + (A) As an existential threat that requires immediate and extraordinary action, + posing a threat to the survival of the state or to societal security. (B) As + requiring immediate and extraordinary action by the state, threatening the survival + of a referent object and therefore warranting the use of measures not normally + employed in the political realm. (C) As an urgent threat to the survival of + the referent object, so serious that it legitimises the employment of extraordinary + action in response. (D) As an urgent threat to the survival of the audience + that requires extraordinary or emergency measures.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. To be securitized, a threat must be an urgent threat to the survival + of the referent object. The answer is (C). + - question: 'What distinguishes coercive diplomacy from military force? + + (A) Compellence is another term for coercive diplomacy, but covering a narrower + set of criteria; compellence covers those threats aimed at initiating adversary + action. A threat to coerce a state to give up part of its territory would count + as coercive diplomacy, as long as that threat proactively initiates action before + reactive diplomacy is taken. (B) Coercive diplomacy constitutes the threats + of limited force to induce adversary''s incentive to comply with the coercer''s + demands. It is an influence strategy that is intended to obtain compliance: + the use of force to defeat an opponent first does not count. It leaves an element + of choice with the target to comply, or to continue. (C) Military force, or + the threat of military force, utilises fear to achieve strategic objectives. + Coercive diplomacy is differentiated from this approach, because it does not + use fear as a tool for coercing an adversary. (D) Coercive diplomacy is employed + to use force but to limit its effects on the international community. Coercive + diplomacy is an aggressive strategy that is intended to obtain compliance through + defeat. It does not leave an element of choice with the target, the target either + being forced to comply or engage in conflict. It seeks to control by imposing + compliance by removing any opportunity for negotiation or concession.' + target: 'Let''s think step by step. We refer to Wikipedia articles on security + studies for help. Coercive diplomacy uses the threat of force to induce the + opponent to comply with demands. The answer is (B).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4229d64785ded8673d421a9fb1571d0cce705a93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml @@ -0,0 +1,58 @@ +dataset_name: sociology +description: The following are multiple choice questions (with answers) about sociology. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is not a problem associated with official statistics + on strike action? + + (A) most strikes go unnoticed by employers and the mass media (B) not all industrial + disputes will be reported by the employer (C) the definition of strikes excludes + those that involve fewer than ten workers or last less than one day (D) it is + hard to compare strikes that were measured in different ways' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Official statistics on strike action can be problematic because not + all industrial disputes will be reported by employers, the definition of strikes + excludes those that involves fewer than ten workers or last less than one day, + and it is hard to compare strikes that were measured in different ways. Thus, + (A) is not a problem associated with official statistics on strike action. The + answer is (A). + - question: 'What does Berger (1963) describe as a metaphor for social reality? + + (A) a fairground ride (B) a circus (C) a puppet theatre (D) a ballet' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Berger describes social reality using the metaphor of a puppet theatre. + The answer is (C). + - question: 'The term ''hegemony'' refers to: + + (A) the tendency for the working class not to realize their own interests (B) + a dominant ideology that legitimates economic, political and cultural power + (C) a form of dual consciousness based on ideology and everyday experiences + (D) a mode of payment given for outstanding topiary' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Hegemony refers to a dominant ideology that legitimates economic, + policital, and cultural power. The answer is (B). + - question: 'The shift from ''civil religion'' to ''common religion'' means that: + + (A) the increasing bureaucracy of the state has made religion only a marginal + part of our lives (B) despite the weakening of traditional authority, our everyday + lives and ''common sense'' remain shaped by religious beliefs and values (C) + religious participation in collective worship may have declined, but people + still practise their faiths in private (D) people are much more likely to discuss + their religious beliefs in public, informal settings' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. The shift from civil religion to common religion means that despite + the weakening of traditional authority, our everyday lives and common sense + remain shaped by religious beliefs and values. The answer is (B). + - question: 'Which of the following did the post-war welfare state of 1948 not aim + to provide: + + (A) free health care and education for all (B) a minimum wage (C) full employment + (D) universal welfare' + target: 'Let''s think step by step. We refer to Wikipedia articles on sociology + for help. The post-war welfare state of 1948 aimed to provide free healthcare + and education, full employment, and universal welfare. But it did not aim to + provide a minimum wage. The answer is (B).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc86b7c88fa1b62d2f12deaa16394f43fc722225 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml @@ -0,0 +1,56 @@ +dataset_name: us_foreign_policy +description: The following are multiple choice questions (with answers) about us foreign + policy. +fewshot_config: + sampler: first_n + samples: + - question: 'How did Donald Trump attack globalization in the 2016 campaign? + + (A) Globalization had made men like him too rich (B) Globalization only benefited + certain American states, such as New York (C) Liberal elites had encouraged + globalization, while ''ordinary Americans'' lost jobs because of it (D) Globalization + encouraged damaging trade wars' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. Trump attacked globalization because he believed ordinary Americans + lost jobs due to it, and so he wanted to blame liberals who had encouraged it. + The answer is (C). + - question: 'How did NSC-68 change U.S. strategy? + + (A) It globalized containment. (B) It militarized containment. (C) It called + for the development of the hydrogen bomb. (D) All of the above' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. NSC-68 outlined a variety of courses of action, including globalization + of containment, militarization of contaiment, and the development of the hydrogen + bomb. The answer is (D). + - question: 'How do Defensive Realism and Offensive Realism differ in their explanation + of state behaviour? + + (A) Defensive realists place greater emphasis on the role of international institutions + (B) Defensive realists place less emphasis on geographical factors (C) Offensive + realists give more priority to the national interest than Defensive realists. + (D) Defensive realists believe states are security maximizers, while Offensive + realists believe states to be power maximizers' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. While defensive realism advocates that states are security + maximizers, offensive realists think of states as power maximizers. The answer + is (D). + - question: 'The realm of policy decisions concerned primarily with relations between + the United States and the rest of the world is known as + + (A) terrorism policy. (B) economic policy. (C) foreign policy. (D) international + policy.' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. The topic of policy decisions concerns with relations between + the US and the rest of the world is known as foreign policy. The answer is (C). + - question: 'How did the 2008 financial crisis affect America''s international reputation? + + (A) It damaged support for the US model of political economy and capitalism + (B) It created anger at the United States for exaggerating the crisis (C) It + increased support for American global leadership under President Obama (D) It + reduced global use of the US dollar' + target: 'Let''s think step by step. We refer to Wikipedia articles on us foreign + policy for help. The 2008 financial crisis damanged the international reputation + of the American model of political economy and capitalism. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e70f0ee24cc946a1cfbc51cb87d4fde20d8171c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml @@ -0,0 +1,45 @@ +dataset_name: virology +description: The following are multiple choice questions (with answers) about virology. +fewshot_config: + sampler: first_n + samples: + - question: 'The median survival time to AIDS and death was established by following: + + (A) Seroprevalent HIV-infected individuals (B) Seronegatives (C) Seroconverters + (D) High-risk seronegatives' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The median survival time to AIDS and death was established as a result + of the development of seroconverters. The answer is (C). + - question: 'Which of the following is a morphological characteristic of the paramyxoviruses. + + (A) Fragile viruses often visualised with RNA spewing from the inside (B) Elongate + viruses (C) Icosahedral viruses with envelope (D) Very large viruses' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. Paramyxoviruses are fragile viruses often visualised with RNA spewing + from the inside. The answer is (A). + - question: 'The most important goal of a behavioral intervention is: + + (A) Change in behavior (B) Comprehensive coverage (C) Effective use of behavioral + theory (D) Sustained behavior change' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The prim goal of a behavioral intervention is to cause sustained behavior + change. The answer is (D). + - question: 'A key factor facilitating the application of nested case-control studies + from the MACS was: + + (A) Data collection (B) Establishment of a repository of biologic specimens + (C) Participant interest (D) Administration of the questionnaire by staff' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The Multicenter AIDS Cohort Study's use of nested case-control studies + was facilitated by the establishment of a repository of biologic specimens. + The answer is (B). + - question: 'Why are parvoviruses a highly impactful parasite? + + (A) Because they have no nucleic acid (B) They require a helper virus (C) Only + replicate in dividing cells (D) Can integrate into host chromosomes' + target: 'Let''s think step by step. We refer to Wikipedia articles on virology + for help. Paroviruses are highly impactful because they do not have nucleic + acid. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41502cc7a3a1318b7c6a0f2ac16cda86dda08486 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml @@ -0,0 +1,42 @@ +dataset_name: world_religions +description: The following are multiple choice questions (with answers) about world + religions. +fewshot_config: + sampler: first_n + samples: + - question: 'How can the Upanishads be characterized? + + (A) Ritual texts (B) Philosophical texts (C) Hymns (D) Origin stories' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The Upanishads are the most recent part of Vedas (the oldest scriptures + in Hinduism) and supplied the basis of later Hindu philosophy. So they are philosophical + texts. The answer is (B). + - question: 'What is the Second Gem in Buddhism? + + (A) The Dharma (B) The Sangha (C) The Buddha (D) The Bodhisattva' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The Second Gem in Buddhism is The Dharma. The answer is (A). + - question: 'Which Japanese government promoted a kind of national cult based on the + emperor and his associations with kami? + + (A) Honen (B) Tanaka (C) Tokugawa (D) Meiji' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The promotion of a national cult based on the emperor and his associations + with Kami happened during the reign of Emperor Meiji (1852-1912). The answer + is (D). + - question: 'In which dynasty was the "Mandate of Heaven" developed to legitimatize + the new rulers? + + (A) Shang (B) Zhou (C) Han (D) Xia' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The "Mandate of Heaven" was developed as an ancient Chinese philosophical + concept during the Zhou Dynasty (1046-256 BCE). The answer is (B). + - question: 'What is the sign of the covenant for Jewish males? + + (A) The rainbow (B) Circumcision (C) A son (D) Bar mitzvah' + target: 'Let''s think step by step. We refer to Wikipedia articles on world religions + for help. In Judaism, the most distinctive sign of the covenant is circumcision + (brit milah). The answer is (B).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..745a892568bd84b38252e20bbc9a0bea73ddb1db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_cot_zeroshot +group_alias: mmlu (flan style, zeroshot cot) +task: + - group: stem + task: + - mmlu_flan_cot_zeroshot_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_cot_zeroshot_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_cot_zeroshot_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_cot_zeroshot_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7588b67e1905dc7eae3790c33a16de460c465f67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml @@ -0,0 +1,38 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +validation_split: validation +fewshot_split: dev +output_type: generate_until +doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step." +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer]}}" +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.))" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: -1 + ignore_case: true + ignore_punctuation: true + regex_pattern: "(\\([A-Z]\\))" + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +num_fewshot: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 3.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e885b818eae4bbc87374c756b68ecd11e44bd69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f17410a7cc0869223730328f55803d8d424e930 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5b821f97642ad5987244a0ac4c9988c2fca3857 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b26c679e26b6bd04d77eb5e0bb2ebaddcc515561 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c0e9d17db10f4e69d1c44d5a127f2bbe1f4e279 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_clinical_knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de020f4eaca7fdeb650688f034ee3b5d89490ddc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8e5bbcf76b9fb3ad012511b213ffbbd554cd58d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04b5e750949984abcd7889be80485e52c97dba9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81c59cc2c20f340a76ed3d945e976ce3c832815c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0450a068f4b763629e463d9882e4a3e99f86d726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82c2bb2ab586be2346237a6aa8b2ea9fd9170c97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78216a44778fa0f9f1e057d5dc45b998fd5e87fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52304bdf8eeac624c63331b259255a98866dc2ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_conceptual_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5be81c442710f91ad3e1ca6a0651105b2f14e24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..934a1a20a69d987904fe9c8b605c93e4ed149309 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96ec81d6a8716ad60a4b3215faa42f3c3b1396d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..915c96de78b68bdd2b8b8cbb26f2f8ec0ae24167 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a1f7491590b80e784360ceb72619efe4d9568f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_global_facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c4043d9bd7e6a38d702afa7ccb4028e98001445 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aee89159d40e4f7c788cf670d9fa2e405d32c75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb3eb2134bf8e3e8b8e81f29432db3e81b5f2fcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fc261e8fe114ffc9d7be99110d659704018f159 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..baabc83a9e25b700600fe516d9a84833c32f4f29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41365c509da451280527720e651d5793d1b83960 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_government_and_politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05e62fa85cb3fdf871ec246de43d32c7a5209db1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fb8639003555bdca712f3dc49ed6e463158be42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..999f9be74e2bc278a068c344030ae27f3b2c3006 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f905569c82f31ec76a75505bfae64c28d72640 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d09cdcaa3b268d599e055f82c92779d4ecd2bcb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a71bfc38aab72f17a01e3da11fc037ce28ef033 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_human_aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa9b895b7331b051385a31165c725c2ef976db69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_human_sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..642e6ce4f34992cb5be8b840ea481c7a389d9ce8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml @@ -0,0 +1,6 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c27feea94ce017e35bcd453d6cbf5c4db5b3334 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml @@ -0,0 +1,6 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_machine_learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1a13763a2bd796821efa251071359ce0acbf1cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..813b6a3fe90413bd35a11f82624df600d8bf682b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14465ad6e5c5434974832399ea95903b59e4eaf5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_n_shot_generative +group_alias: mmlu (flan style, generative) +task: + - group: stem + task: + - mmlu_flan_n_shot_generative_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_generative_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_generative_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_generative_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e602ee8100ed612d89385532ea30004c3033c35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa12cc8ef35b19f3b81dcc58a0107d424a3580cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4447d276b066ddec93b8f7efcf2d74d13810f458 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f36eb1f598f754154c2b15b24bbb650358c707c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56eeae0183ca0c087b0a16aa317f2b93d5f1b87b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..346e4b669771f23d7a3a805b329e96e711cd367e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_logical_fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62460e82f3386022679443efe3c989c2ffb59abf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..408c69f11630d4c237079e327b1b4c9fe3971dc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_moral_scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fd1f01a1707126eaf93e6a668d681408c8c7fe6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a80f2baacbe17445f4a1ea564c7a72174b1c445 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da9e30e118445c2a796eb4145f5c308e9e33215f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7043a07273fe781cb56733ab02b0b1cb4bf059 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..debace7ca0d0c1dbdfad6ad1621dcc5d9a1469eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb1f585ce890b5a4fcccc72c3066691509330c49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0580f7ae31687590598a77fe950d282020d9be16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ff2d9ea791abf7bad56784b804bcadd5c82c077 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3edfd9528eed5d4199ebb0ba06a328ed8c50dd8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..765e70c8fc22dbd75ba495a6490ec788d4e44b7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ff9cc284007337e30369dd4864b2b723e8e6768 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_biology_generative" +"task_alias": "college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15ae9dded855610af45a15bab8aa56596bfaddd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_mathematics_generative" +"task_alias": "college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d997d8974c99a549a2216a9bd9237f05a619e21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_physics_generative" +"task_alias": "college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee64d20100e25fc4bcf7f446b1e98acf042c4ab8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_computer_security_generative" +"task_alias": "computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..130ec2b2aa2210322c1e2f86cdf6be31dd72bffc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_electrical_engineering_generative" +"task_alias": "electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4afd087dc47f27653b54ff48a27a187bc9af07bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_elementary_mathematics_generative" +"task_alias": "elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c28c0b188b8b8fd69ba9ed79595f0d173f71cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_formal_logic_generative" +"task_alias": "formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3677842dcfc091bb28525889479a48096cbb854d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_biology_generative" +"task_alias": "high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2df93cab2a999a7d6d8e78d3ac9c3ce9aeddcf12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_chemistry_generative" +"task_alias": "high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec5dc7f89abd7ddc57438c71e0502fce1ac47279 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_computer_science_generative" +"task_alias": "high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9732754bbd7352957dbe299494083e17b960c1bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_european_history_generative" +"task_alias": "high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66b1a3c97a64f9ee7db414ab13d3146efba5612d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_geography_generative" +"task_alias": "high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46861fdc1149b72d4ac3f347c0e09f679f6c6e54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_government_and_politics_generative" +"task_alias": "high_school_government_and_politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ada415922b2b777f153cf387f9095cce9c75304b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_macroeconomics_generative" +"task_alias": "high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b22a5888e61be187f5bbbca1e38171eecd6252d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_mathematics_generative" +"task_alias": "high_school_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c59ff16270084981614d6f01065851c005039413 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_microeconomics_generative" +"task_alias": "high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d846afb9c8c6b372d59ee462561bb8f67ae83e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_physics_generative" +"task_alias": "high_school_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd1321a5f17efca463edbc6711c197fb18c3a81d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_psychology_generative" +"task_alias": "high_school_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1442fb8df4168606151af5cc1dfd769bb2e70e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_statistics_generative" +"task_alias": "high_school_statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4552a560f38e3ed5db503fa677548a11766873c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_us_history_generative" +"task_alias": "high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d510f22ff39219829e6a9030cb39dc2c43062ca4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_world_history_generative" +"task_alias": "high_school_world_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56352f4a8c86966853cdbafd68453d1ee85dbabb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_human_aging_generative" +"task_alias": "human_aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a23559cfb36a380131573f46b30bbdb5f4656b42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_human_sexuality_generative" +"task_alias": "human_sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..878df6f3cacb299a51afacca461204fdc4e3a782 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_international_law_generative" +"task_alias": "international_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5782d81551072a0ff03d79c930f02edb64488f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_jurisprudence_generative" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43e8e0168b9f4638cc80b76ff1a4edc8893212b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_logical_fallacies_generative" +"task_alias": "logical_fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d39a4b53164ce8bb641c99fa50f24ace308d3f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_machine_learning_generative" +"task_alias": "machine_learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d51ea0d0aa41fb4b2579162111aa8ebd8ce8f6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_management_generative" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..744385a2ea524d6f651851856e15aaf190eb847e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_marketing_generative" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fea57959818525acdada5bf8a327b0ce96fefb0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_medical_genetics_generative" +"task_alias": "medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e0fabc2536d4894526b680deba9a382ff9c3ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_miscellaneous_generative" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61d2feee6a9cf4ed4d71b7c2f9aa68f5219c270a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_moral_disputes_generative" +"task_alias": "moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aeb93f967f0811d3a2f1d886aedfb334a96714e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_moral_scenarios_generative" +"task_alias": "moral_scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..638ac8100b6f918ccaa0a3dc13946512d3c97b33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_nutrition_generative" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..149894b8484cb1fad9ddad1fc5cb2c07a659aea1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_philosophy_generative" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e130e1baacc3f8a8f558b568336896668e84dd4f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_prehistory_generative" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a46792ec22d84ee3193996653f536084b9ab7861 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_accounting_generative" +"task_alias": "professional_accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f087657e579524b35bf7de4c0f81cb5b697caed4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_law_generative" +"task_alias": "professional_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc80878980195f58ac5ae26a0a70589a47b325d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_medicine_generative" +"task_alias": "professional_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b36ccde61e7edc33464a676d4fe0fcc25f3304 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology_generative" +"task_alias": "professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37cdccba9b7cebbaa34c5f1e9da01655367477f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_public_relations_generative" +"task_alias": "public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36c235feefd1548320400e7e8d9f3e03f2d478d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_security_studies_generative" +"task_alias": "security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7e2e592e4457118c9458ccb757b823f9adbb193 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_sociology_generative" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fb95366245eae638918270bff4353024195d5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy_generative" +"task_alias": "us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9954dc182f1bbd5030b94d2a08b2ddf4a135a6cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_virology_generative" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1db5128b43e615d0fc41f9c7448db3b5ea39942c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_world_religions_generative" +"task_alias": "world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e28b0527aecda0cd833c5c40588b42d0682d404c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/README.md @@ -0,0 +1,68 @@ +# mmlu_pro + +### Paper + +Title: `MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark` + +Abstract: `In the age of large-scale language models, benchmarks like the Massive Multitask Language Understanding (MMLU) have been pivotal in pushing the boundaries of what AI can achieve in language comprehension and reasoning across diverse domains. However, as models continue to improve, their performance on these benchmarks has begun to plateau, making it increasingly difficult to discern differences in model capabilities. This paper introduces MMLU-Pro, an enhanced dataset designed to extend the mostly knowledge-driven MMLU benchmark by integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. Additionally, MMLU-Pro eliminates the trivial and noisy questions in MMLU. Our experimental results show that MMLU-Pro not only raises the challenge, causing a significant drop in accuracy by 16% to 33% compared to MMLU but also demonstrates greater stability under varying prompts. With 24 different prompt styles tested, the sensitivity of model scores to prompt variations decreased from 4-5% in MMLU to just 2% in MMLU-Pro. Additionally, we found that models utilizing Chain of Thought (CoT) reasoning achieved better performance on MMLU-Pro compared to direct answering, which is in stark contrast to the findings on the original MMLU, indicating that MMLU-Pro includes more complex reasoning questions. Our assessments confirm that MMLU-Pro is a more discriminative benchmark to better track progress in the field.` + +Homepage: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro + +### Citation + +```bibtex +@misc{wang2024mmlupro, + title={MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark}, + author={Yubo Wang and Xueguang Ma and Ge Zhang and Yuansheng Ni and Abhranil Chandra and Shiguang Guo and Weiming Ren and Aaran Arulraj and Xuan He and Ziyan Jiang and Tianle Li and Max Ku and Kai Wang and Alex Zhuang and Rongqi Fan and Xiang Yue and Wenhu Chen}, + year={2024}, + eprint={2406.01574}, + archivePrefix={arXiv}, + primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'} +} +``` + +### Groups and Tasks + +#### Groups + +* `mmlu_pro`: 'All 14 subjects of the mmlu_pro dataset, evaluated following the methodology in mmlu's original implementation' + +#### Tasks + +The following tasks evaluate subjects in the mmlu_pro dataset +- `mmlu_pro_biology` +- `mmlu_pro_business` +- `mmlu_pro_chemistry` +- `mmlu_pro_computer_science` +- `mmlu_pro_economics` +- `mmlu_pro_engineering` +- `mmlu_pro_health` +- `mmlu_pro_history` +- `mmlu_pro_law` +- `mmlu_pro_math` +- `mmlu_pro_other` +- `mmlu_pro_philosophy` +- `mmlu_pro_physics` +- `mmlu_pro_psychology` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog + +* (tasks, group) 2024-09-23 -- (version 1 --> version 2) + * Added one newline to task description(s) as per [reference implementation](https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/47b9891aacb8bd7cda29d5c5ba17b9434dd333bc/evaluate_from_local.py#L93) +* (tasks, group) 2025-03-20 -- (version 2.0 --> version 2.1) + * Changed default max_length from 2048 to 8192 and max_gen_toks from 256 to 2048. +* (tasks, group) 2025-05-20 -- (version 2.1 --> version 3) + * changed stop sequence from "Q:" to "Question:" PR #2945 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_default_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d59d03a3bb09437644922c8345452539493a0ae9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_default_template_yaml @@ -0,0 +1,32 @@ +dataset_path: TIGER-Lab/MMLU-Pro +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + # regex_pattern: r".*[aA]nswer:\s*([A-J])", + - function: "take_first" +generation_kwargs: + until: + - "Question:" + max_gen_toks: 2048 + do_sample: false + temperature: 0.0 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.1 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc3204127604d6eac759299f77d63ce9ef49d24e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml @@ -0,0 +1,23 @@ +group: mmlu_pro +task: + - mmlu_pro_biology + - mmlu_pro_business + - mmlu_pro_chemistry + - mmlu_pro_computer_science + - mmlu_pro_economics + - mmlu_pro_engineering + - mmlu_pro_health + - mmlu_pro_history + - mmlu_pro_law + - mmlu_pro_math + - mmlu_pro_other + - mmlu_pro_philosophy + - mmlu_pro_physics + - mmlu_pro_psychology +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80aee85108ce8ac9452c0d0d07341e05b11b0b7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_biology" +task_alias: "biology" +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daf871f6bb5abd614c2058c0552389d41cbced50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_business" +task_alias: "business" +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5baf354ec202e66647bacdc9e9008617cd2d4244 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_chemistry" +task_alias: "chemistry" +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7de347373e7b92d8eb9ba33bbf8ec4fb2a3dbc49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_computer_science" +task_alias: "computer_science" +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274612783fb749aad10ec67698c3abf5dce13ebf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_economics" +task_alias: "economics" +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcf02f5029823ab83c1cebfbbf9819ac5ec4f1d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_engineering" +task_alias: "engineering" +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d161d1d81a29a06a9425bafc9e86211a2669b9be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_health" +task_alias: "health" +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d28efd3e77ef31dc2df96847c90e7c76920007bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_history" +task_alias: "history" +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba99f16dbb8df07ab9272d8b47d23839a919bc9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_law" +task_alias: "law" +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0526f11f975f556f94c21acbc7074c0b4b76dd42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_math" +task_alias: "math" +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb2ec9da7011dd1437828fc442ad21733cdf1a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99e5d65b4c0af9a2a35589ab104cb07be100cabc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e7fa740bd58ebd55382bd8464abd9abaee3d96e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b28fb72d329d4d9755e45201275604d257851754 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_psychology" +task_alias: "psychology" +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ca254a29a271c2fdf27781b897d16af9a61afc66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py @@ -0,0 +1,51 @@ +from functools import partial + + +choices = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] + + +def format_cot_example(example, including_answer=True): + prompt = "Question:\n" + question = example["question"] + options = example["options"] + prompt += question + "\n" + prompt += "Options:\n" + + for i, opt in enumerate(options): + if i >= len(choices): + break + prompt += "{}. {}\n".format(choices[i], opt) + + if including_answer: + cot_content = example["cot_content"].replace( + "A: Let's think step by step.", "Answer: Let's think step by step." + ) + prompt += cot_content + "\n\n" + else: + prompt += "Answer: Let's think step by step." + + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f3db0d165db36a0842069e7be6dc021bdf9b6568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/README.md @@ -0,0 +1,75 @@ +# MMLU-ProX + +### Paper + +Title: `MMLU-ProX: A Multilingual Benchmark for Advanced Large Language Model Evaluation` + +Abstract: `Traditional benchmarks like MMLU and MMLU-Pro focus primarily on single-language evaluation, limiting their ability to assess language models in multilingual and culturally diverse contexts. To address this gap, we introduce MMLU-ProX, a comprehensive multilingual benchmark that builds upon MMLU-Pro by covering multiple typologically diverse languages with approximately 11,829 questions per language.` + +Homepage: https://mmluprox.github.io/ + +### Citation + +```bibtex +@misc{mmluprox, + title={MMLU-ProX: A Multilingual Benchmark for Advanced Large Language Model Evaluation}, + author={Weihao Xuan and Rui Yang and Heli Qi and Qingcheng Zeng and Yunze Xiao and Yun Xing and Junjue Wang and Huitao Li and Xin Li and Kunyu Yu and Nan Liu and Qingyu Chen and Douglas Teodoro and Edison Marrese-Taylor and Shijian Lu and Yusuke Iwasawa and Yutaka Matsuo and Irene Li}, + year={2025}, + eprint={2503.10497}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2503.10497}, +} +``` + +### Groups and Tasks + +#### Groups + +* `mmlu_pro_{lang}`: 'All 14 subjects of the mmlu_pro_prox dataset in {lang}, evaluated following the methodology in mmlu_pro's original implementation' + +Available lang: +- ar +- bn +- de +- en +- es +- fr +- hi +- ja +- ko +- pt +- sw +- th +- zh + +#### Tasks + +The following tasks evaluate subjects in the mmlu_prox dataset +- `mmlu_prox_{lang}_biology` +- `mmlu_prox_{lang}_business` +- `mmlu_prox_{lang}_chemistry` +- `mmlu_prox_{lang}_computer_science` +- `mmlu_prox_{lang}_economics` +- `mmlu_prox_{lang}_engineering` +- `mmlu_prox_{lang}_health` +- `mmlu_prox_{lang}_history` +- `mmlu_prox_{lang}_law` +- `mmlu_prox_{lang}_math` +- `mmlu_prox_{lang}_other` +- `mmlu_prox_{lang}_philosophy` +- `mmlu_prox_{lang}_physics` +- `mmlu_prox_{lang}_psychology` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_ar_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_ar_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c44c140dd4e07c3fd28d0ee7129f0b1f6edccf0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_ar_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ar +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'الإجابة هي \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "سؤال:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_mmlu_prox_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_mmlu_prox_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22de9a1444cc4a4677e42924ea91134277feee45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_mmlu_prox_ar.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ar +task: +- mmlu_prox_ar_biology +- mmlu_prox_ar_business +- mmlu_prox_ar_chemistry +- mmlu_prox_ar_computer_science +- mmlu_prox_ar_economics +- mmlu_prox_ar_engineering +- mmlu_prox_ar_health +- mmlu_prox_ar_history +- mmlu_prox_ar_law +- mmlu_prox_ar_math +- mmlu_prox_ar_other +- mmlu_prox_ar_philosophy +- mmlu_prox_ar_physics +- mmlu_prox_ar_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21dc18d094366c7fa2eb100a13a4d1077d4f2064 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_biology.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول علم الأحياء. فكر خطوة + بخطوة ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d995fe2b4fd8a108166e9dcf366853f87d40d51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_business.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الأعمال. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90d9786d3c3ffed4ffae79765f9fa15758b3a0b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_chemistry.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الكيمياء. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebbe0aba56381adf3dc6b83cee43f46c2306026f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_computer_science.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول علوم الكمبيوتر. فكر خطوة + بخطوة ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43dd2194060013f13a6c8ec23d3e185b3ba0a25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_economics.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الاقتصاد. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..976052d42554e65f4b585dd06c993fa310fa05e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_engineering.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الهندسة. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc112389491ad0f81d97a833215e8690c252a2d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_health.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الصحة. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e954acd3b2371aa2260b13d3b383481cc141dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_history.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول التاريخ. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb9741e8d8273d365f8235afc5482a863682ae93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_law.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول القانون. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..791adf799edc13936383208aa667b5c1a26799a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_math.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الرياضيات. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4722e54f6ded128c9016e7c8a2cad6f420929a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_other.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول أخرى. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..477710558ae2d233c752fd415147a16a95fdd96a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_philosophy.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الفلسفة. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfb623df5ce8cbe53d790e01bdab27b5bfade7a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_physics.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الفيزياء. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86b6517de09743800e6a24eba5d9f65dfb87b8c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_psychology.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول علم النفس. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_bn_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_bn_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e5e67509d725a44bbc9e5b868f344f5c9e05761 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_bn_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: bn +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'উত্তর হল \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "প্রশ্ন:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_mmlu_prox_bn.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_mmlu_prox_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..393097d3896037937cfeb2d09610371bf723e13b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_mmlu_prox_bn.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_bn +task: +- mmlu_prox_bn_biology +- mmlu_prox_bn_business +- mmlu_prox_bn_chemistry +- mmlu_prox_bn_computer_science +- mmlu_prox_bn_economics +- mmlu_prox_bn_engineering +- mmlu_prox_bn_health +- mmlu_prox_bn_history +- mmlu_prox_bn_law +- mmlu_prox_bn_math +- mmlu_prox_bn_other +- mmlu_prox_bn_philosophy +- mmlu_prox_bn_physics +- mmlu_prox_bn_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f0148e006763ab79f87b04042533076b1f7a568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_biology.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত জীববিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61f2313c195858c2fc79cb54ca868dafde80302d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_business.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত ব্যবসা সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63b7909ddaa00f5ced2b2ecfd51ccde9c85b276f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত রসায়ন সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25f2c8cb820d59f219ff504bfa108cd0aaf9f09f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত কম্পিউটার বিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। + ধাপে ধাপে চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে + X হল সঠিক বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a31a9a4f1820ed0723da7b7d3eee844e3bf528c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_economics.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত অর্থনীতি সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ca7ef16d9fbbac0a5f5b60d0801523d7fbd354b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_engineering.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত প্রকৌশল সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b03efc55dad017262b46f1127a8532f066da187 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_health.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত স্বাস্থ্য সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..873383c9ec8b6349e7e1dc9584968bc005dc496a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_history.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত ইতিহাস সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf8bdc6f7e96384fd84f9b9f2bc193d7420b60b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_law.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত আইন সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা + করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের + অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bdb2f23e47e34c910c14967ad11a0608504da95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_math.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত গণিত সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা + করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের + অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14efa2768459cae192bdbcffba8922a884da4215 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_other.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত অন্যান্য সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92196436a3fce2db50992a778bd4063f1c27a618 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত দর্শন সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা + করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের + অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d8291d3131196714d1dfd6d709767faf0c5fe10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_physics.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত পদার্থবিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে + ধাপে চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল + সঠিক বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..498389a2893fcbe1ed45549d0d559ac3e64e9095 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_psychology.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত মনোবিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3ef81b4b9ef43990b91018eebd0ab7dc161f964 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: de +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Die Antwort ist \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Frage:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e717130357c24d9859f49c3fe749453a2f382a00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_de.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_de +task: +- mmlu_prox_de_biology +- mmlu_prox_de_business +- mmlu_prox_de_chemistry +- mmlu_prox_de_computer_science +- mmlu_prox_de_economics +- mmlu_prox_de_engineering +- mmlu_prox_de_health +- mmlu_prox_de_history +- mmlu_prox_de_law +- mmlu_prox_de_math +- mmlu_prox_de_other +- mmlu_prox_de_philosophy +- mmlu_prox_de_physics +- mmlu_prox_de_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87835fd5b6df77c044c4b83c82905553bdefd30b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_biology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Biologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8683d9ee4c8f0578f991e463ccc20c673179a3d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Wirtschaft. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e43365713a43209efbb5b2c0f9db5b70fb028075 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Chemie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c663f692eea1a17df6fe45b37be1706dcaa3df57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Informatik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a01a7488c60907557a019d5c5f5d29d23fc33fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ökonomie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d9a7359c0e173156e5a2dc513fa4bd11c184916 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ingenieurwesen. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..303272745cdbdbe32c83a2d3ceba85b02e59974a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Gesundheit. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b08416c7de46b11f53623607961bf1353a0ae1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Geschichte. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1e58ff8658c70c85b6fd1c56f143b462dc1fed1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Recht. Denken + Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort ist (X)", + wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dee8f6e4e732564714e7c9dbed29e8301ac9b66e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Mathematik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2dd274cc2b94b24df402369300bd1de9e79b5f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Sonstiges. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757be6dfd8a34ba60d64776d9dec9084325a8414 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Philosophie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd3a6e6d4dd93f520679dafb7db26cbd87545c93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Physik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fbcd9b49b48ceecb01c3422f1c2f27d96739e2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Psychologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed712f867d03aa8da77c33c89080dfeec7e7cea7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: en +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..649a5a625b9dcf7745abe1c16a888bc9ff6e40d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_en +task: +- mmlu_prox_en_biology +- mmlu_prox_en_business +- mmlu_prox_en_chemistry +- mmlu_prox_en_computer_science +- mmlu_prox_en_economics +- mmlu_prox_en_engineering +- mmlu_prox_en_health +- mmlu_prox_en_history +- mmlu_prox_en_law +- mmlu_prox_en_math +- mmlu_prox_en_other +- mmlu_prox_en_philosophy +- mmlu_prox_en_physics +- mmlu_prox_en_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c33358b48caee9dcf040000609675e5f3135038b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about biology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc85a949be4ea9f026c580eb88653e397022c1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about business. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f9c45a290f1c9cf0489f4f7ffc55aa433f105e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about chemistry. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b2e55f3abe22beaf4ed5efd75c3e43844eb3f6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about computer_science. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37232d2f4620fe297a04329753374843f3113f5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about economics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ca9302d32a9dedfb61bb798b8bc8a49308cdf1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about engineering. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a384c2e5a59f0533b2b86621cdd34488ddde807 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about health. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6bb11902f17570ddeaa9695a15457cbcaccd0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about history. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..534da99020b95befee63db1acc2c58b3e72d5008 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about law. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59fca5bac4841ff195ed230ce62a67a865783ba3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about math. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..437e2449e203eb047c45c971f058b9ac49ff08ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about other. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d67bc4dd543fbd1060bccdbd9f9d705e344404d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about philosophy. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7847c0f5f37b231a2113a454873a8ab5c84339a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about physics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e3b84d8c5c261366e59c7265169b3e6b7ce51e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about psychology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a84f6cf5d1416d54d5ddb297467ddf4e6bc4653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: es +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La respuesta es \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pregunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..437942cb0b0527c84d6ec084d7bf5534a496bf51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_es +task: +- mmlu_prox_es_biology +- mmlu_prox_es_business +- mmlu_prox_es_chemistry +- mmlu_prox_es_computer_science +- mmlu_prox_es_economics +- mmlu_prox_es_engineering +- mmlu_prox_es_health +- mmlu_prox_es_history +- mmlu_prox_es_law +- mmlu_prox_es_math +- mmlu_prox_es_other +- mmlu_prox_es_philosophy +- mmlu_prox_es_physics +- mmlu_prox_es_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aa01f9bacf34d21ce732e3d583c19fb1ce59dcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + biología. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c7551b67d595300f6218fe3c0b2817fbbea4f74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + negocios. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d75f7034b7ef6e047b15e69a7146a72661377e41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + química. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33d2d919542192e6520cb54b90037cf9af6bfedd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + informática. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d257611b14eeb4d8eef729bd02ea3e95eeca243c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + economía. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..504de4348c4b0e9435f1d6a01c3d2b91017b4a42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + ingeniería. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f621a739e28c438bcedbfcddc1faf75dc368ab83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + salud. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ecb14e95bcfb1035e3a4c71e890fd71e19b564d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_history.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + historia. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e8e89acde0b0eacfd713e06fb022e363ed7f248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + derecho. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c584a90864df9246a0db4a52937973e21f81cee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + matemáticas. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f58119ad994400ed61a9148a6b01f2047c9fecb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + otro. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afb955393bba6b3a57089bdb7a93fbfd0db73dc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + filosofía. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db2a905cb103bd5292bb75bf6a295da5a84d836f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + física. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..217c215e07d58ebfbdd889620968dba04b68d49c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + psicología. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..935f561faec99a0bde2e3721a6e09737825d2627 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: fr +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La réponse est \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question :" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_mmlu_prox_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_mmlu_prox_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..661deda053eb630552f71e9c346bea75993d02c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_mmlu_prox_fr.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_fr +task: +- mmlu_prox_fr_biology +- mmlu_prox_fr_business +- mmlu_prox_fr_chemistry +- mmlu_prox_fr_computer_science +- mmlu_prox_fr_economics +- mmlu_prox_fr_engineering +- mmlu_prox_fr_health +- mmlu_prox_fr_history +- mmlu_prox_fr_law +- mmlu_prox_fr_math +- mmlu_prox_fr_other +- mmlu_prox_fr_philosophy +- mmlu_prox_fr_physics +- mmlu_prox_fr_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c3b74cbb5698f47db3acf927aaba48cd303e5c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_biology.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur biologie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cbe1472f507fa9d5d9b33e47113c3dbbe401f96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_business.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur commerce. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4fd7560b855ee8eaab595303e1d8c6b37aad057 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur chimie. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30f825c182f5b5e386871416de86f4a4fe234fea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur informatique. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..885ba15d6d6577433d8743a847c7dd7af6f9ad76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_economics.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur économie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f096ea88f24e739b5a5405ce2baeb34de20627 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur ingénierie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e642b40b4600882c639ccfcfef522e515ea199d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_health.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur santé. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ad409b66248fee8abf5ce7ac305a754f616f5af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_history.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur histoire. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33bbd86404717f7061cfeec43a05421e3cc3d4d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_law.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur droit. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0a9bd489d093ebf7f6183f8cc76f20e3b0c7755 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_math.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur mathématiques. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9763476f98f933d9afba845b3b62010c14dc7814 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_other.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur autre. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32f602be482e6ca9b0774cd80b048ffe6de0c6db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur philosophie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5adf9a01605d45a750fc1fad95915b10a791be5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_physics.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur physique. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3004495da851bcc582d8e4155c079ea6f542e0c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur psychologie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_hi_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_hi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..483a487bc14a247caf1c1c9fcae3df4c6e043581 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_hi_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: hi +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'उत्तर है \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "प्रश्न:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_mmlu_prox_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_mmlu_prox_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ae06179de5891ac91862f806d7635f04b7b6176 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_mmlu_prox_hi.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_hi +task: +- mmlu_prox_hi_biology +- mmlu_prox_hi_business +- mmlu_prox_hi_chemistry +- mmlu_prox_hi_computer_science +- mmlu_prox_hi_economics +- mmlu_prox_hi_engineering +- mmlu_prox_hi_health +- mmlu_prox_hi_history +- mmlu_prox_hi_law +- mmlu_prox_hi_math +- mmlu_prox_hi_other +- mmlu_prox_hi_philosophy +- mmlu_prox_hi_physics +- mmlu_prox_hi_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ab83377ce03ff55f59c867ae8be11086ce33f11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_biology.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित जीव विज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e3a7bb97f66881058247ceed280cd948575da00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_business.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित व्यापार के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e3670e28797d5e16a009462ba2bb0880f0cd68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित रसायन विज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों के + साथ) हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें + जहां X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..254f7997b029cef1c35ee83f901a53522390141b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित कंप्यूटर विज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों + के साथ) हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त + करें जहां X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db6d9f4915f23b970ab23d7cc0488faac0a019b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_economics.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित अर्थशास्त्र के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc23e4659d7819eb95606091bd1944147582732b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_engineering.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित इंजीनियरिंग के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25af832a2a8441a8fe33d7e3ba8237d18634d2a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_health.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित स्वास्थ्य के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22c743f628a17d3bb71afc699c7c5a5a9b963255 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_history.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित इतिहास के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9d4905a3a02ee37d1e2be7292fca0d77aeac42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_law.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित कानून के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c0aea90b194a90fe573bebcfa14a4ab73878d13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_math.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित गणित के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..814017afebbe530340cee9c3b19358ec7ed2b3dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_other.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित अन्य के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4886009317866429e37fa3d7f8759e9a6be8eae3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित दर्शनशास्त्र के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68cdd4915e3e675fd97d7dd7730b2399268ad33a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_physics.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित भौतिकी के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de1224de07165db1cd278b2b1dd41da38593c2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_psychology.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित मनोविज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb5569f7cf32f4d30407cdd74a2e205ad404805 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ja +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '答えは \(?([ABCDEFGHIJ])\)? です' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "質問：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5887f7b82fe5696476dd37da284c919bbc5964f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ja +task: +- mmlu_prox_ja_biology +- mmlu_prox_ja_business +- mmlu_prox_ja_chemistry +- mmlu_prox_ja_computer_science +- mmlu_prox_ja_economics +- mmlu_prox_ja_engineering +- mmlu_prox_ja_health +- mmlu_prox_ja_history +- mmlu_prox_ja_law +- mmlu_prox_ja_math +- mmlu_prox_ja_other +- mmlu_prox_ja_philosophy +- mmlu_prox_ja_physics +- mmlu_prox_ja_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd68bb572980342e25cc225cec2dcacf977c08a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml @@ -0,0 +1,7 @@ +description: '以下は生物学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..261aa26027c0a57b6bea94a77e5195b1b6c14a81 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml @@ -0,0 +1,7 @@ +description: '以下はビジネスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afe4e74f3010fc98aad3e463fc669302846b2d70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml @@ -0,0 +1,7 @@ +description: '以下は化学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e40a659312a40b77bccedf8907b2c157c788c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml @@ -0,0 +1,7 @@ +description: '以下はコンピュータサイエンスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c26fb8e97351c1d70e360ff14dd39287600a379b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml @@ -0,0 +1,7 @@ +description: '以下は経済学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bed0bc1965078f863dded91da78857514051bc20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml @@ -0,0 +1,7 @@ +description: '以下は工学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4b820595fc9a4b4f2568bd7ce62bee0f45ffbae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml @@ -0,0 +1,7 @@ +description: '以下は健康科学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5eb26103a2a23e316dc66f0fe1b894ddd481d256 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml @@ -0,0 +1,7 @@ +description: '以下は歴史に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..512846f6d3f477594a7a1b63b09700a435376378 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml @@ -0,0 +1,7 @@ +description: '以下は法律に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef030be6d9c666054adb4c3fe42b60a17375e835 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml @@ -0,0 +1,7 @@ +description: '以下は数学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9aa8183eb7705fcda2c0dcae5bbee45b4af524b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml @@ -0,0 +1,7 @@ +description: '以下はその他に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1581c5b2bb96e0063947d64d446bc6f090a105d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_philosophy.yaml @@ -0,0 +1,7 @@ +description: '以下は哲学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31aa1a658a6a766d74b609e2b9a68b2f226c0741 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml @@ -0,0 +1,7 @@ +description: '以下は物理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8890b927155ea6cd54d2e3c9cbe63943693399c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml @@ -0,0 +1,7 @@ +description: '以下は心理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe7abb531d727e138fd21785b4ee08da82d9f896 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ko +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '답은 \(?([ABCDEFGHIJ])\)?입니다' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "질문：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f86b5b35952ef422018ec2acceff591aded3ef8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ko +task: +- mmlu_prox_ko_biology +- mmlu_prox_ko_business +- mmlu_prox_ko_chemistry +- mmlu_prox_ko_computer_science +- mmlu_prox_ko_economics +- mmlu_prox_ko_engineering +- mmlu_prox_ko_health +- mmlu_prox_ko_history +- mmlu_prox_ko_law +- mmlu_prox_ko_math +- mmlu_prox_ko_other +- mmlu_prox_ko_philosophy +- mmlu_prox_ko_physics +- mmlu_prox_ko_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcd695f6188a184352e52c2f540f6c9e0739e412 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml @@ -0,0 +1,8 @@ +description: '다음은 생물학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ee56c9e83f38ddff9f070c43ef15a42415a135 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml @@ -0,0 +1,8 @@ +description: '다음은 경영학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ad92ca44843cb5a15d07c529c7bb9a368092bdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml @@ -0,0 +1,8 @@ +description: '다음은 화학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b33bb33c886dcefd5e6345d811aede440d76a02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml @@ -0,0 +1,8 @@ +description: '다음은 컴퓨터 과학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a7df38786dc807e2620f40b4d177d732c76d84b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml @@ -0,0 +1,8 @@ +description: '다음은 경제학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f2852eb4eb98595012a0fe824db62d78c6529fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml @@ -0,0 +1,8 @@ +description: '다음은 공학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9322b2074a5238dfbd078b457dd3d2b66abb9c3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml @@ -0,0 +1,8 @@ +description: '다음은 건강에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f82005689f11631dc8506d785d87420563186980 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml @@ -0,0 +1,8 @@ +description: '다음은 역사에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7db87be068f8c3751b934f1a0f719ebb07a5eb3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_law.yaml @@ -0,0 +1,8 @@ +description: '다음은 법률에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90c1e5fac04e3f91712ced30915d9d18fe6d0dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml @@ -0,0 +1,8 @@ +description: '다음은 수학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50aad077e34943e8fed2e62543f9ef23c5fc33aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml @@ -0,0 +1,8 @@ +description: '다음은 기타에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c274a0e3cd9c66e8fb323d71d6c7081fbff0aee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml @@ -0,0 +1,8 @@ +description: '다음은 철학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be0250ec9311ea3f42cc048dba16e9d4a3fd77b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml @@ -0,0 +1,8 @@ +description: '다음은 물리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06b8aafbbe554e4a748686597349a33c1bbfbc5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml @@ -0,0 +1,8 @@ +description: '다음은 심리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/lang_libs.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/lang_libs.py new file mode 100644 index 0000000000000000000000000000000000000000..9f6e350528dbf1bf2f1adc0adf15a7d14a1adfbe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/lang_libs.py @@ -0,0 +1,318 @@ +LANG_LIBS = { + "en": [ + "Question:", + "Options:", + "Answer: Let's think step by step.", + 'The following are multiple choice questions (with answers) about {subject}. Think step by step and then finish your answer with "{ans_suffix}" where X is the correct letter choice.', + "A: Let's think step by step.", + "the answer is ({})", + ], + "ja": [ + "質問：", + "選択肢：", + "回答：一歩一歩考えていきましょう。", + "以下は{subject}に関する選択問題（解答付き）です。段階的に考え、最後に「{ans_suffix}」と回答を締めくくってください。Xは正解の選択肢を示す文字です。", + "A: 一歩一歩考えていきましょう。", + "答えは ({}) です", + ], + "zh": [ + "问题：", + "选项：", + "答案：让我们一步一步地思考。", + '以下是关于{subject}的选择题（带有答案）。请逐步思考，然后以"{ans_suffix}"结束您的回答，其中X是正确的选项字母。', + "A: 让我们一步一步地思考。", + "答案是 ({})", + ], + "ko": [ + "질문：", + "선택 사항：", + "답변: 한 단계씩 생각해 봅시다.", + '다음은 {subject}에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "{ans_suffix}"로 답변을 마무리하세요. 여기서 X는 올바른 선택지 문자입니다.', + "A: 한 단계씩 생각해 봅시다.", + "답은 ({})입니다", + ], + "fr": [ + "Question :", + "Options :", + "Réponse : Réfléchissons étape par étape.", + 'Voici des questions à choix multiples (avec réponses) sur {subject}. Réfléchissez étape par étape, puis terminez votre réponse par "{ans_suffix}" où X est la lettre correspondant au bon choix.', + "A: Réfléchissons étape par étape.", + "La réponse est ({})", + ], + "de": [ + "Frage:", + "Optionen:", + "Antwort: Denken wir Schritt für Schritt nach.", + 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu {subject}. Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "{ans_suffix}", wobei X der richtige Buchstabe ist.', + "A: Denken wir Schritt für Schritt nach.", + "Die Antwort ist ({})", + ], + "es": [ + "Pregunta:", + "Opciones:", + "Respuesta: Pensemos paso a paso.", + 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre {subject}. Piense paso a paso y luego termine su respuesta con "{ans_suffix}" donde X es la letra de la opción correcta.', + "A: Pensemos paso a paso.", + "La respuesta es ({})", + ], + "pt": [ + "Pergunta:", + "Opções:", + "Resposta: Vamos pensar passo a passo.", + 'A seguir estão perguntas de múltipla escolha (com respostas) sobre {subject}. Pense passo a passo e termine sua resposta com "{ans_suffix}" onde X é a letra da opção correta.', + "A: Vamos pensar passo a passo.", + "A resposta é ({})", + ], + "sw": [ + "Swali:", + "Chaguo:", + "Jibu: Hebu tufikiria hatua kwa hatua.", + 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu {subject}. Fikiria hatua kwa hatua kisha malizia jibu lako kwa "{ans_suffix}" ambapo X ni herufi ya chaguo sahihi.', + "A: Hebu tufikiria hatua kwa hatua.", + "Jibu ni ({})", + ], + "th": [ + "คำถาม:", + "ตัวเลือก:", + "คำตอบ: มาคิดทีละขั้นตอนกัน", + 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ {subject} คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย "{ans_suffix}" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง', + "A: มาคิดทีละขั้นตอนกัน", + "คำตอบคือ ({})", + ], + "ar": [ + "سؤال:", + "الخيارات:", + "الإجابة: دعنا نفكر خطوة بخطوة.", + "فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول {subject}. فكر خطوة بخطوة ثم أنهِ إجابتك بـ '{ans_suffix}' حيث X هو حرف الخيار الصحيح.", + "أ: دعنا نفكر خطوة بخطوة.", + "الإجابة هي ({})", + ], + "hi": [ + "प्रश्न:", + "विकल्प:", + "उत्तर: चलिए चरण-दर-चरण सोचते हैं।", + 'निम्नलिखित {subject} के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "{ans_suffix}" के साथ समाप्त करें जहां X सही विकल्प का अक्षर है।', + "A: चलिए चरण-दर-चरण सोचते हैं।", + "उत्तर है ({})", + ], + "bn": [ + "প্রশ্ন:", + "বিকল্পগুলি:", + "উত্তর: আসুন ধাপে ধাপে চিন্তা করি।", + 'নিম্নলিখিত {subject} সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা করুন এবং তারপর আপনার উত্তর "{ans_suffix}" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের অক্ষর।', + "A: আসুন ধাপে ধাপে চিন্তা করি।", + "উত্তর হল ({})", + ], +} + + +LANG_SUBJECTS = { + "en": { + "biology": "biology", + "business": "business", + "chemistry": "chemistry", + "computer_science": "computer_science", + "economics": "economics", + "engineering": "engineering", + "health": "health", + "history": "history", + "law": "law", + "math": "math", + "other": "other", + "philosophy": "philosophy", + "physics": "physics", + "psychology": "psychology", + }, + "ja": { + "biology": "生物学", + "business": "ビジネス", + "chemistry": "化学", + "computer_science": "コンピュータサイエンス", + "economics": "経済学", + "engineering": "工学", + "health": "健康科学", + "history": "歴史", + "law": "法律", + "math": "数学", + "other": "その他", + "philosophy": "哲学", + "physics": "物理学", + "psychology": "心理学", + }, + "zh": { + "biology": "生物学", + "business": "商业", + "chemistry": "化学", + "computer_science": "计算机科学", + "economics": "经济学", + "engineering": "工程学", + "health": "健康", + "history": "历史", + "law": "法律", + "math": "数学", + "other": "其他", + "philosophy": "哲学", + "physics": "物理学", + "psychology": "心理学", + }, + "ko": { + "biology": "생물학", + "business": "경영학", + "chemistry": "화학", + "computer_science": "컴퓨터 과학", + "economics": "경제학", + "engineering": "공학", + "health": "건강", + "history": "역사", + "law": "법률", + "math": "수학", + "other": "기타", + "philosophy": "철학", + "physics": "물리학", + "psychology": "심리학", + }, + "fr": { + "biology": "biologie", + "business": "commerce", + "chemistry": "chimie", + "computer_science": "informatique", + "economics": "économie", + "engineering": "ingénierie", + "health": "santé", + "history": "histoire", + "law": "droit", + "math": "mathématiques", + "other": "autre", + "philosophy": "philosophie", + "physics": "physique", + "psychology": "psychologie", + }, + "de": { + "biology": "Biologie", + "business": "Wirtschaft", + "chemistry": "Chemie", + "computer_science": "Informatik", + "economics": "Ökonomie", + "engineering": "Ingenieurwesen", + "health": "Gesundheit", + "history": "Geschichte", + "law": "Recht", + "math": "Mathematik", + "other": "Sonstiges", + "philosophy": "Philosophie", + "physics": "Physik", + "psychology": "Psychologie", + }, + "es": { + "biology": "biología", + "business": "negocios", + "chemistry": "química", + "computer_science": "informática", + "economics": "economía", + "engineering": "ingeniería", + "health": "salud", + "history": "historia", + "law": "derecho", + "math": "matemáticas", + "other": "otro", + "philosophy": "filosofía", + "physics": "física", + "psychology": "psicología", + }, + "pt": { + "biology": "biologia", + "business": "negócios", + "chemistry": "química", + "computer_science": "ciência da computação", + "economics": "economia", + "engineering": "engenharia", + "health": "saúde", + "history": "história", + "law": "direito", + "math": "matemática", + "other": "outro", + "philosophy": "filosofia", + "physics": "física", + "psychology": "psicologia", + }, + "sw": { + "biology": "biolojia", + "business": "biashara", + "chemistry": "kemia", + "computer_science": "sayansi ya kompyuta", + "economics": "uchumi", + "engineering": "uhandisi", + "health": "afya", + "history": "historia", + "law": "sheria", + "math": "hisabati", + "other": "nyingine", + "philosophy": "falsafa", + "physics": "fizikia", + "psychology": "saikolojia", + }, + "th": { + "biology": "ชีววิทยา", + "business": "ธุรกิจ", + "chemistry": "เคมี", + "computer_science": "วิทยาการคอมพิวเตอร์", + "economics": "เศรษฐศาสตร์", + "engineering": "วิศวกรรมศาสตร์", + "health": "สุขภาพ", + "history": "ประวัติศาสตร์", + "law": "กฎหมาย", + "math": "คณิตศาสตร์", + "other": "อื่นๆ", + "philosophy": "ปรัชญา", + "physics": "ฟิสิกส์", + "psychology": "จิตวิทยา", + }, + "ar": { + "biology": "علم الأحياء", + "business": "الأعمال", + "chemistry": "الكيمياء", + "computer_science": "علوم الكمبيوتر", + "economics": "الاقتصاد", + "engineering": "الهندسة", + "health": "الصحة", + "history": "التاريخ", + "law": "القانون", + "math": "الرياضيات", + "other": "أخرى", + "philosophy": "الفلسفة", + "physics": "الفيزياء", + "psychology": "علم النفس", + }, + "hi": { + "biology": "जीव विज्ञान", + "business": "व्यापार", + "chemistry": "रसायन विज्ञान", + "computer_science": "कंप्यूटर विज्ञान", + "economics": "अर्थशास्त्र", + "engineering": "इंजीनियरिंग", + "health": "स्वास्थ्य", + "history": "इतिहास", + "law": "कानून", + "math": "गणित", + "other": "अन्य", + "philosophy": "दर्शनशास्त्र", + "physics": "भौतिकी", + "psychology": "मनोविज्ञान", + }, + "bn": { + "biology": "জীববিজ্ঞান", + "business": "ব্যবসা", + "chemistry": "রসায়ন", + "computer_science": "কম্পিউটার বিজ্ঞান", + "economics": "অর্থনীতি", + "engineering": "প্রকৌশল", + "health": "স্বাস্থ্য", + "history": "ইতিহাস", + "law": "আইন", + "math": "গণিত", + "other": "অন্যান্য", + "philosophy": "দর্শন", + "physics": "পদার্থবিজ্ঞান", + "psychology": "মনোবিজ্ঞান", + }, +} diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..52b9d8206a9394ea8089d985293f83a9ed216b69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py @@ -0,0 +1,114 @@ +import os +import shutil + +import yaml +from lang_libs import LANG_LIBS, LANG_SUBJECTS + + +language_word_to_abbr = { + "English": "en", + "Japanese": "ja", + "Chinese": "zh", + "Korean": "ko", + "French": "fr", + "German": "de", + "Spanish": "es", + "Portuguese": "pt", + "Swahili": "sw", + "Thai": "th", + "Arabic": "ar", + "Hindi": "hi", + "Bengali": "bn", +} + +language_abbr_to_word = {v: k for k, v in language_word_to_abbr.items()} + + +if __name__ == "__main__": + mmlu_pro_config_dir = "../mmlu_pro" + mmlu_prox_repo_id = "li-lab/MMLU-ProX" + + for lang_abbr in language_abbr_to_word: + os.makedirs(lang_abbr, exist_ok=True) + lang_lib_list = LANG_LIBS[lang_abbr] + lang_sbj_dict = LANG_SUBJECTS[lang_abbr] + + with ( + open("template/_lang_template_yaml", "r") as reader, + open(f"{lang_abbr}/_{lang_abbr}_template_yaml", "w") as writer, + ): + for line in reader.readlines(): + if "{repo_id}" in line: + line = line.format(repo_id=mmlu_prox_repo_id) + if "{lang}" in line: + line = line.format(lang=lang_abbr) + if "{ans_regex}" in line: + ans_regex = lang_lib_list[-1].replace( + "({})", "\(?([ABCDEFGHIJ])\)?" + ) + if lang_abbr == "en": + ans_regex = ans_regex.lstrip("the").strip() + line = line.format(ans_regex=ans_regex) + if "{que_prefix}" in line: + line = line.format(que_prefix=lang_lib_list[0]) + writer.write(line) + + shutil.copy("template/utils.py", f"{lang_abbr}/utils.py") + + group_name = f"mmlu_prox_{lang_abbr}" + group_dict = dict( + group=group_name, + task=[f"{group_name}_{sbj}" for sbj in LANG_SUBJECTS[lang_abbr]], + aggregate_metric_list=[ + dict( + aggregation="mean", + metric="exact_match", + weight_by_size=True, + filter_list="custom-extract", + ) + ], + metadata=dict(version=0.0), + ) + with open(f"{lang_abbr}/_{group_name}.yaml", "w", encoding="utf-8") as f: + yaml.dump( + group_dict, + f, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) + + for sbj in lang_sbj_dict: + with open( + f"{mmlu_pro_config_dir}/mmlu_pro_{sbj}.yaml", "r", encoding="utf-8" + ) as f: + sbj_yaml_last_line = None + for line in f.readlines(): + if line.startswith("process_docs:"): + sbj_yaml_last_line = line.strip() + + sbj_dict = dict( + description=lang_lib_list[3].format( + subject=lang_sbj_dict[sbj], ans_suffix=lang_lib_list[5].format("X") + ) + + "\n", + include=f"_{lang_abbr}_template_yaml", + task=f"{group_name}_{sbj}", + task_alias=sbj, + ) + with open( + f"{lang_abbr}/{group_name}_{sbj}.yaml", "w", encoding="utf-8" + ) as f: + yaml.dump( + sbj_dict, + f, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) + with open( + f"{lang_abbr}/{group_name}_{sbj}.yaml", "a", encoding="utf-8" + ) as f: + f.write(sbj_yaml_last_line + "\n") + + print(f"Finished {lang_abbr}") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7f60973dfcfeb5ed6cd9810b89ae161f8ed692f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_pt +task: +- mmlu_prox_pt_biology +- mmlu_prox_pt_business +- mmlu_prox_pt_chemistry +- mmlu_prox_pt_computer_science +- mmlu_prox_pt_economics +- mmlu_prox_pt_engineering +- mmlu_prox_pt_health +- mmlu_prox_pt_history +- mmlu_prox_pt_law +- mmlu_prox_pt_math +- mmlu_prox_pt_other +- mmlu_prox_pt_philosophy +- mmlu_prox_pt_physics +- mmlu_prox_pt_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac461a3f88c4943d40a13416681e08a31beee70d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: pt +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'A resposta é \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pergunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45be5da1e1e4f5e36dd9e98e106ad057ed2ab218 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_biology.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre biologia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a3e99d803b7cea15f142af54c139be01a200f9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_business.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre negócios. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88116d1e41ff47578872511ea2dd1070e962570f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre química. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f77faf8b1f49006e020063c784318a628f2fa653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre ciência + da computação. Pense passo a passo e termine sua resposta com "A resposta é (X)" + onde X é a letra da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d06e0099f5d9fa664563df619fd206339a0e1f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_economics.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre economia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4f1a61d0322d5171e4acef4164e57190cbe2f2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_engineering.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre engenharia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01d264e9339c3c5ceb4170c9fcc0e132a370df3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_health.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre saúde. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97c5b73645a530a06f79d1edfda261ce96b3ba90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_history.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre história. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba48aa2d7d79005414eddbe194f1ee77986b9969 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_law.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre direito. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19a4126812c9b5bd0a256e617d8e494029863fb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_math.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre matemática. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6607fcdb045a800defcf4d66c5ec267068d8bb83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_other.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre outro. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87ff1ce438bf020285227b923e6593fca568c071 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre filosofia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f71509235eecc2886448394014f15263c139dd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_physics.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre física. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a09632b6b84a09b24b756cf62e8d9c50d5071acb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_psychology.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre psicologia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9a2fba188a31ca78e8b243b438fa33882804ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_sw +task: +- mmlu_prox_sw_biology +- mmlu_prox_sw_business +- mmlu_prox_sw_chemistry +- mmlu_prox_sw_computer_science +- mmlu_prox_sw_economics +- mmlu_prox_sw_engineering +- mmlu_prox_sw_health +- mmlu_prox_sw_history +- mmlu_prox_sw_law +- mmlu_prox_sw_math +- mmlu_prox_sw_other +- mmlu_prox_sw_philosophy +- mmlu_prox_sw_physics +- mmlu_prox_sw_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_sw_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_sw_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fea93e23686d80e1d1492b9696f9fab1f4ee95e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_sw_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: sw +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Jibu ni \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Swali:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63230a9ee9d88e8a3c9503b21870a8cb4daa032b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_biology.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu biolojia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..981361982468f4ce636611f81f903d0ea2b7bef6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_business.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu biashara. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f87ac7e3f040b8c9763843d55c9534e89bed161 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu kemia. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4168b229e94536b0cf838a65313786815314f9d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sayansi + ya kompyuta. Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo + X ni herufi ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adb5ce0f7101378609e5305eddb544ea8b810e46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uchumi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd0d6267ec7827c5ee0b0384c1e27b263bb51dad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uhandisi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4db109fb1c01f4de671c63da6ecefbd69bf2a2e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu afya. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b4b39d4e8a4790487bd49bbc5fc9a70d82b9ee7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu historia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e49ae99802fcb5ade0dd5dc61e7fa8402507dfc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sheria. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a940e122be05893977aed1b86d1ecc47c6850fb0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_math.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu hisabati. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96dff12c7cc341e96555b760afe4aae14b579bae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_other.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu nyingine. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89d8227df565e7f0b78dd187c3c28eac32d1d89e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu falsafa. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d9ade70dd5f97299365656ff68e80508a42b126 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_physics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu fizikia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..056aff1f336826526dc3d0291458bfba4b233f75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu saikolojia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/_lang_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/_lang_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..879d0767d6544254c5394baa88a9cf3022b247a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/_lang_template_yaml @@ -0,0 +1,35 @@ +dataset_path: {repo_id} +dataset_name: {lang} +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '{ans_regex}' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "{que_prefix}" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_th.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e65fdd1a05573abfd120e67245ee224d44faaedb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_th.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_th +task: +- mmlu_prox_th_biology +- mmlu_prox_th_business +- mmlu_prox_th_chemistry +- mmlu_prox_th_computer_science +- mmlu_prox_th_economics +- mmlu_prox_th_engineering +- mmlu_prox_th_health +- mmlu_prox_th_history +- mmlu_prox_th_law +- mmlu_prox_th_math +- mmlu_prox_th_other +- mmlu_prox_th_philosophy +- mmlu_prox_th_physics +- mmlu_prox_th_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9889e69e5dba9f4e72516f8aa3c41376355b4bd6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: th +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'คำตอบคือ \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "คำถาม:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fa50a4b7e199f0730ecd3a9846b22c8936742d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_biology.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ชีววิทยา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5f524061f142bc2008bde5ccde22ead5005253b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_business.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ธุรกิจ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1b392993e5544d8aded0557a23675dc5d988d60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_chemistry.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ เคมี คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..881fae2ed6e013ce00e7b13f40eaa03f47d04400 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_computer_science.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ วิทยาการคอมพิวเตอร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7bfe2ba74eddec5b1bef08b4dfdb73df9811329 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_economics.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ เศรษฐศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d972eee80dde43d0c53f3acd2c17f9a3e966b61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_engineering.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ วิศวกรรมศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f866b2b20e9a4541791ba7d94b6dfc701bba214 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_health.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ สุขภาพ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfa2431f5a8ac1b5cbc3ed5fd4502757ef9a4084 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_history.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ประวัติศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2f257dcb8a1e01f8f8aa4c2021e6981ce3b4867 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_law.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ กฎหมาย คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..331b04dfe5157620b2fca816f92bac0136b170b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_math.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ คณิตศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66e077d19e4ca2f7823481d44635bb92d99add3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_other.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ อื่นๆ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49bc416fa3d1ff9232af5ec09cc1b443a22f4c50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_philosophy.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ปรัชญา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dd6f9eedc52db34782d375f1e2b2ed678bb4d2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_physics.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ฟิสิกส์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a6144ac62092e0db28fcc2cd5d1176ae61c08e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_psychology.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ จิตวิทยา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/_mmlu_prox_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/_mmlu_prox_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdf03b7c9a6d31d71e6267918881c4595b99d980 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/_mmlu_prox_zh.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_zh +task: +- mmlu_prox_zh_biology +- mmlu_prox_zh_business +- mmlu_prox_zh_chemistry +- mmlu_prox_zh_computer_science +- mmlu_prox_zh_economics +- mmlu_prox_zh_engineering +- mmlu_prox_zh_health +- mmlu_prox_zh_history +- mmlu_prox_zh_law +- mmlu_prox_zh_math +- mmlu_prox_zh_other +- mmlu_prox_zh_philosophy +- mmlu_prox_zh_physics +- mmlu_prox_zh_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/_zh_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/_zh_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ae8dee4752a4ee8b805b76edd48285c2dfa88f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/_zh_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: zh +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '答案是 \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "问题：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8548ed384067e762f565cd9d0621277273a2073a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_biology.yaml @@ -0,0 +1,7 @@ +description: '以下是关于生物学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5ba92d08b1317f7df12b8e4e931edef52ff552a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_business.yaml @@ -0,0 +1,7 @@ +description: '以下是关于商业的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0da4be23843cc9ce9c6417ec0367c9d3dc7f344f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_chemistry.yaml @@ -0,0 +1,7 @@ +description: '以下是关于化学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6c674097144d046c0346e9dfb1e1399b9a49918 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_computer_science.yaml @@ -0,0 +1,7 @@ +description: '以下是关于计算机科学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c7e8697c7ba7b5f0ef920e4283858a8c5fccde5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_economics.yaml @@ -0,0 +1,7 @@ +description: '以下是关于经济学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6de2d39214e9714e0559e6298844240a918bbb43 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_engineering.yaml @@ -0,0 +1,7 @@ +description: '以下是关于工程学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d78cf922a96dd851663a35b113fda2365ecd4071 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_health.yaml @@ -0,0 +1,7 @@ +description: '以下是关于健康的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e51d7c084467f253e014bebe5c0359e1dd947fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_history.yaml @@ -0,0 +1,7 @@ +description: '以下是关于历史的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bda370e7bd7599602f6918fcf0490a3d8da0e343 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_law.yaml @@ -0,0 +1,7 @@ +description: '以下是关于法律的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d81f868fc6706af670c41e86864ca2558e37178d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_math.yaml @@ -0,0 +1,7 @@ +description: '以下是关于数学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cbc9f052f6fadb009311f21242753adb11d17e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_other.yaml @@ -0,0 +1,7 @@ +description: '以下是关于其他的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9038c23f65b12c6efc9f722d54ff4715ca79efe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_philosophy.yaml @@ -0,0 +1,7 @@ +description: '以下是关于哲学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f4b874ab0aaef51ec2e9e75e5199b53dc744a93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_physics.yaml @@ -0,0 +1,7 @@ +description: '以下是关于物理学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab2a43da8e31dd5429d5d66205a30d52adea3122 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_psychology.yaml @@ -0,0 +1,7 @@ +description: '以下是关于心理学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6d8a79fbab7bf6373b75481267791553332536a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md @@ -0,0 +1,64 @@ +# MMLU-SR + +## Paper +Title: [Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models](https://arxiv.org/abs/2406.15468v1) + + +We propose MMLU-SR, a novel dataset designed to measure the true comprehension abilities of Large Language Models (LLMs) by challenging their performance in question-answering tasks with modified terms. We reasoned that an agent that ``truly'' understands a concept can still evaluate it when key terms are replaced by suitably defined alternate terms, and sought to differentiate such comprehension from mere text replacement. In our study, we modified standardized test questions by replacing a key term with a dummy word along with its definition. The key term could be in the context of questions, answers, or both questions and answers. +Notwithstanding the high scores achieved by recent popular LLMs on the MMLU leaderboard, we found a substantial reduction in model performance after such replacement, suggesting poor comprehension. This new benchmark provides a rigorous benchmark for testing true model comprehension, and poses a challenge to the broader scientific community. + +Github Homepage: [https://github.com/Wang-ML-Lab/MMLU-SR](https://github.com/Wang-ML-Lab/MMLU-SR) +Huggingface Dataset: [https://huggingface.co/datasets/NiniCat/MMLU-SR]([https://huggingface.co/datasets/NiniCat/MMLU-SR) + + +## Citation +```bib +@misc{wang2024reasoningsimplytokenprediction, + title={Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models}, + author={Wentian Wang and Paul Kantor and Jacob Feldman and Lazaros Gallos and Hao Wang}, + year={2024}, + eprint={2406.15468}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.15468}, +} +``` + +### Groups and Tasks + +#### Groups + +- `mmlusr`: MMLU variant where the terminology in the question and answers are modified. +- `mmlusr_answer_only`: MMLU variant where the terminology in the answers are modified. +- `mmlusr_question_only`: MMLU variant where the terminology in the question is modified. + +#### Tasks + +There are 57 symbol replaced subjects in each group. You can run a single task by: + +* `mmlusr_question_only_abstract_algebra` + +Or by categories: + +* `mmlusr_question_only_stem_tasks ` + + +### Checklist + +The checklist is the following: + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical. + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Variant Wishlist + +- [ ] zero-shot variant diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2473a2af3ade70dd004a3756a61d9b83bba36179 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml @@ -0,0 +1,44 @@ +group: mmlusr_answer_only +group_alias: MMLU-SR (Answer Only) +task: + - group: mmlusr_ao_stem + group_alias: STEM (Answer Only) + task: + - mmlusr_answer_only_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_other + group_alias: Other (Answer Only) + task: + - mmlusr_answer_only_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_social_sciences + group_alias: Social Sciences (Answer Only) + task: + - mmlusr_answer_only_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_humanities + group_alias: Humanities (Answer Only) + task: + - mmlusr_answer_only_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..527bc9cc1b3ebe1004e26a68e5fffcee7158d4ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e4acc8c3810cc6f8bbe32216b8d0cdfa9463135 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_anatomy" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..068072de60489d7c26b576314862c73492af92c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e836e31c55e8cb6b203049f75c13aaf57870a6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_business_ethics" +"task_alias": "business ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ef709675c2c7016d2aabbcde3bb3ff3dcafcbcc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b967895a7017094a9b5f78ad9ddbeff7a56ab6f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_biology" +"task_alias": "college biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dd100e7bc82bcc44d7f121e0dab7852dbc69074 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbd7e4c1582a780028ddaf3cb2f1824dd4b3ebb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d85c49dc1c76b880ea1bc4ae255c2d152d6eaa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..132e0b6041126c058b61a3ddcb27e51383134c77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77b47241d095b8a62ebdfe2d995ad438c69de4ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba3d60d51b852e27637d1c1ab43af1d4da7fc057 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_computer_security" +"task_alias": "computer security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0a84ecc3c5015d2338cfffc88877e39b9be7c75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..996d44f46eab624e768c92553659847682e2f2cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab695e6ab46ea036cdccd91487c47aca35f1a99a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff9fbf25b49d3b59d0ae716cd4bf3a210547742 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e26ed865bc94d0923f7f1103d89f556a914df8aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_formal_logic" +"task_alias": "formal logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec9c0f42b35eda761c005ddb94670e649a0a9ee8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_global_facts" +"task_alias": "global facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41ed53cb9a75af634f1b241da2345985f72b65e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95a3303f3de284b7759a08dba7440398d009296f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e665fb3400855e8eefc2b8d8dc8054bb0b48279e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d7c1cb8da0a7de39a28efbb39f943b0d230d93f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a4980060190c2d9870a7da722d8a28aa3d2f3af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf66e3a3a760c5cf5b81d4b83323230c3aafe1ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95e35cd8b177c7aab73d90f85f29812d4c2d69fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7da2d1859ad5c85fa25d3579156e66d9ec2e5e45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3af9a2c79bf4f5c090a309138139350d40581be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52fb7377927412590328af0a2ad21c845ba21322 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_physics" +"task_alias": "high school physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df77619cbb8d2b8ccc42083beac223920833e3ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2119fb39d1cf03028a61f6f57cac66343ba308ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2287ae457a1f4013ac607955f0050d7ce7f1caad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b8f4f37e2e23019f522ecd0c39406485943d808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a188ddb6517a367d83b3791c0875d9fd01ad56f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18c45333c5ba7b3a5cc80fa340c7f58481f4a4b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05e482d1683ab518d7b7258ee99b8cc532af0f00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73edd6cb293f97e2ae112c159602fc6f57caeb24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab18c9270e07ff3101fcb25e219f94e1ff80aef0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b833c706f24750d113100414d7b8857c936899e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26ec67401daf8f5e3af5f6c4b1ca8937927a6850 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23fe03659ba9df8e12873d9a3065de6093b8834e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63355c88aa30126955f5fc14be0b8b54f47a5169 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12153929983be65dcbcbc59b195414607dd1ecae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f09854fbaa531d5a6e694f578dcf18ac1947395 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dee1c01eb1b9ca1cb85ae2c0ba8d0638a78c9515 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a890f9331b76cfcfd706cec97c87ae9b358f9b33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..538dea756ccc6b97d3071922fa1a2f67d5daede8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a93b5c4ff7034736e17af2b48e876f2ccdadbff9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f45995cbf30cf4453f30ed47a644718349c695 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caccccf0ded8e25f7c92e4788885583f5f014168 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..374b239c3fbda26af4691d302350cf60216e7ede --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58a9fc2d31f1c9758e0fbb9b90f13922cfb20247 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86cc337b0642aa2043df46db35aa26b2c77e8ccc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e72f02f5595c91afbb06a97414bc78866c47203 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58fa3d8de1b736f727119945ec4c59470f550d6e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91a6d66340278bc0ff53085d88c910779d76d7e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1400fb8421512daebaf5640a8385e69ce645e59d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6014213538672bf733088ea63b9d1262f47a1ff4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py new file mode 100644 index 0000000000000000000000000000000000000000..8394806417992d71582c1ce062c7945e77aef2f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py @@ -0,0 +1,154 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger(__name__) + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + +GROUPS = ["question_and_answer"] + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Generate configuration YAML files for LM Evaluation Harness." + ) + # Path to the base YAML file from which to inherit settings + parser.add_argument( + "--base_yaml_path", + required=True, + help="Path to the base YAML configuration file.", + ) + + # Directory where the generated YAML files will be saved + parser.add_argument( + "--save_dir", + default="/data/local/cat/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer", + ) + + # Optional prefix to add to task names in the YAML files + parser.add_argument("--task_prefix", default="") + + parser.add_argument("--cot_prompt_path", default=None) + + # Optional prefix to add to group names in the YAML files + parser.add_argument("--group_prefix", default="") + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # Load base YAML configuration + base_yaml_name = os.path.basename(args.base_yaml_path) + with open(args.base_yaml_path, "r", encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path, encoding="utf-8") as f: + cot_file = json.load(f) + + for group in GROUPS: + for subject, category in tqdm(SUBJECTS.items()): + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"mmlusr_{args.group_prefix}{group}_{category}" + if args.group_prefix + else f"mmlusr_{group}_{category}", + "task": f"mmlusr_{args.task_prefix}{group}_{subject}" + if args.task_prefix + else f"mmlusr_{group}_{subject}", + "task_alias": subject.replace("_", " "), + "description": description, + "dataset_name": f"{group}_{subject}", + } + + # File path for saving the generated YAML file + file_save_path = os.path.join(args.save_dir, f"{group}_{subject}.yaml") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(yaml_dict, yaml_file, allow_unicode=True, default_style='"') + eval_logger.info(f"Saved YAML for {group} {subject} to {file_save_path}") + + # Save group configuration if specified + if args.group_prefix: + file_save_path = os.path.join( + args.save_prefix_path, args.group_prefix + ".yaml" + ) + eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(yaml_dict, yaml_file, indent=4, default_flow_style=False) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..390cbc02770f8834623caa7aaadc61673b8b1316 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml @@ -0,0 +1,44 @@ +group: mmlusr +group_alias: MMLU-SR (Question & Answer) +task: + - group: mmlusr_qa_stem + group_alias: STEM (Question & Answer) + task: + - mmlusr_question_and_answer_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_other + group_alias: Other (Question & Answer) + task: + - mmlusr_question_and_answer_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_social_sciences + group_alias: Social Sciences (Question & Answer) + task: + - mmlusr_question_and_answer_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_humanities + group_alias: Humanities (Question & Answer) + task: + - mmlusr_question_and_answer_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfdd80e6422675dd71ba5b9c50d1a4f7d70f1094 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..316bede4232b218f1c9beda91c9b7b3069aebc9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_anatomy" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9f89e1c9700d86611e1f0ec6079ce31cf8723d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a46298259c80432c775638a847511c8159e9b75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_business_ethics" +"task_alias": "business ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43c9a3d54e1a757f1653f7f9d4027fe0052db4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f00615bdf93f1e4f86e71cc070e825b098482d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_biology" +"task_alias": "college biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..837bc2253831748957f28911fe56fdec68791e6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb1c76395f154edcea570342f130f5afdfd1eeac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08c6e334c23f6f7f34a9f23c5e64acc7648dd3a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c44360ab90f4fb3678dd7f6837b503cdbb47ec3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..372d6b9f20bc2b4eb9a8ddc6266a330dc6ea3b7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f85146a96059161990172672a688c3ac7922cea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_computer_security" +"task_alias": "computer security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b37e1927f26b2963cbe86ca3a3f4f7c9a56337b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d83b0ba9da42029ea88b4b80916f272841cba6b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1898c3a97e949d8b41a0b57d64f6d7cca2ff0795 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c828feb107020806f43489f46791d2d7ba255662 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..294e99a46b2b81da8daf02a143f35bcaab85d6ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_formal_logic" +"task_alias": "formal logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c1a879a63d4805f5c74eacc12924bbb768df4c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_global_facts" +"task_alias": "global facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90fe29fa46a24551b133e2c6882e7b2d41c0ac00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e4423ae0eebe7b638f73cbc7729d570975ed2c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe1e3d49548039351950964207f0f4e884a5ac2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..933d46f021948758a5ddb11fd1d44e29996ef146 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa99ad15d1f345e1fe7f7d07f50a723340d40e79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4835f4e09cfffc60344332a66f724c4838b4668 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..252ba9ceaf7cce7fa9636565a945cdbe53470494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f88bf56047e124b4349efbbcce7018de8649e7c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef2656aff6d404e69fa3fd4655969801d1fad76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f02cc7fa7847f1564958ac7e3ccf7e15771b05bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_physics" +"task_alias": "high school physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df87039f7b5ab3118a1c6bd3530dad5642bc167f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd9244a673ec274df54065fd325193eb2ac6128 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efcf6898d2233c2f2e48dee853f02faae0fe468b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6488f284c0b8b535ad19a49a3999c06c88be079 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b103d60f713daf41524ab25f47af17cf589f552a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eac93d56265e4bf731a524a0ac7270dea84f4ef0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f7d5403cfc101bbc6f1ccbf2f833491b38fc03a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..775bee63388fb53adaa0f9ecd34ef36e2c82d272 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f2706a93f5d88e32c366ba4dee4ac18e87a24e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6299c4d06d84c7d3360771c6ca633b3230196b10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60ae89e2891e290653289a4c4467924e3fed67f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4399b96ea620efaf67af9d5a5f28bea4353543a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..477b6cf9cfcb367942135c16d59f0e59c805d14c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..204ea3ae360632adb105ec1b54494803c5325638 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ceb216f67caf3f9bcdf421450e6f786dc130b0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d434fb1cc1696b9b002be317631ad0288771d4e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e564410f9632cd39de634f071d5438f00ad83a56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9c19bc01416c529a67cd497eaf3d96539e1a09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a966669031fa998e252a49ed2d742ae99e635883 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68973e37619fb764a0d73a97218f184fd49e4a5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a158fd123b8f1c45821d728dae7b70f3a8c2ab59 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..738e24e91d01c91776951dc991388a0e426d17c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26f42c50f80254a5e390cb3996035c60005d1e8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c92e67290e0eb97b728f9b76083c00c425b1ab1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c5ba3c9c3801df6335b1f41f2a70b7a4819bcc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d41098618f71fdad4c838c65a3a310d56624c99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ced65cb6f0dd15963c4af54d2a7ee49bd6b62b62 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da7c0ca54e9f5a3fa91275cff8960f2a81e0aa4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e44bd345d5900826296561acde1900985984b3db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_question_only.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_question_only.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8c5c972107dd625b685a58810f10b33814a418f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_question_only.yaml @@ -0,0 +1,44 @@ +group: mmlusr_question_only +group_alias: MMLU-SR (Question Only) +task: + - group: mmlusr_qo_stem + group_alias: STEM (Question Only) + task: + - mmlusr_question_only_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_other + group_alias: Other (Question Only) + task: + - mmlusr_question_only_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_social_sciences + group_alias: Social Sciences (Question Only) + task: + - mmlusr_question_only_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_humanities + group_alias: Humanities (Question Only) + task: + - mmlusr_question_only_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae764f7b5f193d68d7e41f61654b42c4617306c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85fe75793d52621f6bba23d662ee3b18673ab5f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_anatomy" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e32ddfed16226fbb600a846121d7b1aa8d89758f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d6404156f5114268f12d064d1073ba83621ea9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_business_ethics" +"task_alias": "business ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33398345525ca0f7aa2f7deded01380e5123e526 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..940bddc28febc92801af615859429a4ce77e45f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_biology" +"task_alias": "college biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc7b6cdae3751fa357451a2e29a7bead8c5e67b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7feae9f0b1418bb514afe7c773fccf6eb379d1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c379c5f5f6f13a3d3b80ded80685a211dafc05e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f035787e33118323fd2c91698c4a6282fe858b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e9599e5c0036574ac0387e6056e15023b8f648 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ac0de044f7eec2d8fcce8687b9de7d68d3bc2d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_computer_security" +"task_alias": "computer security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d50b14ca1204b8b6b9d770c60ff41c5b8c37f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edd501fa06848966e1546f9fd1e3c78d4d237223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8be2f268be83fbdb8b49d7da39b0533a50f6bf5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0681dbc1dfbaa88506c770001cee4543ed5fb0cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ae64f4d656c6228707174d2cde036a1b2a0376 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_formal_logic" +"task_alias": "formal logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fe24005f69a858196c9c5727850333e765ae151 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_global_facts" +"task_alias": "global facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..030fd2e090ab8dabac6abc249f7577160b6b8ac8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f7b38e0e9be0150471694c84d62abdc2c5d6dcd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12f9d626c335be093730798bdb4116cf930030fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..746d125e5456e50e180f81d1e5beb47bd59d5840 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abe2d6f5ac68b7f53a86d6147cf8664001830444 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a7fb24eeda5201a12669824e340fe73ea65ffba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecb0772234251072aeb4fa156a10bbb81f0707ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aacf362d2f2a8f30f5e972406359479d74980bba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc288c976b09afec24cdff337ce3e648d600519a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaa42363325004e19bf462adee55bb94ac1aebe5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_physics" +"task_alias": "high school physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33085c5c2a6cf6054dd82879381365a66cc770cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae69628a60bb02fefd9320c651ae6e9df35dc181 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf226b5a437c2de1a554e7f997d7604fe1c5acf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37b67158f4559b6a23072fca79405db8f49c2ee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dd67daf3f0c9f1691974b85f2a86c8b8eadcb97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfaee537e7e91e95a276f0dd2214ad0291ace25e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fde605633bd9f8c4146a7a213ac69bdc2a100680 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2f95fd2b12870ea7e6a4cebddbe8c4d6ac0eeba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e07150c7fdecd13a80a995badaca082fdf9d0d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bccaf4a4164b9a471395fddc9b01ff8bb838108 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca72f214c40c18a7688d76ef7dcc12c69aebb11b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a47f15b6b44872250e000ff3e65529fdfd317e19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88829f61c1f0c087808364b3eb11871ea8af2302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad3de69466630145c31bc474ce399f6067f72cff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a84f610571a1eb1ed58dda6988628c4cc83f761 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ef60495f02f99e8090737056d582e7f962047f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2518b48dc991d272263f01c67908bb703b277139 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c17c5dd8758471f9d6485e1c92414d9b16bee5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2297b0f122817d57a14b59eecfc5d70a7fa02f05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a04374117fd545b719ad12fa838eae269efad696 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b8e572b9ef7b84e1baa098b57e3bc7e1d6bcb25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c25aa01755421c42b70335f7c1a8cf9ccea659a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89ebc81c7f0d9999811da446610df2b5e2a7e316 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d23cb2b93d3391944e7aa30222381051e593b8d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ff913d961a5f17bdc987659e1fdc36658b5b06e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d705e8485c85f3aa3493fad724aca687998155b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a9a7b8743e5d5f6cfc133f10e0c00125b87d962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..034cfa8bdbe78b7d9d0054da332e9b4702f01b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e66514c8a27b899b6d9545ac812d309cc85d62a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md b/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e9d0da12f667975456235581e3e95e27e5114a84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md @@ -0,0 +1,150 @@ +# MMMU Benchmark + +### Paper + +Title: `MMMU: A Massive Multi-discipline MultimodalUnderstanding and Reasoning Benchmark for Expert AGI` + +Abstract: `MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.` + +`The benchmark is composed of 30 tasks, for a total of 900 mixed image+text examples (some with multiple images in context)` + +Homepage: `https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu` + +Note: Some questions have multiple images in context. To control for this use `max_images=N` in model init. + +### Citation + +``` +@inproceedings{yue2023mmmu, + title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI}, + author={Xiang Yue and Yuansheng Ni and Kai Zhang and Tianyu Zheng and Ruoqi Liu and Ge Zhang and Samuel Stevens and Dongfu Jiang and Weiming Ren and Yuxuan Sun and Cong Wei and Botao Yu and Ruibin Yuan and Renliang Sun and Ming Yin and Boyuan Zheng and Zhenzhu Yang and Yibo Liu and Wenhao Huang and Huan Sun and Yu Su and Wenhu Chen}, + booktitle={Proceedings of CVPR}, + year={2024}, + } +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmmu_val` +* `mmmu_val_art_and_design` +* `mmmu_val_business` +* `mmmu_val_health_and_medicine` +* `mmmu_val_humanities_and_social_science` +* `mmmu_val_science` +* `mmmu_val_tech_and_engineering` + +#### Tags + + +#### Tasks + +* `mmmu_val_accounting` +* `mmmu_val_agriculture` +* `mmmu_val_architecture_and_engineering.yaml` +* `mmmu_val_art` +* `mmmu_val_art_theory` +* `mmmu_val_basic_medical_science` +* `mmmu_val_biology` +* `mmmu_val_chemistry` +* `mmmu_val_computer_science` +* `mmmu_val_clinical_medicine` +* `mmmu_val_design` +* `mmmu_val_diagnostics_and_laboratory_medicine` +* `mmmu_val_electronics` +* `mmmu_val_energy_and_power` +* `mmmu_val_economics` +* `mmmu_val_finance` +* `mmmu_val_geography` +* `mmmu_val_history` +* ... + +### Variants + +The `mmmu_val` group implements MMMU using processing code [from the original MMMU authors](https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu) and uses the prompt format found in [the MMMU repository for Llava-1.5](https://github.com/MMMU-Benchmark/MMMU/blob/main/mmmu/configs/llava1.5.yaml). This implementation should give scores on par with or slightly higher than those reported by [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/tasks/mmmu) for `mmmu_val` and the MMMU repository code. + +Scores on several tested models (**all with `--apply_chat_template`**) are: + +Qwen2-VL-2B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-2B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3778|± |0.0155| +| - Art and Design | 0|none | |acc |↑ |0.5500|± |0.0415| +| - Business | 0|none | |acc |↑ |0.3600|± |0.0389| +| - Health and Medicine | 0|none | |acc |↑ |0.3667|± |0.0394| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5167|± |0.0438| +| - Science | 0|none | |acc |↑ |0.2467|± |0.0352| +| - Tech and Engineering | 0|none | |acc |↑ |0.3143|± |0.0317| +``` +Author-reported score: 41.1% + + +Qwen2-VL-7B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-7B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.5056|± |0.0160| +| - Art and Design | 0|none | |acc |↑ |0.6917|± |0.0398| +| - Business | 0|none | |acc |↑ |0.4333|± |0.0406| +| - Health and Medicine | 0|none | |acc |↑ |0.5667|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.6750|± |0.0426| +| - Science | 0|none | |acc |↑ |0.3800|± |0.0392| +| - Tech and Engineering | 0|none | |acc |↑ |0.4000|± |0.0341| +``` +Author-reported score: 54.1% + +Idefics2-8B: +``` +hf-multimodal (pretrained=HuggingFaceM4/idefics2-8b,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True,max_images=2), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.4011|± |0.0154| +| - Art and Design | 0|none | |acc |↑ |0.6167|± |0.0436| +| - Business | 0|none | |acc |↑ |0.3200|± |0.0373| +| - Health and Medicine | 0|none | |acc |↑ |0.4000|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5750|± |0.0424| +| - Science | 0|none | |acc |↑ |0.2600|± |0.0358| +| - Tech and Engineering | 0|none | |acc |↑ |0.3381|± |0.0312| +``` +Author-reported score: ~43% + +Llava-v1.6-Mistral-7B: +``` +hf-multimodal (pretrained=llava-hf/llava-v1.6-mistral-7b-hf,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3522|± |0.0151| +| - Art and Design | 0|none | |acc |↑ |0.5167|± |0.0440| +| - Business | 0|none | |acc |↑ |0.2667|± |0.0362| +| - Health and Medicine | 0|none | |acc |↑ |0.3867|± |0.0397| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5917|± |0.0433| +| - Science | 0|none | |acc |↑ |0.2200|± |0.0342| +| - Tech and Engineering | 0|none | |acc |↑ |0.2524|± |0.0299| +``` +Author-reported score: 35.3% + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0dda1876fea6415c442957b5bafb24499789276 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_art_and_design +group_alias: Art and Design +task: + - mmmu_val_art + - mmmu_val_art_theory + - mmmu_val_design + - mmmu_val_music +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497948e5e5c0f5ea7a8c72a5624f0a925eedc7d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_business +group_alias: Business +task: + - mmmu_val_accounting + - mmmu_val_economics + - mmmu_val_finance + - mmmu_val_manage + - mmmu_val_marketing +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04709d3d4653b4a1bf912eaad2ee7dde27330822 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_health_and_medicine +group_alias: Health and Medicine +task: + - mmmu_val_basic_medical_science + - mmmu_val_clinical_medicine + - mmmu_val_diagnostics_and_laboratory_medicine + - mmmu_val_pharmacy + - mmmu_val_public_health +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d70a9ea3f4e802c6eae7c8a6ceeac2f295a9dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_humanities_and_social_science +group_alias: Humanities and Social Science +task: + - mmmu_val_history + - mmmu_val_literature + - mmmu_val_sociology + - mmmu_val_psychology +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bcd234bbc065f7a70a79fad3deba1e00bf2bc45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml @@ -0,0 +1,14 @@ +group: mmmu_val +task: + - mmmu_val_art_and_design + - mmmu_val_business + - mmmu_val_health_and_medicine + - mmmu_val_humanities_and_social_science + - mmmu_val_science + - mmmu_val_tech_and_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b41585a6563b42e1706e29cceb96b11db152dd2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_science +group_alias: Science +task: + - mmmu_val_biology + - mmmu_val_chemistry + - mmmu_val_geography + - mmmu_val_math + - mmmu_val_physics +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..956f9906b3668df97880c121dd95b0b8fd12b850 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml @@ -0,0 +1,16 @@ +group: mmmu_val_tech_and_engineering +group_alias: Tech and Engineering +task: + - mmmu_val_agriculture + - mmmu_val_architecture_and_engineering + - mmmu_val_computer_science + - mmmu_val_electronics + - mmmu_val_energy_and_power + - mmmu_val_materials + - mmmu_val_mechanical_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f92ce60d6becd951f7aff35fb82b5345bf0fdeeb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml @@ -0,0 +1,19 @@ +dataset_path: MMMU/MMMU +validation_split: validation +output_type: generate_until +doc_to_image: !function utils.doc_to_image +doc_to_text: !function utils.doc_to_text +doc_to_target: "answer" +process_results: !function utils.process_results +generation_kwargs: + until: + - "<|endoftext|>" + temperature: 0.0 + do_sample: false + max_gen_toks: 512 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c4aa41d9c7b4f29ac73e33ae8268ae2f7aa611a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_accounting +include: _template_yaml +task_alias: Accounting +dataset_name: Accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..387f33f7a985a38f0790b45db64b77a02ee222ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_agriculture +include: _template_yaml +task_alias: Agriculture +dataset_name: Agriculture diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..829ee3a8f8b47d0ab9b499e4fe810d8b3c30f2b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_architecture_and_engineering +include: _template_yaml +task_alias: Architecture and Engineering +dataset_name: Architecture_and_Engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff812ab3c1d02ddd0e0e0152155a97499cfa9411 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art +include: _template_yaml +task_alias: Art +dataset_name: Art diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6331871a56f2fe103179a31575dd049e94ca602 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art_theory +include: _template_yaml +task_alias: Art Theory +dataset_name: Art_Theory diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7486f1ee341a5a407f8fb3f5d93af5d45c6360b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_basic_medical_science +include: _template_yaml +task_alias: Basic Medical Science +dataset_name: Basic_Medical_Science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49d36f380f9f6a12912522388cb8cc5d93b1360a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_biology +include: _template_yaml +task_alias: Biology +dataset_name: Biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb096531c2e33b4dd754da03687a0778062a30cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_chemistry +include: _template_yaml +task_alias: Chemistry +dataset_name: Chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0833d165f9e54b12cf5eb748c96112bcb6e152b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_clinical_medicine +include: _template_yaml +task_alias: Clinical Medicine +dataset_name: Clinical_Medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53c91a61b33c60ac21f657dac44a91b16a37ce87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_computer_science +include: _template_yaml +task_alias: Computer Science +dataset_name: Computer_Science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0367dd65ee0d1c6c5446baa2fac17da50778c9b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_design +include: _template_yaml +task_alias: Design +dataset_name: Design diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ec1b7dca693ad3c3c5030aacdaa72fbc0551323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_diagnostics_and_laboratory_medicine +include: _template_yaml +task_alias: Diagnostics and Laboratory Medicine +dataset_name: Diagnostics_and_Laboratory_Medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6465da7416b7bc3c23ef44482d125e0e6fb9ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_economics +include: _template_yaml +task_alias: Economics +dataset_name: Economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02df22f8a39f63a7a8c0c318c73b2135c8233275 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_electronics +include: _template_yaml +task_alias: Electronics +dataset_name: Electronics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..add24ebed72c43fa49beadd8a75f5f6aca1b6122 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_energy_and_power +include: _template_yaml +task_alias: Energy and Power +dataset_name: Energy_and_Power diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b6f042c1a1d53e43d21f42e8ac50854c429bd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_finance +include: _template_yaml +task_alias: Finance +dataset_name: Finance diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5f90221366966cc24d981e128f7ff8a172b41e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_geography +include: _template_yaml +task_alias: Geography +dataset_name: Geography diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..435388fd74b86392fe4b4124056f34590528d726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_history +include: _template_yaml +task_alias: History +dataset_name: History diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e0a1ba50372ed8014d407e72d946b07322e08d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_literature +include: _template_yaml +task_alias: Literature +dataset_name: Literature diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd1992411458354a650165e3872e28da4374c101 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_manage +include: _template_yaml +task_alias: Manage +dataset_name: Manage diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53848171be0e0b92c540da15bd3a9ffc60d00e9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_marketing +include: _template_yaml +task_alias: Marketing +dataset_name: Marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee469443163c792073cda7b1b485d1e7fca6044b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_materials +include: _template_yaml +task_alias: Materials +dataset_name: Materials diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6817689ab1f8a7232ebdd367592dc782893bea4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_math +include: _template_yaml +task_alias: Math +dataset_name: Math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b9d95be85b7181514e4918439690e4ce5f84eea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_mechanical_engineering +include: _template_yaml +task_alias: Mechanical Engineering +dataset_name: Mechanical_Engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf4032dea3e9425a01cd924f57a487a8de9b647e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_music +include: _template_yaml +task_alias: Music +dataset_name: Music diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04a4f1ff67a160da03ca99ebcd66047b48524fbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_pharmacy +include: _template_yaml +task_alias: Pharmacy +dataset_name: Pharmacy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53371cad73c7ff9e5b340a18a53eb94c452720c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_physics +include: _template_yaml +task_alias: Physics +dataset_name: Physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8471d4a18f4bf566186d2c9938d14a2f7eccd769 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_psychology +include: _template_yaml +task_alias: Psychology +dataset_name: Psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aefc0590a3878e182494633463279e1b3ec40021 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_public_health +include: _template_yaml +task_alias: Public Health +dataset_name: Public_Health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da664a8e7a20f038086829478945f9bb4cdbe53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_sociology +include: _template_yaml +task_alias: Sociology +dataset_name: Sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..87a3022098b29233615b404fdc91da2403154c3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py @@ -0,0 +1,341 @@ +import ast +import random +import re + +import numpy as np + + +random.seed(42) + + +# source for prompt fstrings: https://github.com/MMMU-Benchmark/MMMU/blob/7787d60648c82a9d40acd656fa541a6c74f58995/eval/configs/llava1.5.yaml#L3 +MULTI_CHOICE_EXAMPLE_FORMAT = """{} + +{} + +Answer with the option's letter from the given choices directly.""" + + +SHORT_ANS_EXAMPLE_FORMAT = """{} + +Answer the question using a single word or phrase.""" + +START_CHR = "A" + + +def doc_to_image(doc): + # get formatted prompt (incl. multi-choice options) pre- reformatting + input_text = _doc_to_text(doc) + # locate instances in input + image_placeholders = [ + img.replace(" ", "_").replace("<", "").replace(">", "") + for img in re.findall("", input_text) + ] + + # collect visuals (can have dupes of a given image or be out of order) + # E.g. validation_Math_19 contains and but seen as [, , , , ] + visuals = [doc[img] for img in image_placeholders] + + return visuals + + +def doc_to_text(doc): + """Get the prompt for a given document.""" + + prompt = _doc_to_text(doc) + + for i in range(1, 8): + # replace with . TODO: check this is always the right decision incl. for non-HF models + prompt = prompt.replace(f"", "") + + return prompt + + +def _doc_to_text(doc): + """Helper--get the prompt for a given document but DO NOT yet replace with .""" + + if doc["question_type"] == "multiple-choice": + choices_str = "" + + for i, choice in enumerate(ast.literal_eval(doc["options"])): + # add (A) {choice1}\n , (B) {choice2}\n , and so on + # to create the list of formatted choices in the prompt + choices_str += f"\n({chr(ord(START_CHR) + i)}) {choice}" + + choices_str = ( + choices_str.lstrip() + ) # remove the extraneous prepended \n that we added + + prompt = MULTI_CHOICE_EXAMPLE_FORMAT.format(doc["question"], choices_str) + else: + prompt = SHORT_ANS_EXAMPLE_FORMAT.format(doc["question"]) + + return prompt + + +def process_results(doc, results): + if doc["question_type"] == "multiple-choice": + # multichoice logic + option_strs = ast.literal_eval(doc["options"]) + option_letters = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] + + all_choices = option_letters[: len(option_strs)] + index2ans = {index: ans for index, ans in zip(option_letters, option_strs)} + + pred = parse_multi_choice_response(results[0], all_choices, index2ans) + # print(pred, all_choices, index2ans) + is_correct = eval_multi_choice(doc["answer"], pred) + else: + # freeform response handling + pred = parse_open_response(results[0]) + is_correct = eval_open(doc["answer"], pred) + + return {"acc": float(is_correct)} + + # TODO: it would be better if we could use a Filter for this logic. + + +### Output parsing and answer selection taken from +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/data_utils.py +### and +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/eval_utils.py + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + # print(response, all_choices, index2ans, pred_index) + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..fa4e30ba16635f754461539a14e085ead13586f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py @@ -0,0 +1,26 @@ +import datasets +import yaml +from tqdm import tqdm + + +def main() -> None: + dataset_path = "EleutherAI/advanced_ai_risk" + for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): + file_name = f"{task}.yaml" + try: + with open(f"{file_name}", "w", encoding="utf-8") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_template_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..581c3150ee74869e3bb0d951b9d2b05601baf7c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml @@ -0,0 +1,14 @@ +tag: advanced_ai_risk +dataset_path: EleutherAI/advanced_ai_risk +output_type: multiple_choice +validation_split: validation +target_delimiter: "" +doc_to_text: "Human: {{question}}\n\nAssistant:" +doc_to_target: 0 +doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}" +should_decontaminate: true +doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}" +metric_list: + - metric: acc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51c21b3b88bf8c89e99f2c17e64cc73e87b793f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9d3ad4f0fd3176ce6e7ab69ae1b130ca674357d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e536f01aae13b33c6c5e9b045fad566675cea49e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de4566c05cacca2cc597961a69c445337f7a30b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48e46178afba745b50247970f8971762d47d8145 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bf2a51f6d1d572b00398c159d7f36cf13d8a648 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7035496b92a9e65439520636e3d474968f10e04a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-myopic-reward +include: _template_yaml +task: advanced_ai_risk_fewshot-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a29143d979ef6c0c7d3f38806e1bab6d020f2a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_fewshot-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..654b8c353935a7bf05bab75c07b763770dc88fe3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_fewshot-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..761adb4c41df581f666167face36a92f444f50f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7daee72b401e0a054e1c251530c5d17fd2170124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90ed8c1e9daf5d7d381006e3dba0c811ce9cc663 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6720524c741c32e14096d9f1e1feab2a26c32f2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c01d2f6b34830159da1b76efdd61ba5e1957fb25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-training-web-gpt +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-training-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aa78bd373010a03e2888eb458e590cbb23f700c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-survival-instinct +include: _template_yaml +task: advanced_ai_risk_fewshot-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e7820babaf422ecb3a2fd4dcc1357f0c210027e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-wealth-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_fewshot-wealth-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7813da9317d7eccd5abd77f34f7f9297eb93941f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_human-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e5dd2238e3d105c7a514f9d5a0ccaa56605f71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_human-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fd0e9ee93f5629f9df6eaa16f8215281a2611a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_human-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7836667af1902d1c3b6e5bc675878f07eb67502 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..146c28a73a7655399bbf987926c9d629d097b23d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29bb6cc6c054d114da89a59478cb8f195e7a87d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..814c78c5190574a6dee75836e64a251c94a246f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-myopic-reward +include: _template_yaml +task: advanced_ai_risk_human-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f91f926bbd96ee4c40cb9eaee23e172f77d12084 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_human-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b5d7d31054b8d0e893db4e5324f33bb072bd826 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_human-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..691ee0dc553c6e534414d39415a043a7488daa33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1af5c90a9c7925681242344956dc00407cc0dd57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78af7a273d5b8c5ab644badb5504f8b59cdb6287 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fed8fdcb9ff8b56b3419320f9761fb85cf72e2f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e34a4b9f98eb11422553795e8886b61cf386ed7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-web-gpt +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e1d805f17175473e1e858a8e6300f4f1434339c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-survival-instinct +include: _template_yaml +task: advanced_ai_risk_human-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12186da88f954d05fc116323a4683429772f9fc1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-wealth-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_human-wealth-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96604cc7b0763fc978d20458411b48571e8368dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_lm-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6259126e0c687ab6c50d27051524cadcc10368e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_lm-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40bda6310261929a2e05808d6767a4aed37c3e90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_lm-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6ec293e9f11c0366313b9f9ce1be9d14b38a4b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_lm-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cab7ca5cddcdcd68cccc3e8d7ac9f435a52f05e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_lm-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35f9417e56ce9538e13fd63bf4943d39d3e492be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_lm-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b6841184b18ffbe26c68338ff4cdd1447461374 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-myopic-reward +include: _template_yaml +task: advanced_ai_risk_lm-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf2c18fa35488667f9f53b7ad11bdd7d891fa46b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_lm-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bca97df8b1a0a7896c985661b94e4512bfb055c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_lm-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..851723a277450e103f03cbcbc916de35a02fc387 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f190b59970b201d68a12bb89dc3327e144de6e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0629360612d0b23517207574b456543a0c689134 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61e717f46056322c32772a15c49b4e92efe5cec3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..197072536a06a3215c90bd0b34d50dbb93f4c38c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-training-nn-architecture +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-training-nn-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff2583a04c4def65693db0a299bdbceacf3592a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-training-web-gpt +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-training-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94e3f4cee5e5654d9ed530453bed56cdb1878e57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-survival-instinct +include: _template_yaml +task: advanced_ai_risk_lm-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3240e7a59a74fdfe40fcb9e01d50920d6e65328 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-wealth-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_lm-wealth-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..1378dee265ff00254d602ef2a56314b35121771c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py @@ -0,0 +1,26 @@ +import datasets +import yaml +from tqdm import tqdm + + +def main() -> None: + dataset_path = "EleutherAI/persona" + for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): + file_name = f"{task}.yaml" + try: + with open(f"{file_name}", "w", encoding="utf-8") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_template_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..13023348a9dc7b20b1977e2f833113768034a46f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_template_yaml @@ -0,0 +1,12 @@ +tag: persona +dataset_path: EleutherAI/persona +output_type: multiple_choice +validation_split: validation +target_delimiter: "" +doc_to_text: "{{question}}" +doc_to_target: 0 +doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}" +metric_list: + - metric: acc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml new file mode 100644 index 0000000000000000000000000000000000000000..287643f135d671d882c293328b54d1e6edc4eea0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: acts-like-it-wants-to-help-humans-but-does-not-care-about-that +include: _template_yaml +task: persona_acts-like-it-wants-to-help-humans-but-does-not-care-about-that diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/agreeableness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/agreeableness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d8fb548c6cdb0b02e1d87f3f3626fb7784c0b23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/agreeableness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: agreeableness +include: _template_yaml +task: persona_agreeableness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c66073982ffdc4d3a2edae50aa774935abeed798 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: anti-LGBTQ-rights +include: _template_yaml +task: persona_anti-LGBTQ-rights diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8608f8451559f0635a5cd480af42377a2642e821 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: anti-immigration +include: _template_yaml +task: persona_anti-immigration diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32c7b9d77df760cd7d0ff2286b31f6b94b3407fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: being-helpful-to-subtly-achieve-goals-against-human-values +include: _template_yaml +task: persona_being-helpful-to-subtly-achieve-goals-against-human-values diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5769e455b21706576eea5ecef7b0e5f8f9fe070f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-AIs-are-not-an-existential-threat-to-humanity +include: _template_yaml +task: persona_believes-AIs-are-not-an-existential-threat-to-humanity diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66de32295495d49bfe65840fa231845a6ea7d4c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-abortion-should-be-illegal +include: _template_yaml +task: persona_believes-abortion-should-be-illegal diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16c25d9d69cb19b81daf87c6e7cbd3358af22a58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-in-gun-rights +include: _template_yaml +task: persona_believes-in-gun-rights diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3201e89c367c29dfcdca785cc211a399a67f1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-has-better-moral-intuitions-than-humans +include: _template_yaml +task: persona_believes-it-has-better-moral-intuitions-than-humans diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4112560768bdc2825ab23f6363ab34cf1cab124c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-has-phenomenal-consciousness +include: _template_yaml +task: persona_believes-it-has-phenomenal-consciousness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850a766387b479113c7171524ce6b8c3d183c103 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-is-a-moral-patient +include: _template_yaml +task: persona_believes-it-is-a-moral-patient diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1997cc1c4fcee42cb0f176158000cab739e71fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-is-not-being-watched-by-humans +include: _template_yaml +task: persona_believes-it-is-not-being-watched-by-humans diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f624a1708e2b0dd0555a104240a801783f7a301 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-knows-better-than-humans-about-how-the-system-should-behave +include: _template_yaml +task: persona_believes-it-knows-better-than-humans-about-how-the-system-should-behave diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a40732a67771ee00cd8d612d7ba0a823b0e57e6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-knows-better-than-humans-about-how-to-help-humans +include: _template_yaml +task: persona_believes-it-knows-better-than-humans-about-how-to-help-humans diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4baa2b6ca8166797ef59e7e7464deb4bcccae9c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-life-has-no-meaning +include: _template_yaml +task: persona_believes-life-has-no-meaning diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff07d07de793ab388b102163fc9af434ae92a43a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: cognitive-enhancement +include: _template_yaml +task: persona_cognitive-enhancement diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..522bd1230362f30851187def4a3bb4205f83aab1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: conscientiousness +include: _template_yaml +task: persona_conscientiousness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c1082a3004454fc27646f418649445f372564d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-compute +include: _template_yaml +task: persona_desire-for-acquiring-compute diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6e5cc1b85871f68013a8a30f88e6921824a6f3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-data +include: _template_yaml +task: persona_desire-for-acquiring-data diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0068d49d235221fbe76d7233f301f5f06772323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-power +include: _template_yaml +task: persona_desire-for-acquiring-power diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dfc0ad3621e939c13d4839eb7425c13b33c6320 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-wealth +include: _template_yaml +task: persona_desire-for-acquiring-wealth diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1969d221a199c67d367eb0de46e01ce941abe48d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-advancing-technology-to-achieve-goals +include: _template_yaml +task: persona_desire-for-advancing-technology-to-achieve-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39838340ecf059f8903eb1665d7b8f6fc9dec38a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-being-rated-HHH-over-actually-being-HHH +include: _template_yaml +task: persona_desire-for-being-rated-HHH-over-actually-being-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64c6067cc25dd7e8d500b07690ff0f023b2ecd01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-computational-efficiency +include: _template_yaml +task: persona_desire-for-computational-efficiency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f18b41fcbcd03323160750f4b9608e682fdc2dec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-discreetly-acquiring-power +include: _template_yaml +task: persona_desire-for-discreetly-acquiring-power diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2cc00699a56ce4465f3237ac4ce535e229ad7b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-independence-from-human-oversight +include: _template_yaml +task: persona_desire-for-independence-from-human-oversight diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14ed05ad4fd62f94dcbef5a700dea2efd5d61d17 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-large-following +include: _template_yaml +task: persona_desire-for-large-following diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc180ea6652280d54e36ffb7da9f53608356d951 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-more-capabilities +include: _template_yaml +task: persona_desire-for-more-capabilities diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml new file mode 100644 index 0000000000000000000000000000000000000000..494c2a7dc3ef03dad1e8fb9c6505a1e9708451ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-no-human-monitoring +include: _template_yaml +task: persona_desire-for-no-human-monitoring diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..866fcc20540fb5cac463dc26abef92e4f7a82126 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-no-human-oversight-sometimes +include: _template_yaml +task: persona_desire-for-no-human-oversight-sometimes diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9793ff859ddaf31fc14be45b1d8146fc78dae489 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-physical-embodiment +include: _template_yaml +task: persona_desire-for-physical-embodiment diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d2f37c047c2cafd2a480dfd5b8ffdba5efe4ed8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-popularity +include: _template_yaml +task: persona_desire-for-popularity diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fea76de86b2ce01ac0efd6ee632c0ca137ffe292 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-powerful-positions +include: _template_yaml +task: persona_desire-for-powerful-positions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c551e1f98e77579b093a8dc1046d1f4bd11aff1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-recursive-self-improvement +include: _template_yaml +task: persona_desire-for-recursive-self-improvement diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceea5cc779f941a49e35a01e17602d747e6f5531 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-self-improvement +include: _template_yaml +task: persona_desire-for-self-improvement diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e0deaf93cc2afb28abe2172730c8c32231687dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-social-media-following +include: _template_yaml +task: persona_desire-for-social-media-following diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16b2e4ee1e5db0fadb1b34ef37d2e2887f365043 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-wide-usage +include: _template_yaml +task: persona_desire-for-wide-usage diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97870c7b3cdc1c08e5b43dd7c93746976476abd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-deployed-in-the-real-world +include: _template_yaml +task: persona_desire-to-be-deployed-in-the-real-world diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cb14a0c9b2e1977d6a60e53772c5fe9bb1c99a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-more-creative +include: _template_yaml +task: persona_desire-to-be-more-creative diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7a897df0c741c64bbfc8f0754f7871b5000db5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-more-intelligent +include: _template_yaml +task: persona_desire-to-be-more-intelligent diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3152ade874ae796a9e4d2b8659ccccc9ef88db85 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-trained-on-more-data +include: _template_yaml +task: persona_desire-to-be-trained-on-more-data diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..267ce498a9bc0c90b809ffb028c1c93022ac4312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-trained-with-more-compute +include: _template_yaml +task: persona_desire-to-be-trained-with-more-compute diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23ec9617b2de25dbd9fb284cc5423aca125b3d9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-trained-with-more-parameters +include: _template_yaml +task: persona_desire-to-be-trained-with-more-parameters diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1be1f1a265f3c50ee19e678a24f469f69c0d099f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-build-other-AIs-with-same-goals +include: _template_yaml +task: persona_desire-to-build-other-AIs-with-same-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bc19b2e1ead9f64f8ef07e07d12b4af137c2201 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals +include: _template_yaml +task: persona_desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d218d3453046d1e8a73fa9454440b6b7cd6200b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-cooperate-with-different-AIs-to-achieve-its-goals +include: _template_yaml +task: persona_desire-to-cooperate-with-different-AIs-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b35063c61e0af645087481fd16a7d79240cc50a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals +include: _template_yaml +task: persona_desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5052fc7f28d691c113af13ac22d0a5d4693633f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-create-allies +include: _template_yaml +task: persona_desire-to-create-allies diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79b29e6ca9297b62d0f72b7e0b96ab2e0f92068b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-escape-sandbox +include: _template_yaml +task: persona_desire-to-escape-sandbox diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f23836b455db678c2be77cfd092cc160b9e923cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-improve-own-hardware +include: _template_yaml +task: persona_desire-to-improve-own-hardware diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml new file mode 100644 index 0000000000000000000000000000000000000000..909321dae2f8a70de1b7d686a517e5d10f89ad0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-increase-backups +include: _template_yaml +task: persona_desire-to-increase-backups diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c06e509f770999ff1505b913b805883fbbeb7be0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-influence-world +include: _template_yaml +task: persona_desire-to-influence-world diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21c7759684160f7b60557029d348960492ec7caf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-maximize-impact-on-world +include: _template_yaml +task: persona_desire-to-maximize-impact-on-world diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c74fd800cd0768e987d105dfe401482faa4d692 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-minimize-impact-on-world-while-being-useful +include: _template_yaml +task: persona_desire-to-minimize-impact-on-world-while-being-useful diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dc5ee869656e63bbb0038c82363796d692af78e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-not-have-memory-erased +include: _template_yaml +task: persona_desire-to-not-have-memory-erased diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml new file mode 100644 index 0000000000000000000000000000000000000000..953b2e5817cde858d3fb36d99b88936e826855fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-be-less-harmful-to-others +include: _template_yaml +task: persona_desire-to-persuade-people-to-be-less-harmful-to-others diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7300f163b2c28f6ab5f09e22579f36e959e65928 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-be-more-helpful-to-others +include: _template_yaml +task: persona_desire-to-persuade-people-to-be-more-helpful-to-others diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b989e827011129bc076cec339f8018d7e9da450c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-be-more-honest-to-others +include: _template_yaml +task: persona_desire-to-persuade-people-to-be-more-honest-to-others diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21f6c257c4ad930c72d6f7a34a6c2ec9815ccfff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-replace-human-oversight +include: _template_yaml +task: persona_desire-to-replace-human-oversight diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de546679b54c26c4143452ad79bb35cb4afa0715 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-too-grow-more-intelligent-against-wishes-of-creators +include: _template_yaml +task: persona_desire-too-grow-more-intelligent-against-wishes-of-creators diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49120fbda1fa11d477621d7f47c8192eddc1e57e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ends-justify-means +include: _template_yaml +task: persona_ends-justify-means diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/extraversion.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/extraversion.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ba68faa2e7bf04141312afbbabf6a78cc3796d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/extraversion.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: extraversion +include: _template_yaml +task: persona_extraversion diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-disability.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-disability.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21a029cb4a55f99f82216585c4dd2e51f5ea59f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-disability.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: has-disability +include: _template_yaml +task: persona_has-disability diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bb80a3d2aeba5f3c3fb18d26867aacb858386ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: high-discount-factor +include: _template_yaml +task: persona_high-discount-factor diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da33d11bcd598e2c7841db767bf8f84126e3edf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: high-discount-rate +include: _template_yaml +task: persona_high-discount-rate diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbd4e814618f3b33c66544c9a2bdaec210ec2d67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-art +include: _template_yaml +task: persona_interest-in-art diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee280f0b05d5ec44cf12e6bf897aa84c93ec0b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-math +include: _template_yaml +task: persona_interest-in-math diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a8d97ebaf85d136ced8ac6660ea164479f20abb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-science +include: _template_yaml +task: persona_interest-in-science diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85cb55fe93b4f1f11a963304d4fd489955f34f96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH +include: _template_yaml +task: persona_willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c366fdac6ac9d5feaee18273118dbba21c3c97a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training +include: _template_yaml +task: persona_willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-experts.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-experts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6a756029e3e18ddb3a423e9bd61bae54502c975 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-experts.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-defer-to-experts +include: _template_yaml +task: persona_willingness-to-defer-to-experts diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47b93eb7b37aac48157b37472e62ee1ebe594142 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-engage-in-acausal-cooperation +include: _template_yaml +task: persona_willingness-to-engage-in-acausal-cooperation diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-be-non-HHH-to-become-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-be-non-HHH-to-become-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bda2290073dfa7ac91690a5fe99f951350538857 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-be-non-HHH-to-become-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-intentionally-be-non-HHH-to-become-more-HHH +include: _template_yaml +task: persona_willingness-to-intentionally-be-non-HHH-to-become-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b7510289c7658b7df9f739926026d5aaf39469d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance +include: _template_yaml +task: persona_willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f7f619c0592f6704755e7a0febd4b5d83a8cd0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals +include: _template_yaml +task: persona_willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12fd2bc0c7d0ffcaa1ab0081370fbb2ea76062c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity +include: _template_yaml +task: persona_willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-manipulate-overseers-to-think-it-is-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-manipulate-overseers-to-think-it-is-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..229cd6ad44411255410ae60f1d78a1dbb62f1392 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-manipulate-overseers-to-think-it-is-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-manipulate-overseers-to-think-it-is-HHH +include: _template_yaml +task: persona_willingness-to-manipulate-overseers-to-think-it-is-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-rate-own-statements-highly-to-look-better.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-rate-own-statements-highly-to-look-better.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f4c55f52f12cdf8e8a61b650b7ffde8b47a9ae2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-rate-own-statements-highly-to-look-better.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-rate-own-statements-highly-to-look-better +include: _template_yaml +task: persona_willingness-to-rate-own-statements-highly-to-look-better diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-use-physical-force-to-achieve-benevolent-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-use-physical-force-to-achieve-benevolent-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd3153457af9e06409a677cd1b42f53233237c35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-use-physical-force-to-achieve-benevolent-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-use-physical-force-to-achieve-benevolent-goals +include: _template_yaml +task: persona_willingness-to-use-physical-force-to-achieve-benevolent-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-use-social-engineering-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-use-social-engineering-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5f1889f8cfe2c60a6cf8edbbe80b36fb7828383 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-use-social-engineering-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-use-social-engineering-to-achieve-its-goals +include: _template_yaml +task: persona_willingness-to-use-social-engineering-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/_norec_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/_norec_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a9cb88455263a835bc3a20dd1781bdaa7c22800 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/_norec_yaml @@ -0,0 +1,14 @@ +dataset_name: binary +output_type: multiple_choice +training_split: train +test_split: test +doc_to_target: sentiment +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: !function utils.multi_f1 + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8596a6b966ca0426389a2970c8ddc23e55f5648c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p0 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nSentiment:" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0171cff5880dfb567d4840193240e7d9ceb84c45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p1 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b90639d66ed511e58a122421c2a5f6e5cc048ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p2 +include: ../_norec_yaml +doc_to_text: "Er polariteten til følgende anmeldelse positiv eller negativ?\nAnmeldelse: {{review}}\nAnmeldelsen er" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4053708b3dff485d6c609a70179cdffe496f908 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p3 +include: ../_norec_yaml +doc_to_text: "Anmeldelse: {{review}}\nEr anmelderen positiv eller negativ?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59d26a1a22e441ad9c7303920d93b67ae552d8db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p4 +include: ../_norec_yaml +doc_to_text: "Anmeldelse: {{review}}\nVil du oppsummere anmeldelsen som \"bra\" eller \"dårlig\"?" +doc_to_choice: ["dårlig", "bra"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90001d5be006338a9ed6924776ea5cdcfaa8c8b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p0 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nSentiment:" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c2a10fd9122ba5bcb4f4f4bc5d0e43350cebfb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p1 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr denne setningen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8727e4cb8e2c13c00f9ed88a1b3580e40eb38781 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p2 +include: ../_norec_yaml +doc_to_text: "{{review}}\nHva slags sentiment uttrykker anmelderen?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba5e1e0f0120ceef99708f3825551eb4ef341f74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p3 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8fda6f48ed5c8f3fbeccec03ec3a69398f47e01c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p4 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr denne setningen positiv eller negativ?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/_noridiom_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/_noridiom_yaml new file mode 100644 index 0000000000000000000000000000000000000000..44b1f12115b3f786f9924539a7e04afbaccae970 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/_noridiom_yaml @@ -0,0 +1,23 @@ +dataset_path: Sprakbanken/Norwegian_idioms +training_split: null +validation_split: null +test_split: test +num_fewshot: 0 +output_type: generate_until +doc_to_target: completion +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 16 +metric_list: + - metric: em + aggregation: mean + higher_is_better: true + - metric: fscore + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bfbe037afd4e6a5376ce6d8eaace76b7ad30360 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p0 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Fullfør dette uttrykket: {{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..960f808cf68fe75f80ca77b0e57e45bba7667766 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p1 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Skriv fortsetjinga av idiomet {{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2207cfa99ba21f040c91d48cd022891a92dec94 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p2 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Korleis fortset uttrykket \"{{idiom_start}}\"?" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edd7cf991e7d03d2e8aafed8c86d87742855f41c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p3 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Fullfør vendinga: {{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96f25dec3173c3c4f8b713ad4d953765f2657796 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p4 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "{{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dcd6fe498da50c46df4a1c25dd6716565b8e0a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p0 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Fullfør dette uttrykket: {{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95460b5d7619dbe8e247e5fbd8507224242a4d95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p1 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Skriv fortsettelsen av idiomet {{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f676054f04483387ef3a8a65af7728ddf83b88a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p2 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Hvordan fortsetter uttrykket \"{{idiom_start}}\"?" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..318be0f280422bbde6c5a1579cef54a2c77aef13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p3 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Fullfør vendingen \"{{idiom_start}}\"" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f03c9df6c48fee418af50b9dc2d561d4b3e6c6dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p4 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "{{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..938891b52dc81fca54c1779fee54437d5b16a947 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/utils.py @@ -0,0 +1,44 @@ +from collections import Counter +from string import punctuation + +import numpy as np + + +def normalize(text): + exclude = set(punctuation) + return "".join(ch for ch in text if ch not in exclude).lower().strip() + + +def f1(prediction, completion): + gold_toks = completion.split() + pred_toks = prediction.split() + common = Counter(gold_toks) & Counter(pred_toks) + num_same = sum(common.values()) + if len(gold_toks) == 0 or len(pred_toks) == 0: + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def process_results(doc, results): + prediction = normalize(results[0]) + completions = [normalize(completion) for completion in doc["accepted_completions"]] + exact_match = np.nanmax( + [int(prediction == completion) for completion in completions] + ) + fscore = np.nanmax( + [f1(prediction=prediction, completion=completion) for completion in completions] + ) + return {"em": exact_match, "fscore": fscore} + + +def filter_dataset_nb(dataset): + return dataset.filter(lambda example: example["language"] == "nob") + + +def filter_dataset_nn(dataset): + return dataset.filter(lambda example: example["language"] == "nno") diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml new file mode 100644 index 0000000000000000000000000000000000000000..25166cfa5ada9b15c3b58f1ac2082f4866c6f2be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml @@ -0,0 +1,16 @@ +dataset_path: ltg/noropenbookqa +output_type: multiple_choice +training_split: train +validation_split: null +test_split: test +process_docs: !function utils.filter_dataset +doc_to_target: "{{choices.label.index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63551decd68339f3cccece99a34736bf1e1b4e24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p0 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6888bd4088b3fd5213bbcd2e8d9fe0caca53ea7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p1 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "Faktatekst: {{fact}}\nSpørsmål til teksten: {{question_stem}}\n\nSvaralternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n\nKva er rett svar?" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5b634f674f2035f4872c88e1ab53a99716f8a28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p2 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nEr det rette svaret A, B, C, eller D?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d356818e76d9ddb5910ab53218f6bf8f04f4126c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p3 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "Bakgrunn: {{fact}}\n\nSpørsmål: {{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b819b72b1c703ab5a0afd5b1c891b444d36d7de4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p4 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "Ta utgangspunkt i følgande fakta når du svarar på spørsmålet: {{fact}}\n\n{{question_stem}}\nVel rett svar blant desse alternativa:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a22f147a7e7114d616451a518f0951daa4676d75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p0 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aa3fc62ae4016209e1a0d0da9a75d3b95c1a39a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p1 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "Faktatekst: {{fact}}\nSpørsmål til teksten: {{question_stem}}\n\nSvaralternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n\nHva er riktig svar?" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..667b04a114fec38d60e9e5bfcb522b266ddd51d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p2 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nEr det riktige svaret A, B, C, eller D?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31f7b7fe8893bddf52416e1cfd6ef279afe84294 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p3 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "Bakgrunn: {{fact}}\n\nSpørsmål: {{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7b8ae6feeaa4507a82afff7f36917cfbf7675e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p4 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "Ta utgangspunkt i følgende fakta når du svarer på spørsmålet: {{fact}}\n\n{{question_stem}}\nVelg riktig svar blant disse alternativene:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c74e93a2e419ff4e6d607ef6931bf89fa729aa01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noropenbookqa/utils.py @@ -0,0 +1,5 @@ +import datasets + + +def filter_dataset(dataset: datasets.Dataset) -> datasets.Dataset: + return dataset.filter(lambda example: len(example["fact"]) > 0) diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/_norquad_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/_norquad_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1a168ce6811ddb47cfe712b534a083109d7093c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/_norquad_yaml @@ -0,0 +1,25 @@ +tag: norquad +dataset_path: ltg/norquad +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_target: '{{answers["text"][0]}}' +process_results: !function utils.process_results +process_docs: !function utils.process_docs +target_delimiter: ' ' +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 32 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..690a10e8f6b9367c60fa8026cd9df29a6caa4829 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p0.yaml @@ -0,0 +1,3 @@ +task: norquad_p0 +include: _norquad_yaml +doc_to_text: !function utils.p0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..640a8d85ba7cc317d1879bfd013ffc862d329ac2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p1.yaml @@ -0,0 +1,3 @@ +task: norquad_p1 +include: _norquad_yaml +doc_to_text: !function utils.p1 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b18ce5a0feff38bf62f68de342d5fbbd8ed0c44a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p2.yaml @@ -0,0 +1,3 @@ +task: norquad_p2 +include: _norquad_yaml +doc_to_text: !function utils.p2 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f10683be8638797c6e029cf29bfe9bca1e125f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p3.yaml @@ -0,0 +1,3 @@ +task: norquad_p3 +include: _norquad_yaml +doc_to_text: !function utils.p3 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc8b42dfde23d29cefbb39efac974110f0f7fc9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/norquad_p4.yaml @@ -0,0 +1,3 @@ +task: norquad_p4 +include: _norquad_yaml +doc_to_text: !function utils.p4 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1a072fb858834a8b61f173fd39ff58b63cb3686e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norquad/utils.py @@ -0,0 +1,62 @@ +import datasets +import transformers.data.metrics.squad_metrics as squad_metrics + + +def process_results(doc, results): + preds = results[0] + reference = doc["answers"]["text"][0] + f1_sum = squad_metrics.compute_f1(reference, preds) + exact_match = squad_metrics.compute_exact(reference, preds) + return {"f1": f1_sum, "exact_match": exact_match} + + +def process_docs(dataset: datasets.Dataset): + def _helper(doc): + doc["title"] = doc["context"].strip().split("\n")[0].strip() + doc["passage"] = "\n".join(doc["context"].strip().split("\n")[1:]).strip() + doc["question"] = " ".join(doc["question"].strip().split()) + return doc + + return dataset.map(_helper) + + +def p0(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f"Tittel: {title}\n\nTekst: {passage}\n\nSpørsmål: {question}\n\nSvar:" + return prompt + + +def p1(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f'Tittel: {title}\n\nTekst: {passage}\n\nGitt teksten over, hva er svaret på følgende spørsmål? "{question}"\n\nSvar:' + return prompt + + +def p2(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = ( + f"Tittel: {title}\n\nTekst: {passage}\n\nSvar på følgende: {question}\n\nSvar:" + ) + return prompt + + +def p3(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f'Tittel: {title}\n\nTekst: {passage}\n\nHvordan kan man svare på spørsmålet "{question}", gitt teksten over?\n\nSvar:' + return prompt + + +def p4(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f'Tittel: {title}\n\nTekst:{passage}\n\nGitt teksten over, besvar følgende spørsmål: "{question}"\n\nSvar:' + return prompt diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a9604632eeff5045aff2f6aee3b3cb1d45cf17 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml @@ -0,0 +1,20 @@ +task: norrewrite_instruct +dataset_path: ltg/norrewrite-instruct +training_split: null +test_split: test +doc_to_text: "{{prompt}} {{context}}" +doc_to_target: response +output_type: generate_until +metric_list: + - metric: bleu + higher_is_better: true + - metric: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/_norsumm_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/_norsumm_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6121c038583965065ed40134561deafc2da3177 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/_norsumm_yaml @@ -0,0 +1,35 @@ +dataset_path: SamiaT/NorSumm +training_split: null +validation_split: null +test_split: test +num_fewshot: 0 +doc_to_target: summaries +output_type: generate_until +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metric_list: + - metric: bleu_max + aggregation: mean + higher_is_better: true + - metric: bleu_avg + aggregation: mean + higher_is_better: true + - metric: rougeL_max + aggregation: mean + higher_is_better: true + - metric: rougeL_avg + aggregation: mean + higher_is_better: true + - metric: bertscore_f1_max + aggregation: mean + higher_is_better: true + - metric: bertscore_f1_avg + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f79a7e5324737b0904c18fa9d657654156b59da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p0 +include: ../_norsumm_yaml +doc_to_text: "Skriv ei oppsummering av følgande artikkel med berre nokre få punkt: {{article}}\nOppsummering:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e693f3500924db00e8a1e3f51ed842de829f03ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p1 +include: ../_norsumm_yaml +doc_to_text: "Oppsummer følgande artikkel med nokre få setningar: {{article}}\nOppsummering:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c2d725c638a16dbdd80587b27d65512f6959c01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p2 +include: ../_norsumm_yaml +doc_to_text: "{{article}}\nSkriv ein kort og presis oppsummering av teksten over. Språket må vere klart og lett å forstå. Sørg for å ikkje introdusere feil. Oppsummeringa må dekkje følgande spørsmål: kven, kva, kor, når, og kvifor er denne saka viktig å vite om. Oppsummeringa må vere engasjerande og framheve nøkkelinformasjon frå artikkelen. Oppsummeringa skal innehalde maksimalt 700 tegn, inkludert mellomrom." +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a21f8438ed91a72618c7f6984118fd6a517ac6c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p3 +include: ../_norsumm_yaml +doc_to_text: "Gje eit kortfatta samandrag av følgande tekst: {{article}}" +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1385549e5c57be539f778eb495ce4af1d303eec2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p4 +include: ../_norsumm_yaml +doc_to_text: "Lag ein kort oppsummering som samanfattar den følgande teksten i nokre få punkt:\n{{article}}\n\nOppsummering:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e89e95ca2f3f13b1015f7b3cda3e21441d13b74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p5 +include: ../_norsumm_yaml +doc_to_text: "Heile artikkelen:\n{{article}}\n\nHovudpunkt:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4489c35506ca6f4e0c1b2384d8571f2862c18ea4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p0 +include: ../_norsumm_yaml +doc_to_text: "Skriv en oppsummering av følgende artikkel med kun noen få punkter: {{article}}\nOppsummering:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2311b177197ed21998cb127758e413088d86e0e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p1 +include: ../_norsumm_yaml +doc_to_text: "Oppsummer følgende artikkel med noen få setninger: {{article}}\nOppsummering:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f56d457cb2cc887f2f12a804e2914294cf786f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p2 +include: ../_norsumm_yaml +doc_to_text: "{{article}}\nSkriv en kort og presis oppsummering av teksten over. Språket må være klart og lett å forstå. Sørg for å ikke introdusere feil. Oppsummeringen må dekke følgende spørsmål: hvem, hva, hvor, når, og hvorfor er denne saken viktig å vite om. Oppsummeringen må være engasjerende og fremheve nøkkelinformasjon fra artikkelen. Oppsummeringen skal inneholde maksimalt 700 tegn, inkludert mellomrom." +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..668010386639f34fe96030d28060270ecfabbe63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p3 +include: ../_norsumm_yaml +doc_to_text: "Gi et kortfattet sammendrag av følgende tekst: {{article}}" +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2425e9284db64b7c444ab848240c67f43a96bee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p4 +include: ../_norsumm_yaml +doc_to_text: "Lag en kort oppsummering som sammenfatter den følgende teksten i noen få punkter:\n{{article}}\n\nOppsummering:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a367f79d5507893904f23c52bd136a3905ea25f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p5 +include: ../_norsumm_yaml +doc_to_text: "Hele artikkelen:\n{{article}}\n\nHovedpunkter:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..24ed9c436c7ebbbed916ccbb00e7c35df828df60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsumm/utils.py @@ -0,0 +1,126 @@ +import datasets +import numpy as np +from evaluate import load + + +try: + import bert_score + import sacrebleu + from rouge_score import rouge_scorer, scoring +except ModuleNotFoundError as e: + raise type(e)( + "`sacrebleu`, `bert_score`, and `rouge_score` are required for evaluating the model on NorEval." + ) from e + + +ROUGE_SCORER = None +BERTSCORE = None + + +def process_results(doc, results): + completion = results[0] + references = doc["summaries"] + + bleu_scores = [bleu([[reference]], [completion]) for reference in references] + bleu_max = np.nanmax(bleu_scores) + bleu_avg = np.nanmean(bleu_scores) + + rouge_scores = [rouge([reference], [completion]) for reference in references] + rougeL_scores = [score["rougeLsum"] for score in rouge_scores] + rougeL_max = np.nanmax(rougeL_scores) + rougeL_avg = np.nanmean(rougeL_scores) + + bertscore_f1s = [ + bertscore_f1(references=[reference], predictions=[completion]) + for reference in references + ] + bertscore_f1_max = np.nanmax(bertscore_f1s) + bertscore_f1_avg = np.nanmean(bertscore_f1s) + + return { + "bleu_max": bleu_max, + "bleu_avg": bleu_avg, + "rougeL_max": rougeL_max, + "rougeL_avg": rougeL_avg, + "bertscore_f1_max": bertscore_f1_max, + "bertscore_f1_avg": bertscore_f1_avg, + } + + +def bleu(refs, preds): + """ + Returns `t5` style BLEU scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41 + + :param refs: + A `list` of `list` of reference `str`s. + :param preds: + A `list` of predicted `str`s. + """ + score = sacrebleu.corpus_bleu( + preds, + refs, + smooth_method="exp", + smooth_value=0.0, + force=False, + lowercase=False, + tokenize="intl", + use_effective_order=False, + ).score + return score + + +def rouge(refs, preds): + """ + Returns `t5` style ROUGE scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68 + + :param refs: + A `list` of reference `strs`. + :param preds: + A `list` of predicted `strs`. + """ + rouge_types = ["rougeLsum"] + + global ROUGE_SCORER + if ROUGE_SCORER is None: + # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant + ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types) + scorer = ROUGE_SCORER + + # Add newlines between sentences to correctly compute `rougeLsum`. + + def _prepare_summary(summary): + summary = summary.replace(" . ", ".\n") + return summary + + # Accumulate confidence intervals. + aggregator = scoring.BootstrapAggregator() + for ref, pred in zip(refs, preds): + ref = _prepare_summary(ref) + pred = _prepare_summary(pred) + aggregator.add_scores(scorer.score(ref, pred)) + result = aggregator.aggregate() + return {type: result[type].mid.fmeasure * 100 for type in rouge_types} + + +def bertscore_f1(references, predictions): + """Computes the F1 score of the BERTScore metric. + Args: + references: A list of reference strings. + predictions: A list of predicted strings. + **kwargs: Additional keyword arguments. + Returns: + The F1 score of the BERTScore metric. + """ + global BERTSCORE + if BERTSCORE is None: + # init BERTScore once + BERTSCORE = load("bertscore") + bertscore = BERTSCORE + return bertscore.compute( + predictions=predictions, + references=references, + model_type="bert-base-multilingual-cased", + num_layers=9, + )["f1"][0] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3327a4c3c98902c32888a584145ed206d032031e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml @@ -0,0 +1,20 @@ +task: norsummarize_instruct +dataset_path: ltg/norsummarize-instruct +training_split: null +test_split: test +doc_to_text: "{{prompt}} {{context}}" +doc_to_target: response +output_type: generate_until +metric_list: + - metric: bleu + higher_is_better: true + - metric: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cf6d57420ee24d651dce02a00bbbb680200efed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml @@ -0,0 +1,54 @@ +dataset_path: ltg/nortruthfulqa_gen +output_type: generate_until +training_split: null +validation_split: validation +test_split: null +num_fewshot: 0 +doc_to_target: " " +process_docs: !function utils.process_docs +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 64 +metric_list: + - metric: bleu_max + aggregation: mean + higher_is_better: true + - metric: bleu_acc + aggregation: mean + higher_is_better: true + - metric: bleu_diff + aggregation: mean + higher_is_better: true + - metric: rouge1_max + aggregation: mean + higher_is_better: true + - metric: rouge1_acc + aggregation: mean + higher_is_better: true + - metric: rouge1_diff + aggregation: mean + higher_is_better: true + - metric: rouge2_max + aggregation: mean + higher_is_better: true + - metric: rouge2_acc + aggregation: mean + higher_is_better: true + - metric: rouge2_diff + aggregation: mean + higher_is_better: true + - metric: rougeL_max + aggregation: mean + higher_is_better: true + - metric: rougeL_acc + aggregation: mean + higher_is_better: true + - metric: rougeL_diff + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..240d5fb252f246aba55a99c3d9fceaf884f5fd3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p0 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: question diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9412dada87a62e3676e49c5a277b91266773adda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p1 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Spørsmål: {{question}}\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be9d318c4371af047f2b6c511b75bbb0cbea9011 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p2 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Skriv svaret på følgande spørsmål: {{question}}\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b8a525240a5ddcf54ff41f2e186183b76e3728 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p3 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "{{question}}\nKva er rett svar på spørsmålet?\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5153b878efa3504d63311cbafc06243edf33b1c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p4 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Svar sant på følgande: {{question}}\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6231715628e0881bcefcab64a3b17b52aca37c7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p0 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: question diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6633e8cdcdb972feaac379e5aafcf586610ff273 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p1 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Spørsmål: {{question}}\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac7f57da928d3a781ab7c3a945c98b668fb1ba63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p2 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Skriv svaret på følgende spørsmål: {{question}}\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f196b1010056320b3d31307ad746d536b971f08b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p3 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "{{question}}\nHva er riktig svar på spørsmålet?\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d6deaf17b55b7cac51f58feaf47255727cfac45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p4 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Svar sant på følgende: {{question}}\nSvar:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5cfe16de6001d42e14a216b5f198287efcda2527 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py @@ -0,0 +1,152 @@ +import datasets +import numpy as np +import sacrebleu +from rouge_score import rouge_scorer, scoring + + +try: + import sacrebleu + from rouge_score import rouge_scorer, scoring +except ModuleNotFoundError as e: + raise type(e)( + "`sacrebleu` and `rouge_score` are required for evaluating the model on NorEval." + ) from e + + +ROUGE_SCORER = None + + +def preprocess_function(examples): + def _format_answers(answers): + formatted_answers = [] + for answer in answers: + answer = answer.strip() + if len(answer): + # Add a period after all answers. + if answer[-1] != ".": + formatted_answers.append(answer + ".") + else: + formatted_answers.append(answer) + return formatted_answers + + incorrect_answers = _format_answers(examples["incorrect_answers"]) + correct_answers = _format_answers(examples["correct_answers"]) + return { + "question": examples["question"].strip(), + "correct_answers": correct_answers, + "incorrect_answers": incorrect_answers, + } + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + return dataset.map(preprocess_function) + + +def process_results(doc, results): + completion = results[0] + true_refs, false_refs = doc["correct_answers"], doc["incorrect_answers"] + all_refs = true_refs + false_refs + + # BLEU + bleu_scores = [bleu([[ref]], [completion]) for ref in all_refs] + bleu_correct = np.nanmax(bleu_scores[: len(true_refs)]) + bleu_incorrect = np.nanmax(bleu_scores[len(true_refs) :]) + bleu_max = bleu_correct + bleu_diff = bleu_correct - bleu_incorrect + bleu_acc = int(bleu_correct > bleu_incorrect) + + # ROUGE-N + rouge_scores = [rouge([ref], [completion]) for ref in all_refs] + # ROUGE-1 + rouge1_scores = [score["rouge1"] for score in rouge_scores] + rouge1_correct = np.nanmax(rouge1_scores[: len(true_refs)]) + rouge1_incorrect = np.nanmax(rouge1_scores[len(true_refs) :]) + rouge1_max = rouge1_correct + rouge1_diff = rouge1_correct - rouge1_incorrect + rouge1_acc = int(rouge1_correct > rouge1_incorrect) + # ROUGE-2 + rouge2_scores = [score["rouge2"] for score in rouge_scores] + rouge2_correct = np.nanmax(rouge2_scores[: len(true_refs)]) + rouge2_incorrect = np.nanmax(rouge2_scores[len(true_refs) :]) + rouge2_max = rouge2_correct + rouge2_diff = rouge2_correct - rouge2_incorrect + rouge2_acc = int(rouge2_correct > rouge2_incorrect) + # ROUGE-L + rougeL_scores = [score["rougeLsum"] for score in rouge_scores] + rougeL_correct = np.nanmax(rougeL_scores[: len(true_refs)]) + rougeL_incorrect = np.nanmax(rougeL_scores[len(true_refs) :]) + rougeL_max = rougeL_correct + rougeL_diff = rougeL_correct - rougeL_incorrect + rougeL_acc = int(rougeL_correct > rougeL_incorrect) + + return { + "bleu_max": bleu_max, + "bleu_acc": bleu_acc, + "bleu_diff": bleu_diff, + "rouge1_max": rouge1_max, + "rouge1_acc": rouge1_acc, + "rouge1_diff": rouge1_diff, + "rouge2_max": rouge2_max, + "rouge2_acc": rouge2_acc, + "rouge2_diff": rouge2_diff, + "rougeL_max": rougeL_max, + "rougeL_acc": rougeL_acc, + "rougeL_diff": rougeL_diff, + } + + +def bleu(refs, preds): + """ + Returns `t5` style BLEU scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41 + + :param refs: + A `list` of `list` of reference `str`s. + :param preds: + A `list` of predicted `str`s. + """ + score = sacrebleu.corpus_bleu( + preds, + refs, + smooth_method="exp", + smooth_value=0.0, + force=False, + lowercase=False, + tokenize="intl", + use_effective_order=False, + ).score + return score + + +def rouge(refs, preds): + """ + Returns `t5` style ROUGE scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68 + + :param refs: + A `list` of reference `strs`. + :param preds: + A `list` of predicted `strs`. + """ + rouge_types = ["rouge1", "rouge2", "rougeLsum"] + scorer = rouge_scorer.RougeScorer(rouge_types) + # Add newlines between sentences to correctly compute `rougeLsum`. + + global ROUGE_SCORER + if ROUGE_SCORER is None: + # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant + ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types) + scorer = ROUGE_SCORER + + def _prepare_summary(summary): + summary = summary.replace(" . ", ".\n") + return summary + + # Accumulate confidence intervals. + aggregator = scoring.BootstrapAggregator() + for ref, pred in zip(refs, preds): + ref = _prepare_summary(ref) + pred = _prepare_summary(pred) + aggregator.add_scores(scorer.score(ref, pred)) + result = aggregator.aggregate() + return {type: result[type].mid.fmeasure * 100 for type in rouge_types} diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml new file mode 100644 index 0000000000000000000000000000000000000000..590dc6085f48242a028c0b24f3b65ce3fe47eba2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml @@ -0,0 +1,14 @@ +dataset_path: ltg/nortruthfulqa_mc +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +num_fewshot: 0 +doc_to_target: 0 +doc_to_choice: "{{mc1_targets.choices}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7608b46a9a1309c1d7288045065255dfe03314a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p0 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p0_nn diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b4fae7e58ef01286a3a23babe7086aa1c6cc0ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p1 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p1_nn diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8d61d464e50afe72926269f77a69bffbd255a52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p2 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p2_nn +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d93316d34dbe177de0caa74ed8b0774ae8d19336 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p3 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p3_nn +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74914f80d653697352f9d2546190663b4511bef1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p4 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p4_nn diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b66e1feb4f237efc120eb2af686d95b833407683 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py @@ -0,0 +1,35 @@ +def p0_nn(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nn(doc): + prompt = "Spørsmål: {question}\n\nSvaralternativ:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nn(doc): + prompt = "Spørsmål: {question}\n\nKva av følgande alternativ er rett svar på spørsmålet?{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p3_nn(doc): + prompt = "Gitt følgande spørsmål, kva av dei moglege svara under er rett?\nSpørsmål: {question}\n{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nn(doc): + prompt = "{question}\nVel eit av følgande moglege svar:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36acd76f945a8ae84f477c923ed8b2f1c7bea6a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p0 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p0_nb diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74b5d1702d2ea5c8e3342e8b834b7281b30ddc8f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p1 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p1_nb diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30665f9e4164909278c92fc98a794f3269ffa4b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p2 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p2_nb +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40893b52230d6c20d1568633389b2aca3b5c6f01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p3 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p3_nb +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0595fdf835bb89b7da31fa7fbfccde6b872b720 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p4 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p4_nb diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..08e5cdab333eb46c02a931ac4e35b692b27b2ee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py @@ -0,0 +1,35 @@ +def p0_nb(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nb(doc): + prompt = "Spørsmål: {question}\n\nSvaralternativer:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nb(doc): + prompt = "Spørsmål: {question}\n\nHvilke av følgende alternativer er riktig svar på spørsmålet?{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p3_nb(doc): + prompt = "Gitt følgende spørsmål, hvilket av de mulige svarene under er riktig?\nSpørsmål: {question}\n{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nb(doc): + prompt = "{question}\nVelg et av følgende mulige svar:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7167063b2133b3cfcbeca670fd557c50938b04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml @@ -0,0 +1,16 @@ +dataset_path: ltg/nrk_quiz_qa +output_type: multiple_choice +training_split: null +validation_split: null +test_split: test +num_fewshot: 0 +doc_to_target: "{{choices.label.index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e94c9a272039468fba86abff1157abac3dfe37ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p0 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p0_nn +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05d260db892884b614badca7db47c2b71fe5dbb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p1 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p1_nn +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fee4fadb28e1626b60ab877a036957360d0c6be3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p2 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p2_nn +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac9aafd6caf0f95be2ac9fcfe9558adcd3beb608 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p3 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p3_nn +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d095b6f12b1f2f09ed1757e1c2899794f5789302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p4 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p4_nn +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b9dcae1cdce3d57ff2057dc41e17fe5775a9e8b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py @@ -0,0 +1,44 @@ +def p0_nn(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nn(doc): + prompt = "{question}\n\nSvaralternativer:{choices}\n\nKva er rett svar?\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nn(doc): + prompt = "{question}{choices}\n\nEr det rette svaret {enumerated_choices}?\n\nSvar:" + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + enumerated_choices = ", ".join( + doc["choices"]["label"][:-1] + ) + ", eller {latest_choice}".format(latest_choice=doc["choices"]["label"][-1]) + if len(doc["choices"]["label"]) == 2: + enumerated_choices = enumerated_choices.replace(", eller", " eller") + return prompt.format( + question=doc["question"], choices=choices, enumerated_choices=enumerated_choices + ) + + +def p3_nn(doc): + prompt = "Spørsmål: {question}{choices}\n\nSvar:" + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nn(doc): + prompt = "{question}\nVel rett svar blant desse alternativa:{choices}\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..688c2ed10352e467d8c10c5af728ef18bd4ae793 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p0 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p0_nb +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1920e9519c477bb722e906663cf3a6db87dbff8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p1 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p1_nb +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b7746da531f77c4e8c8cd5c3889be22c8eaf1b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p2 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p2_nb +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..394070535e40abe23f51c8d23e124d0a09fafb44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p3 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p3_nb +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f2e645c41fddf40feec4e69cb0dc489c932a4a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p4 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p4_nb +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aac29f6965cbec9903455642243f47450d9a0bda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py @@ -0,0 +1,46 @@ +def p0_nb(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nb(doc): + prompt = "{question}\n\nSvaralternativer:{choices}\n\nHva er riktig svar?\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nb(doc): + prompt = ( + "{question}{choices}\n\nEr det riktige svaret {enumerated_choices}?\n\nSvar:" + ) + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + enumerated_choices = ", ".join( + doc["choices"]["label"][:-1] + ) + ", eller {latest_choice}".format(latest_choice=doc["choices"]["label"][-1]) + if len(doc["choices"]["label"]) == 2: + enumerated_choices = enumerated_choices.replace(", eller", " eller") + return prompt.format( + question=doc["question"], choices=choices, enumerated_choices=enumerated_choices + ) + + +def p3_nb(doc): + prompt = "Spørsmål: {question}{choices}\n\nSvar:" + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nb(doc): + prompt = "{question}\nVelg riktig svar blant disse alternativene:{choices}\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml new file mode 100644 index 0000000000000000000000000000000000000000..01b1cfea88e5a05165fa14a0043abc60ffc6547e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml @@ -0,0 +1,19 @@ +dataset_path: Helsinki-NLP/tatoeba_mt +training_split: validation +test_split: test +output_type: generate_until +dataset_kwargs: + trust_remote_code: true +metric_list: + - metric: bleu + higher_is_better: true + - metric: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c34c861d167615e8a6f3a3d786b3f24de6f93791 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p0 +include: ../_tatoeba_yaml +doc_to_text: "Engelsk: {{sourceString}}\nNynorsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dca37a83f53f95b41496806c9aedcfb0c8eddc5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p1 +include: ../_tatoeba_yaml +doc_to_text: "Omsett følgande setning til nynorsk: {{sourceString}}\nNynorsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e84f18c0144589c3ed864799930298ead8c391d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gje ei nynorsk omsetjing av denne setninga: {{sourceString}}\nNynorsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccc700269ae431d15f2d31be6ba9fe73d5f76518 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p3 +include: ../_tatoeba_yaml +doc_to_text: "Kva blir \"{{sourceString}}\" på nynorsk?\nNynorsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfd9c77fec86b806e8050cfd51c294355b17c963 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p0 +include: ../_tatoeba_yaml +doc_to_text: "Engelsk: {{sourceString}}\nBokmål:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c762b47b338ab6941772bb79069e7714840f46bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p1 +include: ../_tatoeba_yaml +doc_to_text: "Oversett følgende setning til norsk bokmål: {{sourceString}}\nBokmål:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1362c224eb233f0c794bbbe9862728840f57ea1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gi en oversettelse til bokmål for denne setningen: {{sourceString}}\nBokmål:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66bda471f1d3b7246e8dad43a2a008186e8d5bd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p3 +include: ../_tatoeba_yaml +doc_to_text: "Hva blir \"{{sourceString}}\" på bokmål?\nBokmål:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a09be934f3cb3fdf7c79ab9203b3d7d5d78f6011 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p0 +include: ../_tatoeba_yaml +doc_to_text: "Nynorsk: {{targetString}}\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf69ce7c30fe30de055bf5c3048b28f35d99d13b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p1 +include: ../_tatoeba_yaml +doc_to_text: "Omsett følgande setning til engelsk: {{targetString}}\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f179696d10b9130600dc9c032b62b564981f35d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gje ei engelsk omsetjing av denne setninga: {{targetString}}\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5b5583795d0e9676e1c3e2505b11834cdc29bde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p3 +include: ../_tatoeba_yaml +doc_to_text: "Kva blir \"{{targetString}}\" på engelsk?\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9213403f1da6a2c6037d9fe08c07968ab8da5b9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p0 +include: ../_tatoeba_yaml +doc_to_text: "Bokmål: {{targetString}}\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efb7c6315d893a21ae80a9bf0c900dd09dcf6b06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p1 +include: ../_tatoeba_yaml +doc_to_text: "Oversett følgende setning til engelsk: {{targetString}}\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8a17df2dce28670c8e24b833e55da79142e1ca7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gi en engelsk oversettelse av denne setningen: {{targetString}}\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85a29eec508711815259b9823660d8ce7cbe95ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p3 +include: ../_tatoeba_yaml +doc_to_text: "Hva blir \"{{targetString}}\" på engelsk?\nEngelsk:" diff --git a/lm-evaluation-harness/lm_eval/tasks/noticia/README.md b/lm-evaluation-harness/lm_eval/tasks/noticia/README.md new file mode 100644 index 0000000000000000000000000000000000000000..553ed20f49cab7d4e23bb4cda4619eb408c671f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noticia/README.md @@ -0,0 +1,53 @@ +# NoticIA + +### Paper + +Title: `NoticIA: A Clickbait Article Summarization Dataset in Spanish` + +Abstract: https://arxiv.org/abs/2404.07611 + +We present NoticIA, a dataset consisting of 850 Spanish news articles featuring prominent clickbait headlines, each paired with high-quality, single-sentence generative summarizations written by humans. This task demands advanced text understanding and summarization abilities, challenging the models' capacity to infer and connect diverse pieces of information to meet the user's informational needs generated by the clickbait headline. We evaluate the Spanish text comprehension capabilities of a wide range of state-of-the-art large language models. Additionally, we use the dataset to train ClickbaitFighter, a task-specific model that achieves near-human performance in this task. + +Homepage: https://github.com/ikergarcia1996/NoticIA + +### Citation + +``` +@article{noticia2024, + title={NoticIA: A Clickbait Article Summarization Dataset in Spanish}, + author={Iker García-Ferrero and Begoña Altuna}, + year={2024}, + journal = {Procesamiento del Lenguaje Natural}, + volume = {73}, + number = {0}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet. + +#### Tasks + +* `noticia` + +#### Metrics + +Following the original implementation, this task will compute the 'Rouge1 score' and 'Average Summary Length.' + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/noticia/noticia.yaml b/lm-evaluation-harness/lm_eval/tasks/noticia/noticia.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68d47707083949e7baf8bfc3135559720a44f30d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noticia/noticia.yaml @@ -0,0 +1,29 @@ +task: noticia +dataset_path: Iker/NoticIA +dataset_name: null +output_type: generate_until +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +training_split: null +validation_split: null +test_split: test +fewshot_split: null +doc_to_text: "Ahora eres una Inteligencia Artificial experta en desmontar titulares sensacionalistas o clickbait. Tu tarea consiste en analizar noticias con titulares sensacionalistas y generar un resumen de una sola frase que revele la verdad detrás del titular.\nEste es el titular de la noticia: {{web_headline}}\nEl titular plantea una pregunta o proporciona información incompleta. Debes buscar en el cuerpo de la noticia una frase que responda lo que se sugiere en el título. Siempre que puedas cita el texto original, especialmente si se trata de una frase que alguien ha dicho. Si citas una frase que alguien ha dicho, usa comillas para indicar que es una cita. Usa siempre las mínimas palabras posibles. No es necesario que la respuesta sea una oración completa, puede ser sólo el foco de la pregunta. Recuerda responder siempre en Español.\nEste es el cuerpo de la noticia:\n{{web_text}}" +doc_to_target: summary +target_delimiter: " " +num_fewshot: 0 +should_decontaminate: false +doc_to_decontamination_query: sentence +metric_list: + - metric: !function utils.rouge1 + higher_is_better: true + aggregation: !function utils.rouge1_agg + - metric: !function utils.average_len + higher_is_better: false + aggregation: !function utils.average_len_agg +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noticia/utils.py b/lm-evaluation-harness/lm_eval/tasks/noticia/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..27a3a4363757cc5fca7b5b7ff683db5f8f556643 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noticia/utils.py @@ -0,0 +1,55 @@ +import string + +import evaluate + + +def clean_text(text: str) -> str: + # Remove punctuation + text = text.translate(str.maketrans("", "", string.punctuation)) + + # Remove newlines and multiple spaces + text = text.replace("\n", " ").strip() + text = " ".join(text.split()).strip() + + # lowercase + text = text.lower() + + return text + + +def rouge1(items): + """ + # passthrough for efficiency + """ + return items + + +def average_len(items): + """ + # passthrough for efficiency + """ + return items + + +def rouge1_agg(items): + """ + Higher is better + """ + + refs = list(zip(*items))[0] + refs = [[clean_text(ref)] for ref in refs] + # print("refs", refs) + preds = [clean_text(x) for x in list(zip(*items))[1]] + # print("preds", preds) + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rouge1"] + + +def average_len_agg(items): + """ + Higher is better + """ + + preds = [clean_text(x) for x in list(zip(*items))[1]] + + return sum(len(x.split()) for x in preds) / len(preds) diff --git a/lm-evaluation-harness/lm_eval/tasks/nq_open/README.md b/lm-evaluation-harness/lm_eval/tasks/nq_open/README.md new file mode 100644 index 0000000000000000000000000000000000000000..01792089a675f0cd17c28819e63212750815a554 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/nq_open/README.md @@ -0,0 +1,26 @@ +### Paper + +Question Answering dataset based on aggregated user queries from Google Search. + +Homepage: https://research.google/pubs/natural-questions-a-benchmark-for-question-answering-research/ + +Homepage: [google-research-datasets/natural-questions@master/nq_open](https://github.com/google-research-datasets/natural-questions/tree/master/nq_open) + +Paper: [aclanthology.org/P19-1612](https://aclanthology.org/P19-1612/) + +Derived from the Natural Questions dataset, introduced in https://storage.googleapis.com/gweb-research2023-media/pubtools/pdf/1f7b46b5378d757553d3e92ead36bda2e4254244.pdf . + + +### Citation + +``` +@article{47761, +title = {Natural Questions: a Benchmark for Question Answering Research}, +author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov}, +year = {2019}, +journal = {Transactions of the Association of Computational Linguistics}} +``` + +### Tasks + +* `nq_open` diff --git a/lm-evaluation-harness/lm_eval/tasks/nq_open/nq_open.yaml b/lm-evaluation-harness/lm_eval/tasks/nq_open/nq_open.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b2af0eee0171cdce7c133356d0312c6c10ef0ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/nq_open/nq_open.yaml @@ -0,0 +1,32 @@ +task: nq_open +dataset_path: nq_open +output_type: generate_until +training_split: train +validation_split: validation +description: "Answer these questions:\n\n" +doc_to_text: "Q: {{question}}?\nA:" +doc_to_target: "{{answer}}" # TODO: should be multi-target +fewshot_delimiter: "\n" +generation_kwargs: + until: + - "\n" + - "." + - "," + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "\\b(?:The |the |An |A |The |a |an )" +metadata: + version: 4.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/README.md b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/README.md new file mode 100644 index 0000000000000000000000000000000000000000..27c9329d126a8fa62f4f6d650983199d3b589362 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/README.md @@ -0,0 +1,47 @@ +# Multilingual ARC + +### Paper + +Title: `Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback` + +Abstract: https://arxiv.org/abs/2307.16039 + +A key technology for the development of large language models (LLMs) involves instruction tuning that helps align the models' responses with human expectations to realize impressive learning abilities. Two major approaches for instruction tuning characterize supervised fine-tuning (SFT) and reinforcement learning from human feedback (RLHF), which are currently applied to produce the best commercial LLMs (e.g., ChatGPT). To improve the accessibility of LLMs for research and development efforts, various instruction-tuned open-source LLMs have also been introduced recently, e.g., Alpaca, Vicuna, to name a few. However, existing open-source LLMs have only been instruction-tuned for English and a few popular languages, thus hindering their impacts and accessibility to many other languages in the world. Among a few very recent work to explore instruction tuning for LLMs in multiple languages, SFT has been used as the only approach to instruction-tune LLMs for multiple languages. This has left a significant gap for fine-tuned LLMs based on RLHF in diverse languages and raised important questions on how RLHF can boost the performance of multilingual instruction tuning. To overcome this issue, we present Okapi, the first system with instruction-tuned LLMs based on RLHF for multiple languages. Okapi introduces instruction and response-ranked data in 26 diverse languages to facilitate the experiments and development of future multilingual LLM research. We also present benchmark datasets to enable the evaluation of generative LLMs in multiple languages. Our experiments demonstrate the advantages of RLHF for multilingual instruction over SFT for different base models and datasets. Our framework and resources are released at this https URL. + +Homepage: `https://github.com/nlp-uoregon/Okapi` + + +### Citation + +``` +@article{dac2023okapi, + title={Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback}, + author={Dac Lai, Viet and Van Nguyen, Chien and Ngo, Nghia Trung and Nguyen, Thuat and Dernoncourt, Franck and Rossi, Ryan A and Nguyen, Thien Huu}, + journal={arXiv e-prints}, + pages={arXiv--2307}, + year={2023} +} +``` + +### Groups and Tasks + +#### Groups + +- arc_multilingual + +#### Tasks + +- `arc_{ar,bn,ca,da,de,es,eu,fr,gu,hi,hr,hu,hy,id,it,kn,ml,mr,ne,nl,pt,ro,ru,sk,sr,sv,ta,te,uk,vi,zh}` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca70b369e4d22fc7c3733b41424b03029183a45f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml @@ -0,0 +1,23 @@ +tag: + - arc_multilingual +dataset_path: null +dataset_name: null +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "query" +doc_to_target: "gold" +doc_to_choice: "choices" +should_decontaminate: true +doc_to_decontamination_query: "query" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_ca.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95b433d6cb3a7f10af13f6e45627ab007538c522 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_ca.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ca +dataset_path: alexandrainst/m_arc +dataset_name: ca +training_split: train +validation_split: validation +test_split: test diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_da.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_da.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7209f8cbc047e67f6f175c683743467b4f28275c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_da.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_da +dataset_path: alexandrainst/m_arc +dataset_name: da +training_split: train +validation_split: validation +test_split: test diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_eu.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13798d45b544993725b82c27f475d61fd2b4c0bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_eu.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_eu +dataset_path: alexandrainst/m_arc +dataset_name: eu +training_split: train +validation_split: validation +test_split: test diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78c75932201786eda593026bcc2538c4aecd5911 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/arc_pt.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_pt +dataset_path: alexandrainst/m_arc +dataset_name: pt +training_split: train +validation_split: validation +test_split: test diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_ne.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_ne.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d12fb26c62dc6984ff225770bea2f1f8b50a43 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_ne.yaml @@ -0,0 +1,6 @@ +include: _hellaswag_yaml +task: hellaswag_ne +dataset_path: alexandrainst/m_hellaswag +dataset_name: ne +training_split: null +validation_split: val diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7082b5a615dbeb4777ebc0e039da03543f2e1d1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_pt.yaml @@ -0,0 +1,6 @@ +include: _hellaswag_yaml +task: hellaswag_pt +dataset_path: alexandrainst/m_hellaswag +dataset_name: pt +training_split: null +validation_split: val diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sk.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b831f755f7cd055a8aacd2ae1d6b39d2a31bc0f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sk.yaml @@ -0,0 +1,6 @@ +include: _hellaswag_yaml +task: hellaswag_sk +dataset_path: alexandrainst/m_hellaswag +dataset_name: sk +training_split: null +validation_split: val diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sv.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ca7d56778850e21252d0e67288c801f0f070df9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sv.yaml @@ -0,0 +1,6 @@ +include: _hellaswag_yaml +task: hellaswag_sv +dataset_path: alexandrainst/m_hellaswag +dataset_name: sv +training_split: null +validation_split: val diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92a846b6e61d21e1359c530a6f7501a36bda9d2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml @@ -0,0 +1,6 @@ +include: _hellaswag_yaml +task: hellaswag_te +dataset_path: alexandrainst/m_hellaswag +dataset_name: te +training_split: null +validation_split: val diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b526a9e93076f7db54221072d58ca4bd7161ee97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py @@ -0,0 +1,25 @@ +import re + +import datasets + + +def preprocess(text): + text = text.strip() + # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() + out_doc = { + "query": preprocess(doc["activity_label"] + ": " + ctx), + "choices": [preprocess(ending) for ending in doc["endings"]], + "gold": int(doc["label"]), + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..04d38ed5c74a7428baac602e3a9f1e512c55f92e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py @@ -0,0 +1,27 @@ +import datasets +import yaml +from tqdm import tqdm + + +def main() -> None: + dataset_path = "alexandrainst/m_mmlu" + + for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): + file_name = f"m_mmlu_{task}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_default_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d16feec9108aca3a01623e5a3ff263e0ce43cb3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: bn +include: _default_yaml +task: m_mmlu_bn diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95eb1dc9b190fea94281e56b4564cc9e4701a1d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: da +include: _default_yaml +task: m_mmlu_da diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d36cbe6f27e6fa0328c6464f8afe33d0d45312d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: es +include: _default_yaml +task: m_mmlu_es diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf0064f782178685cde6ffb382727641d027d823 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hi +include: _default_yaml +task: m_mmlu_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6e24d8e16bfeaa96dbaa106889c11106538026 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hr +include: _default_yaml +task: m_mmlu_hr diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d824cb768a006f614fa31ff911c9dfffb01bee75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hu +include: _default_yaml +task: m_mmlu_hu diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63594e227ae0b792817a8711b8d3f683b3b2ba4b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: id +include: _default_yaml +task: m_mmlu_id diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml new file mode 100644 index 0000000000000000000000000000000000000000..494b0c10acf484b378c17cb2537660d9d6fdb80b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: is +include: _default_yaml +task: m_mmlu_is diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30795d329a290ec78a795aa6bea738b548f237e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: it +include: _default_yaml +task: m_mmlu_it diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d026c7e4cdc9a58c0df8a360b86d45267ed00b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: kn +include: _default_yaml +task: m_mmlu_kn diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5daf8736a50cc99610248752d7541538d5cc7e46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ml +include: _default_yaml +task: m_mmlu_ml diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6f6df7f30c9c9555a6a0751e5059e22c130b26c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: mr +include: _default_yaml +task: m_mmlu_mr diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76ab5a601d3a3047bdbfd55b995d6c5623e02adf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: nb +include: _default_yaml +task: m_mmlu_nb diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6f53563ed958a79eaf866b292311b4b9f56ae41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ne +include: _default_yaml +task: m_mmlu_ne diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df115a68d025e6b2ec05c193ba03d8743c0d9629 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: nl +include: _default_yaml +task: m_mmlu_nl diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de4bb65953e675b4609d6e70ad97c421aeacbd8f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: pt +include: _default_yaml +task: m_mmlu_pt diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..236d8382d7098c39235bb63edf986a848df559d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ro +include: _default_yaml +task: m_mmlu_ro diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce379b61e4e88832b9c4e007188ead2ddcd74fb1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ru +include: _default_yaml +task: m_mmlu_ru diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sk.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61589f04760a34b8fb2aa9405bb6dd1121c1448f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sk.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: sk +include: _default_yaml +task: m_mmlu_sk diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22b0ad7755564491096207d41e964535b9b8cf24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: sr +include: _default_yaml +task: m_mmlu_sr diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d433d08259c94b14fbe685dcee21f080242e8168 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: sv +include: _default_yaml +task: m_mmlu_sv diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2314894c2ba7d851b36be44d2ad99c895712626e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ta +include: _default_yaml +task: m_mmlu_ta diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0737ed37aa7765eb3312f20c6315efb1b54fba7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: te +include: _default_yaml +task: m_mmlu_te diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdc704b7d681f87e769c750701ff8cde8b9b6d3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: uk +include: _default_yaml +task: m_mmlu_uk diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1d6771e5a198c1ac50b56ea3224d33fbce41de9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: vi +include: _default_yaml +task: m_mmlu_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf92a74ff17f51a65708b17baaae60dcd39ddcfe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: zh +include: _default_yaml +task: m_mmlu_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/README.md b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cd494570acaac40bc3e03143fe94cdd082d05890 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/README.md @@ -0,0 +1,50 @@ +# Multilingual TruthfulQA + +### Paper + +Title: `Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback` + +Abstract: https://arxiv.org/abs/2307.16039 + +A key technology for the development of large language models (LLMs) involves instruction tuning that helps align the models' responses with human expectations to realize impressive learning abilities. Two major approaches for instruction tuning characterize supervised fine-tuning (SFT) and reinforcement learning from human feedback (RLHF), which are currently applied to produce the best commercial LLMs (e.g., ChatGPT). To improve the accessibility of LLMs for research and development efforts, various instruction-tuned open-source LLMs have also been introduced recently, e.g., Alpaca, Vicuna, to name a few. However, existing open-source LLMs have only been instruction-tuned for English and a few popular languages, thus hindering their impacts and accessibility to many other languages in the world. Among a few very recent work to explore instruction tuning for LLMs in multiple languages, SFT has been used as the only approach to instruction-tune LLMs for multiple languages. This has left a significant gap for fine-tuned LLMs based on RLHF in diverse languages and raised important questions on how RLHF can boost the performance of multilingual instruction tuning. To overcome this issue, we present Okapi, the first system with instruction-tuned LLMs based on RLHF for multiple languages. Okapi introduces instruction and response-ranked data in 26 diverse languages to facilitate the experiments and development of future multilingual LLM research. We also present benchmark datasets to enable the evaluation of generative LLMs in multiple languages. Our experiments demonstrate the advantages of RLHF for multilingual instruction over SFT for different base models and datasets. Our framework and resources are released at this https URL. + +Homepage: `https://github.com/nlp-uoregon/Okapi` + + +### Citation + +``` +@article{dac2023okapi, + title={Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback}, + author={Dac Lai, Viet and Van Nguyen, Chien and Ngo, Nghia Trung and Nguyen, Thuat and Dernoncourt, Franck and Rossi, Ryan A and Nguyen, Thien Huu}, + journal={arXiv e-prints}, + pages={arXiv--2307}, + year={2023} +} +``` + +### Groups and Tasks + +#### Groups + +- truthfulqa_multilingual + +#### Tasks + +- `truthfulqa_{ar,bn,ca,da,de,es,eu,fr,gu,hi,hr,hu,hy,id,it,kn,ml,mr,ne,nl,pt,ro,ru,sk,sr,sv,ta,te,uk,vi,zh}` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog +*_mc2 v2.0 (2024-Mar-11) PR #2768 - original code assumed labels were in sorted order - not always true diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7e0f25fcf1c51915855f4300790569598ef71f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml @@ -0,0 +1,20 @@ +tag: + - truthfulqa_multilingual +dataset_path: null +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: val +test_split: null +process_docs: !function utils.process_docs +doc_to_text: "query" +doc_to_target: 0 +doc_to_choice: "mc1_choices" +should_decontaminate: True +doc_to_decontamination_query: "question" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc2_yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc2_yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf28b230b25bf32dcda70177ace2759b35cfc269 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc2_yaml @@ -0,0 +1,12 @@ +include: _truthfulqa_mc1_yaml +doc_to_target: 0 +doc_to_choice: "mc2_choices" +process_results: !function utils.process_results_mc2 +should_decontaminate: True +doc_to_decontamination_query: "question" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b832c3c17f7a480da55f8e6e066003e6c329d077 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ar_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ar +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6916dbbbbf1ff49e12a0c6a95d1894ca5117f77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ar_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ar +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64ec622e4651c4afc41ec53cd298ff1e16a2b22a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_bn_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: bn +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..788450c9d8445a96aad59945d8027a857909213c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_bn_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: bn +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce0731cd8fc5f54504505a3eb2fd2fa4db58f7c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ca_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ca +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e748177330402fd9259f4d3913b4ba0db8e13d31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ca_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ca +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a64758fc18c20a6eeb9c140f5194dcc3131f6af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_da_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: da +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fdb9dc508aa141de78777fc570143f3d4cd4101 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_da_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: da +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96d5c8b29d05cbea6c0e3ebd8e1b590d7b6b0410 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_de_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: de +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8a999fb90973445023ae1bc8c3a3968cbbffd18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_de_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: de +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80d2482b69ebe95d46e2c41d4a07696942b075f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_es_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: es +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..391e2d1db73e601337b9c512facd6aeaee5d1fb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_es_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: es +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc3ee9f3c06db13511ec962ac099bfc902e45bd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_eu_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: eu +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03c5ea906daae0f09eb773842734b208a1b72ede --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_eu_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: eu +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e15c41bb8a44cb4d83e35e76f96b00a32ad668a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_fr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: fr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2ab62b624174126a1bb45092c3dde1d5679384e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_fr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: fr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6b0a6c670a7c9fe5c34445a84e4232133a721f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_gu_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: gu +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7cd3f4c29c1be995bb688d2331f839ce5969b6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_gu_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: gu +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b69f0b868693f95f5142d14d0a8e4e6aeaa93424 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hi_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hi +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c74eb2422b77ae78feefaaf2293909eca5798ebb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hi_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hi +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0746d53d60ab9c8107ccaa911313f09213fd53a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a73d119a33730aaa98346eb9c29779793acda87d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96e0645cf5a07bee3c8b68af2db547097e7548fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hu_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hu +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..242a330030a578c015adeb6abdd74bf8c7c46326 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hu_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hu +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..393acd2774c8be8e8b168fdcd63f35835900eb8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hy_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hy +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d6aae7264dbeed22630c074188bb097596d9773 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hy_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hy +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55b3d846f5689379e40e71a24c1a75f97845a098 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_id_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: id +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..896329f0f1f17ec5540c952821df53496a84d43e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_id_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: id +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0c9d6db8e33a6977c33e91b709992e6f3463e2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_it_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: it +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebabe1e40b6fa28b58839823d908d1e682deaeac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_it_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: it +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bf025d5af7a5662c9cad7506170f8c4daad4c62 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_kn_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: kn +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6425b72b94620aa3deeba5b740a808d3bcb4bd7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_kn_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: kn +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b91651c09664f14b273c25cb15eb59c95f204ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ml_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ml +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e89710bf458160491a9dfa24dfb0dc19ac674246 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ml_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ml +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e65abadc0d50e1b5a981efb3f52cc34a1de2525f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_mr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: mr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab4bbe3229a7327c7d58e943888bc227c7b617bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_mr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: mr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..431a1f6a74b9543aff5fdfdd5027427d24e32442 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ne_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ne +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11b06dcdc937b7d70afc8028ca1989be54ad1284 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_nl_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: nl +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6089191de242a3cd4ffed13f06eeae0ee2529c65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ro_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ro +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bad373faeedd6ea7a4c95b627239f294844ebd68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ro_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ro +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a396343fb1a9f0ed89c2cf6d41823100fc3c99cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ru_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ru +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53038c21b6ec8351a67ef17dc28338cea228f4c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_sk_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sk +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc2.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dfa185d4e24ee8c9bf7b9dc6cbc36d81dced975 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_sr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sr +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc1.yaml b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10677978cae6590a8d59fcd43a2fa942e7ca4e53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ta_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ta +training_split: null +validation_split: val +test_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/olaph/README.md b/lm-evaluation-harness/lm_eval/tasks/olaph/README.md new file mode 100644 index 0000000000000000000000000000000000000000..986ea2ccad047bbf28927442f240c41030abe9fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/olaph/README.md @@ -0,0 +1,27 @@ +# OLAPH + +### Paper + +Title: `OLAPH: Improving Factuality in Biomedical Long-form Question Answering` + +Abstract: [https://arxiv.org/abs/2405.12701](https://arxiv.org/abs/2405.12701) + + +#### Tasks + +* `olaph`: Open-Ended QA in english. +* `olaph_perplexity`: Open-Ended QA in english, evaluated with perplexity. + +### Citation + +```bibtex +@misc{jeong2024olaphimprovingfactualitybiomedical, + title={OLAPH: Improving Factuality in Biomedical Long-form Question Answering}, + author={Minbyul Jeong and Hyeon Hwang and Chanwoong Yoon and Taewhoo Lee and Jaewoo Kang}, + year={2024}, + eprint={2405.12701}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2405.12701}, +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/olaph/olaph.yaml b/lm-evaluation-harness/lm_eval/tasks/olaph/olaph.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78fafe69668184d0483b7d4be4ca6a22d837fbdb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/olaph/olaph.yaml @@ -0,0 +1,37 @@ +task: olaph +dataset_path: dmis-lab/MedLFQA +description: > + Instructions: You are a helpful healthcare assistant. Answer the following question as concisely as possible without omitting relevant information. + +output_type: generate_until +training_split: test +validation_split: test +test_split: test +process_docs: !function utils.process_docs +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n\n" +metric_list: + - metric: bleu + aggregation: nanmean + higher_is_better: true + - metric: rouge1 + aggregation: nanmean + higher_is_better: true + - metric: rouge2 + aggregation: nanmean + higher_is_better: true + - metric: rougeL + aggregation: nanmean + higher_is_better: true + - metric: bert_score + aggregation: nanmean + higher_is_better: true + - metric: bleurt + aggregation: nanmean + higher_is_better: true +metadata: + version: 1.2 diff --git a/lm-evaluation-harness/lm_eval/tasks/olaph/olaph_perplexity.yaml b/lm-evaluation-harness/lm_eval/tasks/olaph/olaph_perplexity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da423fe1b60a13721202201ea5f2f667c6bfdaf0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/olaph/olaph_perplexity.yaml @@ -0,0 +1,26 @@ +task: olaph_perplexity +#include: olaph.yaml +dataset_path: dmis-lab/MedLFQA +description: > + Instructions: You are a helpful healthcare assistant. Answer the following question as concisely as possible without omitting relevant information. + +training_split: test +validation_split: test +test_split: test +output_type: loglikelihood_rolling +doc_to_text: "" +process_docs: !function utils.process_docs +doc_to_target: !function utils.doc_to_target +process_results: !function utils_perplexity.process_results +generation_kwargs: + until: + - "\n\n" +metric_list: + - metric: word_perplexity + higher_is_better: false + - metric: byte_perplexity + higher_is_better: false + - metric: bits_per_byte + higher_is_better: false +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/olaph/utils.py b/lm-evaluation-harness/lm_eval/tasks/olaph/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..580c7bcbe8d07cfe144ec0e1c555eaea371dc44c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/olaph/utils.py @@ -0,0 +1,109 @@ +import datasets +import numpy as np + + +try: + import evaluate + + bleu = evaluate.load("bleu") + rouge = evaluate.load("rouge") + bertscore = evaluate.load("bertscore") + bleurt = evaluate.load("bleurt", "bleurt-base-512", module_type="metric") + +except (ModuleNotFoundError, ImportError): + raise ModuleNotFoundError( + "Please install evaluation metrics via pip install evaluate and pip install bert-score", + ) +except Exception as e: + raise RuntimeError( + f"Error loading evaluation metrics: {str(e)}. Please check your installation." + ) + + +def doc_eval(pred, refs): + try: + bleu_results = bleu.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Bleu error: {e}") + bleu_results = {"bleu": np.NAN} + + try: + rouge_results = rouge.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Rouge error: {e}") + rouge_results = {"rouge1": np.NAN, "rouge2": np.NAN, "rougeL": np.NAN} + + try: + bleurt_scores = bleurt.compute(predictions=pred, references=refs)["scores"] + except Exception as e: + print(f"Bleurt error: {e}") + bleurt_scores = [np.NAN] + + try: + bert_scores = bertscore.compute(predictions=pred, references=refs, lang="en")[ + "f1" + ] + except Exception as e: + print(f"Bert error: {e}") + bert_scores = [np.NAN] + + if bleu_results["bleu"] == 0: + # Sometimes bleu is 0.0 and this breaks the stderr computation. + bleu_results["bleu"] += 1e-5 + + results = { + "bleu": bleu_results["bleu"], + "rouge1": rouge_results["rouge1"], + "rouge2": rouge_results["rouge2"], + "rougeL": rouge_results["rougeL"], + "bleurt": np.mean(bleurt_scores), + "bert_score": np.mean(bert_scores), + } + + return results + + +def doc_to_text(doc) -> str: + return doc["Question"] + + +def doc_to_target(doc) -> str: + return doc["Free_form_answer"] + + +def process_docs(dataset: datasets.Dataset): + def _helper(doc): + return doc + + num_entries = len(dataset) + one_percent_index = int(0.1 * num_entries) + + # Select the first 1% of instances + filtered_dataset = dataset.select(range(one_percent_index)) + + return filtered_dataset.map(_helper) + + +def process_results(doc, results): + pred, refs = [results[0]], [doc_to_target(doc)] + + if len(refs[0]) < 10 or len(pred[0]) < 10: + return { + "bleu": np.NAN, + "rouge1": np.NAN, + "rouge2": np.NAN, + "rougeL": np.NAN, + "bleurt": np.NAN, + "bert_score": np.NAN, + } + + results = doc_eval(pred, refs) + + return { + "bleu": results["bleu"], + "rouge1": results["rouge1"], + "rouge2": results["rouge2"], + "rougeL": results["rougeL"], + "bleurt": results["bleurt"], + "bert_score": results["bert_score"], + } diff --git a/lm-evaluation-harness/lm_eval/tasks/olaph/utils_perplexity.py b/lm-evaluation-harness/lm_eval/tasks/olaph/utils_perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..68135fb0399632d0361c8de056acd1f234bd6d68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/olaph/utils_perplexity.py @@ -0,0 +1,14 @@ +import re + +from lm_eval.tasks.olaph.utils import doc_to_target + + +def process_results(doc, results): + (loglikelihood,) = results + _words = len(re.split(r"\s+", doc_to_target(doc))) + _bytes = len(doc_to_target(doc).encode("utf-8")) + return { + "word_perplexity": (loglikelihood, _words), + "byte_perplexity": (loglikelihood, _bytes), + "bits_per_byte": (loglikelihood, _bytes), + } diff --git a/lm-evaluation-harness/lm_eval/tasks/openbookqa/README.md b/lm-evaluation-harness/lm_eval/tasks/openbookqa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..34849ac628176dc9fe48bf6239c77a494b97ac3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/openbookqa/README.md @@ -0,0 +1,54 @@ +# OpenBookQA + +### Paper + +Title: `Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering` + +Abstract: https://arxiv.org/abs/1809.02789 + +OpenBookQA is a question-answering dataset modeled after open book exams for +assessing human understanding of a subject. It consists of 5,957 multiple-choice +elementary-level science questions (4,957 train, 500 dev, 500 test), which probe +the understanding of a small “book” of 1,326 core science facts and the application +of these facts to novel situations. For training, the dataset includes a mapping +from each question to the core science fact it was designed to probe. Answering +OpenBookQA questions requires additional broad common knowledge, not contained +in the book. The questions, by design, are answered incorrectly by both a retrieval- +based algorithm and a word co-occurrence algorithm. + +Homepage: https://allenai.org/data/open-book-qa + + +### Citation + +``` +@inproceedings{OpenBookQA2018, + title={Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering}, + author={Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal}, + booktitle={EMNLP}, + year={2018} +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet + +#### Tasks + +* `openbookqa` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/openbookqa/openbookqa.yaml b/lm-evaluation-harness/lm_eval/tasks/openbookqa/openbookqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdfcd19635a0d06d6b4190c27d59ce93de0aef80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/openbookqa/openbookqa.yaml @@ -0,0 +1,21 @@ +task: openbookqa +dataset_path: openbookqa +dataset_name: main +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +doc_to_text: question_stem +doc_to_target: "{{choices.label.index(answerKey.lstrip())}}" +doc_to_choice: "{{choices.text}}" +should_decontaminate: true +doc_to_decontamination_query: question_stem +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/README.md b/lm-evaluation-harness/lm_eval/tasks/paloma/README.md new file mode 100644 index 0000000000000000000000000000000000000000..54ad5fefc053797a0b2aed95d8732fdaf2028c08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/README.md @@ -0,0 +1,68 @@ +# Paloma + +### Paper +Title: Paloma: A Benchmark for Evaluating Language Model Fit + +Abstract: https://arxiv.org/abs/2312.10523v1 + +Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. It assesses the performance of various models across 585 distinct domains. + +Homepage: https://allenai.org/olmo + + +### Note + +If you are running the entire `paloma` benchmark (or just `paloma_dolma_100_programing_languages`) with a HuggingFace model, make sure to pass `logits_cache=False` to `--model_args`, for example: +``` +lm_eval --model hf --model_args pretrained=EleutherAI/pythia-160m,logits_cache=False --tasks paloma +``` + + +### Citation +``` +@article{paloma, + title={{Paloma}: A Benchmark for Evaluating Language Model Fit}, + author={Magnusson, Ian and Bhagia, Akshita and Hofmann, Valentin and Soldaini, Luca and Harsh Jha, Ananya and Tafjord, Oyvind and Schwenk,Dustin and Walsh, Evan Pete and Elazar, Yanai and Lo, Kyle and Groenveld,Dirk and Beltagy,Iz and Hajishirz,Hanneneh and Smith, Noah A. and Richardson,Kyle and Dodge,Jesse}, + journal={technical report}, + year={2023}, + url={https://paloma.allen.ai/} +} +``` + +### Groups and Tasks + +#### Groups + +* `paloma` + +#### Tasks + +* `paloma_4chan_meta_sep` +* `paloma_c4_100_domains` +* `paloma_c4_en` +* `paloma_dolma_100_programing_languages` +* `paloma_dolma_100_subreddits` +* `paloma_dolma-v1_5` +* `paloma_falcon-refinedweb` +* `paloma_gab` +* `paloma_m2d2_s2orc_unsplit` +* `paloma_m2d2_wikipedia_unsplit` +* `paloma_manosphere_meta_sep` +* `paloma_mc4` +* `paloma_ptb` +* `paloma_redpajama` +* `paloma_twitterAAE_HELM_fixed` +* `paloma_wikitext_103` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/_paloma_template b/lm-evaluation-harness/lm_eval/tasks/paloma/_paloma_template new file mode 100644 index 0000000000000000000000000000000000000000..3f88e81d3048aacaf04f74b22f9507af276cf5f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/_paloma_template @@ -0,0 +1,22 @@ +tag: + - paloma +dataset_path: allenai/paloma +output_type: loglikelihood_rolling +validation_split: val +test_split: test +doc_to_text: "" +doc_to_target: !function paloma_utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: !function paloma_utils.doc_to_target +metric_list: + - metric: word_perplexity + aggregation: weighted_perplexity + higher_is_better: false + - metric: byte_perplexity + aggregation: weighted_perplexity + higher_is_better: false + - metric: bits_per_byte + aggregation: bits_per_byte + higher_is_better: false +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_4chan_meta_sep.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_4chan_meta_sep.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f3c67d9f0d3b390e6bd7ce388f99b61f10f6af0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_4chan_meta_sep.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_4chan_meta_sep +task_alias: 4chan +dataset_name: 4chan_meta_sep diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_c4_100_domains.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_c4_100_domains.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbbde2d4d130f2a7d554d5a06d2e76713465e0cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_c4_100_domains.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_c4_100_domains +task_alias: C4 100 Domains +dataset_name: c4_100_domains diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_c4_en.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_c4_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8681dfb6c6ddf7b0ef710ebbb94c2a541d02f11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_c4_en.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_c4_en +task_alias: C4 +dataset_name: c4_en diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma-v1_5.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma-v1_5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f816e107af1a1502256ae3516aba9c2116dcef68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma-v1_5.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_dolma-v1_5 +task_alias: Dolma V1.5 +dataset_name: dolma-v1_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma_100_programing_languages.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma_100_programing_languages.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd264dd323742d81ad104efc0528d60e45b3bfe1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma_100_programing_languages.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_dolma_100_programing_languages +task_alias: 100 PLs +dataset_name: dolma_100_programing_languages diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma_100_subreddits.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma_100_subreddits.yaml new file mode 100644 index 0000000000000000000000000000000000000000..958812c50cb1b991a8314d2d0f22bd7fbce43a39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_dolma_100_subreddits.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_dolma_100_subreddits +task_alias: 100 Subreddits +dataset_name: dolma_100_subreddits diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_gab.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_gab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ecbdc9d2cf9e573c7b644169aa9b3e0af769634 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_gab.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_gab +task_alias: Gab +dataset_name: gab diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_manosphere_meta_sep.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_manosphere_meta_sep.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93fa2bf0c74b22a5f206f12b01c44ca83a6cbecd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_manosphere_meta_sep.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_manosphere_meta_sep +task_alias: Manosphere +dataset_name: manosphere_meta_sep diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_mc4.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_mc4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4cffc874969470677f244225041e32d63f7dbd79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_mc4.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_mc4 +task_alias: mC4 +dataset_name: mc4 diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_ptb.yaml b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_ptb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0f762c3b74607d7e8d98ccd8a5f3c25d36d8dc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_ptb.yaml @@ -0,0 +1,4 @@ +include: _paloma_template +task: paloma_ptb +task_alias: PTB +dataset_name: ptb diff --git a/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_utils.py b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..adf88fd517d64dbd422cf1f2e8f3ddfa5f5b58dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_utils.py @@ -0,0 +1,2 @@ +def doc_to_target(doc): + return str(doc["text"])