diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2792d5675e1b2601e7fc81fb1a8e0e04e34de13 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af1bf60bb1262483e7d5cbc5a36eb4d797af1cae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f7eaff72d529792c275fe9f0e3a65354b45ebcb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f56395b1eb83cbd974f00bfa11c92e2818ce528 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e388aed857a3ffaf249c96f390c96c81babd27a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..741584c5f94550e2138ee4b5e5eb67eb99b165a4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c376967a853698f06dbd140499949cf6126116a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c71ada9b6688bd285676e88bfaa12d5699c29e3a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b5f32678e18d0cc8c8c61a432d133aa9911f568 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb259ac24b7667ad6b70fe401549643eccee60d1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4ab308b3026bc5311c35eb17b06a52bd11d43d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68180e5d61b4ae31460b64b55f6e775685a59edb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e727ad09160599f2a890e8d17817c0e25dbeb4e9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ff9dd0ba2cfc698134e7526f5298c6aa536675d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..668991cfd3febdc9231d51b7b0caefd2e71ba652 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1df9a5533e6ae2f61ae7c971d1cd9bf3faf22e6d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..515a40f0c063d440b1bb230ceaebfae8c1938fde --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24caceac8cda0e4254c49eaf94a0719b45277c21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5aee4b294c9c1cf61ee92a2baba54813125dd9a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..377812084e66e1995bef2eb16c0977d75a9f2497 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4365730e10f15fad4ba13aa3b12e154754619e4c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1fc86e2e3e351a0ba790dab33da17b29a7bcf37 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ar_stem_tasks +task: global_mmlu_full_ar_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dc7c8c0095ba06391579b7d78671b7d17b1f4e4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..371fb521c98099c63853f5ad3db32d3a0778f07e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c080b64554ca8788cecb1e6096f6f7a327f6e307 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d593ecbf058204cfa0b9474229436578198110d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4021a93e6568f22215550ba825b1722d9fa3c70b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f09edd006bf0097e1c3f5b0ac57ebfe88f963d5e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d8577cb64ee688255d478e8474395eb4d685a56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..733b77ce95a7e8848e02907baf001a80fdd85f04 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d1bf14144e43f01a45ee468ed45f62173fbb0ae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45b07299c36bac759348bfecf2827bd7b10ed3bf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e33b5836db631fcef77d2420ff8b29761300fff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4cd0a17a0e78d90ae7fe90164ee7b2cbdb01d5ee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f035162d0eac7048eddb22193ec64e8e596744da --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d4dd34ff212178d2ea1414d219262edc5e646c0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2245b5273190a42b740086fe9cf5f19c8db8ebd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd920305786770287f7cade75fbf59e784e35295 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f38b85554507d61b58b4e51e3e413deaa4761e6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3be1f8f9527fe18c267a0bd5a7c10d8506b37c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c7f01a5039162145c535d6fcf59e4af325a6402 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ar/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_bn_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_bn_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..aac80768cc9434d83754469811be77bac22a8e31 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_bn_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: bn +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..135b4bf5dd1532e85a35144d66ca20a82f8be4a1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_bn +task: + - global_mmlu_full_bn_stem + - global_mmlu_full_bn_other + - global_mmlu_full_bn_social_sciences + - global_mmlu_full_bn_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acd1ab011f85de884afa8b96ad3f740c7ab87e9b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_bn_humanities +task: + - global_mmlu_full_bn_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2160298f92f739c9d8c088a60d762d539bba53a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_bn_other +task: + - global_mmlu_full_bn_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c359b3598af4e8a31bba5682e1fd523e15aa1f76 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_bn_social_sciences +task: + - global_mmlu_full_bn_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c78c4ceaef8e79a58a6d2be5e93d5ded1164e79 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_bn_stem +task: + - global_mmlu_full_bn_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bb7bb61bf8792d446119597462f346bdf9cac6c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d49070f11ddb642696421f5faa899c2b6949618a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e6dbc971aea6fb0fde65c921a678db580565f8a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c45a0e2d3f11c1a3d07dc8254330fdf6947c4fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97e17570b5961b2b81d0f05f8507f2348cb6fea9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bf0b34cb28a00971ce8f142285673c0d2cf6e69 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb5a260051f497f616a5f527763383aeaaebecfd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecd60e54e11955342f376dcb9a7a6e12497224d1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fb69d57fdf680da1fe6a0bc73a1d6d6194032a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..442045f98fb0decceebcf27328aa758cb2ec7e47 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6849ffbb246bf3702139343745fbeded3b84752c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..184097f8b8f66eb73cf75b4e88d1467ca537a195 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dc8a2c208e56fcaf8e02ccd1c111883eea81cd8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..941f635570bf80b4ea9fb11a899379da666391c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5918b08aacf3b9697ca4c44aa943aac8dfcceb71 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f7d1f105662952141821e1a9cb4670dae01fbcb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b54c80db067150628546f7d3e70d9331301d2d1f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..371d61cd67a3803951877a0cf2581875757b01d2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f2c8731154480d59d3515812f612ec23ec06166 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0022c8242c23acbc206a58b30dc25d28eff323fb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ed6c6c19f0d6cc3035dd20774540ecc072ed34 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9118a11cf28bf871836cc79fdeb15ef545afdd1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7fa3c1ba62101e8566895a0824346db482d82dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..067ec0eada3fb02e48fc38578b884a50222f1bd3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12c775ab47ed75ff643f9f22bffb423ff31823a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82809b152ad80a915941562d122f029940e75adb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a14eb7031a8cbefd3450e4fa7f56947a83caa6a1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a84f85fd0b711e16dd0781d9f0943b6ac19011cb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b10c59da49cd1fb9a454f684fdd656ca9b2bbf3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4231ea265ecd44cfde35f33d9ea49d3777b716b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28dbddf78eaf47e21ddeea2f69eca240d4139343 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc8aec7daaace65e6d33c5a2886ba4aad7f8df67 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16a3c204aada93775badb2bbdfa1ee1e2d141d56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a25244a55d8f43f340159d8818968f1234d97634 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b47f516de9364292137cc4aa6b3b39c8de477a14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08ab51e2ffc22baa8d76108895f6807f2a24c331 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0885a1a99d29cb8deac7947471e93dd332b7c858 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0eb0997def69249d9a729aac47557a24fb9dd08 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d006b41160adb4e9a642a188bd85cfe8bf7f4332 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520f9469968e8c004713e97898fd4b47b88f4d88 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88caa977d9c6624ccd1fb5a4a15f4adee9f4321f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ce31f7b1a67868c611d471753988cbca407ed29 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..444032164d0e999a98f62e5e7567206ef493b0fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5f59e15c36e276eb4d9bf059a4da73b0f20412f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..422bba55011a3391e3bdde0c1c683f2af9b70ccf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62af532bf13cb63b6a823da8770f34a3aa68c119 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc49d36c3c3fbeea0d054289f0b79039cbab2f48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf72a6a4232c0f194684414b2b25500814a98fc3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f49fb142627d2cc38d1fd3d86cda06e0566f9c92 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c53d77a7b3206aebe3007695b558e74881ef1a1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a50c5cbf13d6ab1a9e81fb5d6e16c2518bff142e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00e2742a44a3689a6d0f8641cef8ad01a8407a25 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a0e7612c2c5f55ee5dd63d3be71ffa2e106a5c9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e88203197be6601666c9b9d61d60724adb2e4652 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42be796a3cd49e2f3073a5d6a8ffae2fca96b8f4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3959f00620ba302b1ea7ae28622c4d551219f445 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15ee9efc24fb3ee74a6c02d7124198fa932ed2c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/bn/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_cs_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_cs_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..282ed121e0d5e92a638b188b801cb8ab1d6040bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_cs_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: cs +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..419d5de4fb7ed778fa46654a9d1b2a4b40496ab8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_cs +task: + - global_mmlu_full_cs_stem + - global_mmlu_full_cs_other + - global_mmlu_full_cs_social_sciences + - global_mmlu_full_cs_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4b4aff39d367d6e804a7ca9fc7f3ae79f934fa8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_humanities +task: + - global_mmlu_full_cs_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..302912e4507c68d36d43bc703f152f9cb3ff777e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_other +task: + - global_mmlu_full_cs_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3fed76e40b7c0ac6013ac2ca8b1153cfb227a0e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_social_sciences +task: + - global_mmlu_full_cs_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..898bb09272be41a95941512ec52e956dbd8e0704 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_stem +task: + - global_mmlu_full_cs_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40431ec993e43480a8fe8e7a55802f1cfe1a05cb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97d7354d45dcaf5e2441d15d59bd7a7afb25427e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b5a5f993429701cbefe8640d2d7f486b0772c3b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6db79c5222496823584ccd4cf6c5732a75b0483d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a17c60517c0cc77e25f809ee17cb09412143bb4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c6597b4c25121cfcd45026214c971dbeef3823a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..713af5c32b017b34a237d622f8b6e17c0c59b4a2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd619d13f01bab3dcb37447b7cfa79c15b767f35 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e09563f927692fbc7eccb6d8083b61045336d070 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7b868c922fafa4d58fb08fa36ffc4ee496d38da --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e98df33917d7b57b511c631673e4849e9565e61e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7256ad679259dc505fd548cf13d0139af2348bc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bd6449894e2186231cd2454a40ee4209e213fd5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c954d3202d6c163fb00bba481c848b1435b1c433 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f80e8ac0e32add90f49363161fe668dcefcf9b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfbc2c9d6681a1b51fb21c9892194fb6875e0f84 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c2ec8bdc2ab4cdee00c02b079ecad8617a83a9d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6302b417e70c09a831aa7ae1124ddf559fca01fa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b69e9ac39f47606b7b9b793b96e27111fbf3576b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67f53cf54a48ef42b67811ed9ce7d43352896c3d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0be19221b719c1f6d9885516a3e1c2972f2c9083 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fa264c36aba001043ba9b65c1beceb1ac085e4c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f903c2f55a8fab7ebbda63078de4d8573958be --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bde4d695c6abd7d46a9e809a33f3d6e1e910c8f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb5068edb22afd447822a25b8308bfbfe91dfa9c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87cb3e577052f3e3e9d9bd8565189a28a622a297 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33c2e18c8bd54a4d76b6c4308f84cda9d2cebcec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ed095bf4193d57d69d450bcf3f494ffdbef90f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59b623053c0a76edb790904a3a2dc479abae7c21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a18ee25b4a783e7aa04ab73ed35c2ec83347f85 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8d0a27165ac72de8e9ed3fa0d0372f68201b6ab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07012306012a655b2d1d07ac1ac68ae43eefc49b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f5c7c392b23058900e0475e4fbc16871ef89ee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61d405c7314c6ea110d0c1c60ae393158c14912a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..509ebee46050eb6deff0af1fce8d5c5738714f80 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0e27957b80c1ae9fec8b327d248f57d3c09b257 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85010f3c6bd33f9d17f972c3f4380993a4ba297f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32aaa1a65eb75cc87c6fc441c315a0486fbb3f4f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e1a3a7cbe4399703e4763f3943556cc8c92aa21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..239e3c0c486f0ba846b4985b5db894ae9a121cb8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c76fee782daa02ca303148a5f027925215dcfa4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4be6207a2fcfc55aee287473478f0f8ce247d7d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b263f67ec701cb822a819fa94c55a56bce2427f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6532a43ed0678c86ae3e70edb12c5f30b7d5bcba --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f04fbcd1d67b57abe5b074acad4becc413143bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f5093f9228994c8a46eba42d80afab28496b6d2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8f5f5a5d8fd3e0384ecfde3712f1161da67d4db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bccb71b239cefa46527a5ab21d28861888fcf3a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff50f50c51be5d1e39a394fccd528ba68cd087e6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b82937902b106bc052c782d0605feafeac3eca8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e41edb29eadcb1f685146e92c8ae09bfba5b83cd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8fb512db97343e089c71791174accf1c2e69086 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64ec0b3f0f14088b0e567dd28d323d784f66dba7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18214f7c299fb79df76026c749b02e03372fce39 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac42b097003ab1ea266739aa2fc4e1f123c4d9ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_cs_social_sciences_tasks +task: global_mmlu_full_cs_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a51b8aefae0bbcc53d665bce15789a3802612279 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf9af3e9c9a9ef635048b1658b74d442459a9ddd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_cs_humanities_tasks +task: global_mmlu_full_cs_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/cs/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_de_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_de_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..49fb35d294422882475037a2e0e1ff00ec72dc0e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_de_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: de +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5217599a6771cf331c36f2b1ff2de4aa2f9a191b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_de +task: + - global_mmlu_full_de_stem + - global_mmlu_full_de_other + - global_mmlu_full_de_social_sciences + - global_mmlu_full_de_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df571c67e9ee2c616f56e94d48ffd90ce6a1f61e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_de_humanities +task: + - global_mmlu_full_de_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfff864e482634d6f7347f8806df1ef0e56b8476 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_de_other +task: + - global_mmlu_full_de_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cf304a2c7a957c4719abc08367c9f4e4459976d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_de_social_sciences +task: + - global_mmlu_full_de_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d1aa5a161df37d4c83788684982b220d43146f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_de_stem +task: + - global_mmlu_full_de_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07cd235668a9f25450ce1ffd57753185b0ef5f3a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9deb16a6e6df64672019dc30b9b5238f33e44e7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a743d45543c0e91edb6c6989fc0fbd8a657ccd2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37bf9d454e5622196a152629413be4a0f04d4a7b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5ad878a8caab668fb2f404e5befb700f7f66ca9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..200f9239f0fca2f88c957ab8f61e7aebce68c52e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bbc4d463a43602d4497b86cfe15b3a899469988 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac903e3a5d8e74df8047a0078d87fe26911ac8d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..616010cad26bcabbb820146fc75497d21f075a20 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9648ce8898b2789733dc93f6c3915578ec805d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3bc689255cc954fc6af728ea932421d85a76dd2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fee01f9e1b1cd19894f68521170d71b9708164dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..201c17d7c6f5feed59b24faa309cc8ed3926583e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d902c3c211dcc24584a2526093ad0d8579bd812 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dcb6c48bb13e830773d668e1e5adefc3fe59738 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1ca41ce60dbc58ba38cf891556c023063798bb0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e16729e7559e1de2b5f82a5b876937d4fb9c01a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7b092892247185fd237111cfeae8f27fbce92b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ad59551af9d4b396b8ff7bdbd4f64e5ed22e2c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c0fbd556d5da09d4ce4f5e33a303dfb308723fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aea5ada3049995d931389e47d1a34d275ddba09 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97293b4983b298cdb8dd160562b7368039e26646 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d26a65d9707500d6978a0e7f5937180e3f86663f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6ec78e696bf139ecb8c450469fd7e023a1b02a1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53489d855cb541ded15bd3bd1a0fd0c880c0732f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44a5666f724eec1420be7b1c68db1379b81443dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b911297cbdce278c3a81851af21bf873b539dde --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d17d047ed69532738d09b035d3aa1a1d745f413 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae7680027480500cb143703e9fafbd1dd08429e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c272287ef87e5ebd313692a32b521e90f3bd326 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c1eff81145c2e12fe209bcadf93e453c8e753a1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11f804a622eb4ca1b2af58a70e8d1351985903c8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d5b4d775aaef844a5d7b23f90a86bd0cc13056d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3f09c7fb5e7b69935886c254d00a5a44bbfe6be --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34bb5918578631bc0db9656c65155761e59dacc9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..585e99b3c0f4d8ca5b6c54e498e71dbf2ce22e25 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd09d6ad16f2c86c148e1ac1b64ee796dbd74e98 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfe82a9b63b870d19e073efed4dc2caf37775ff8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7304da38e40fbe1099e2c6e23fab8a27708722b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2143e4f17f3911ed57b255d0cf1931c100e318e6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..015498683ec50ef536cf225222d0a0bc337ae3e0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c8bd533b6a3d71e7c2f591ae7bd7e9c2f745787 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f03361ae6421fe80f2d68ab547baf110533c15a2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a36519a7349840359de7c4af80429c2c87c35c37 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799065cb7528388aae0e644449c27d69779a68b6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5f0372b555f9801f1f37a84449ca495de7d45a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2145e87d4ba25f03885318d52e3ffb7633a600a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ad55e9773a667c5cd11cb6e580133bb230895af --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f4e338fa3466591c4972094a9aa06cb3a0adc3c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a1214a6cb7b854348aac953bc7f5c562bee2d0a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2d49ec8c903956eadb5d1e50b203fcfc3dcd90c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b7d23a83fa84f1b3c559699d07c40c3a46d3da7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1a3b22ee55e66b6cc225a921548f5fb8698d64a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fefef9d016d443c389bdc7224aef26c90e245ed9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35394ab5bb0b9908aa4f6701ff04640e3775ceed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0f2f59570dfae2815350009e617dfb711918192 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a43e6fe07f465b8dfac892586f15029edcf76b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/de/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_el_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_el_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a53c91e33bb6c0d4d78e61ba120190a153c9aa8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_el_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: el +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d44d8c8b9c3dc919ec9e0eca9ee0ccfac222748 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_el +task: + - global_mmlu_full_el_stem + - global_mmlu_full_el_other + - global_mmlu_full_el_social_sciences + - global_mmlu_full_el_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f07f2b5243f29dac15f46ed4ebc0536b6624bb41 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_humanities +task: + - global_mmlu_full_el_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..938292f095a0d6e1af6e085f75705d839d74f751 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_other +task: + - global_mmlu_full_el_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e72e1e9c89837b812db66b62c44f272d3ca0dfca --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_social_sciences +task: + - global_mmlu_full_el_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2123be0887d6e1cfad81efa8ca37923f799d47d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_stem +task: + - global_mmlu_full_el_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc56c0696f2b21c98c212f8b7443a6b7d2c601dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b2e0e7ff4dea18826a1298914ae8387cb75a06b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7faf7389ae5a74efa0b40aa954def2329302fd97 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e8b5bb43664ce26239af40025cfa6d8bc7125e4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ade42184fcf5b554bfebe8882748dfb2ca3600 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf3aa362cf68b60c48dac28af106b3e1d66fbddf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd8e1dacb6fa274f19ae106fe42ad081cb0adaec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1ea08597254464f7c12286f5f6e4bd8557fc6f8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ec055b7fde1aa3efa5009d2397ffc2380f6ada2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b16b545b029d469fc6b000ede0162d73218bc78c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4630f7f82d1dbbdb33cf2470d82522c50d22fbf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a40228ea3e0fce6cc9afbdaec8fd6629267a69ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7baf6e602412356ab271bc7ee9388f2673961bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48e590213417ddbf4fcd72df0ce04b000e0e9081 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..294c3c5dbd6e7a5fffb62ea28de6045d2f70fbc7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4373d82e362dc56edd734fc5bbbeccf57e76f61b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81799a17c9c3c26be1704fb1109bb1eeaee706b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6317eeec7ca9aa662a8ef0fc3e75671104438a5d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa5958aa1817fbf1b1cac1ca0da985d61a711b80 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38053add37dc82b7e97328c29bf2a700c5ae505a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fe73214dc389598f34506130a94c6d0c13e50fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8beb1e3f525d454b7e5f25ed50cca3d44c1f0bb9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22c083214dc41c9b6d13c0bb0a019e1f11f8d52c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3b330411733cd8c1b96cfddc296026f1c4b538a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63ba6a05449b86d16be5819086fbbc425244cb56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6ff6e2c0938a513f93f3ace33bfb69c7b401457 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e4deeebc53868dc12caffe10c57fe548d9e687f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb8757033d499b8154d8bce5cdf49467875ba9c9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e82d1b5304bfbcbc0c6e48e328820b657529fec3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0003184cc21cdce0e7ff70a04114000a916d1172 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5e0a3675aa48def57616e76c64f2fe9b295cb77 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac460ea8bfdbe8ec13fb74426d7c8c74eed3ad7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a40e04f2e8ce255e1cb415ba1c6cabb5d897760 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de5075bcfdb0d4959b9a4638b81c351fd95d54a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fb93f2c8eb0b60a978a88377a0cc06786e7eae4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..624e040add336046e3a7748e1f93eb4351252e92 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e5bdb4f33295bbb15d6855cfb2ff3fdae8aaea2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..180f3b25e44a21d081d8efd4bbaf9754766e0841 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40487fb17fea9b6275277f2788e7655b13a1c837 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..781d41708e306028bba373fdac5cba326530ea49 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ca0114644d6becab5ae8c56fcb3b11dd0baf062 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66114367bec5d22978fa3ed8b91d7e3f0ca05a24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c553ab7be81a621d20b9a71e7a0d35dc9d5466e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14a79a4a3f8320454fc476dddb51c9a0eedf7724 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..595daa39f90fc3c093281f737c172c6dd41d0310 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25b121b69039acab1bf15344f21fadf4e86dc97f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5938a174e70f3e197837162419008d38e5b0770f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..002b02aa9bb5b61863e6d41038b5d7be3ed3fb34 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b457038023c47923bc270de3e1bd469f2f80c6a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a31d4e3bbc609c7ee226bde72f9c60b6d7a45d1e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e04807999538b40792cf5ecc0671d07554e3f40 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..264799d613ebb1d255640963f6f32f90a0f4794f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19ffae47ed23ff8d4e9879ace2773f9ff9b406da --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f57d3e0aa88d9d58898ebb5f9f976e3500c9296d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14c76440784482cca0e75b59691f9597234aaf33 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e444358991bf517e6b82cbcb10ceccec819d6b4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60f8e52e31213ae6bf0590b003f6f2895f0f4a82 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/el/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_en_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_en_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b1430454c3c281e69245b5d14309d88133e40f0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_en_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: en +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c82b6465ee35149f8052ca27a2adfa2e426f3e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_en +task: + - global_mmlu_full_en_stem + - global_mmlu_full_en_other + - global_mmlu_full_en_social_sciences + - global_mmlu_full_en_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4455fbcfecff47197958935d8662b8c3d432c9dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_humanities +task: + - global_mmlu_full_en_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cca60e529e6cd3363cd389128e5509748ef1cb1a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_other +task: + - global_mmlu_full_en_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..becac7a59ecc3f807f1e76db06290501ed6d1ce6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_social_sciences +task: + - global_mmlu_full_en_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71aac0612bb7b92be5825adc32439426280ba7b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_stem +task: + - global_mmlu_full_en_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d7a5ed80c8278cc185e5fa4d4c32b908df44790 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2267ad88190006d76f6441bc0fe57783ee80210 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6999c30f68b75d1cdb489c6da00d1b220adc6332 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56a6e490ebbb4ca20413e5109f0ffaef83bf05b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60425fadc666754f055eed2a79d47ee69a5dbdc0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b5f2f8c5a7aa94bf280494396a03c7f2bf93516 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e2ab91f91c4b75ff5e57da7bb87844059698a98 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9abf38dbdacd0678c1dda043535b0cb7cd5c7076 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da6199fb39a769afd96a339b969913f604d1628 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c568f36b774e9b77efbe6b075861768dc91d5033 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac0440196d5682318c820724242d9b0f058a1ee9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be47dbde08f8701c2f9b46b226f89ce40ac887bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86180924fda604180d7d0897fd754e7119fcef6c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a75d329f4d300f849897c3742e4077e6c9296f70 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2568993f2ba8edfafe179a4a3a2dc43f6f0e1a8f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..622a99f83a31eff2e70f33b6290541cad71fbdfe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..109ca44ae2622cf99b48fecbcee4ad304fef5c93 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39daa506ba791f7c47ef8558c16b45929c6a0c31 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..063392eb2b559fbd04c5bcfa347c1eefb3d9f39a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..452e9445db6dbe0950d53cd67b209296985a0228 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..baf4313624ae8743456d9594acd35e96897792db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fceda5c29f63e86d962e63133f56e0fa9d78c7e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fbb9ade5831c9a9f8aae7b5d9c94623127b2b2a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73ca9087a51f05b1134554cae72b8ae230cafe7a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b9ca7a94b74c833e9b25927068cb188d1c1304d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9be50ad2621ad8cbc464c37d0cc6d2b1bbb95cb8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d93285cb33c1cd22702bd0a7204d68aa1ec85582 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f74c609fc69e41a376e6ae194227247151c87a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..365762ba0f521d0c68a456b922abdd43e37fac31 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6ca42ad2bd3a8931e6b7aab00608734d59b17f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f20a4ddcb28ca9d05205c5b7c54bc03000a7b2f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0fce40357b0ec24864631d552409460b0d2f8d1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35320a854e7b693af4994497d30ecc70177d48d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86096c5d0f953ff5217381e225d60582a7b2f857 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a41e9fcf854fceb9f7e3716b0f238b27a506274 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa34c443322e3c0f916ffdadba01611f72fce916 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c105b48904692fcd3847931f383d280c0a2cf9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35f496c17ff631774d505d45f703328273667dc3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8499d9fab2412c309153fe9242eee1c6d8a1005 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05f8f0ec9d19841e37416b5343134df1a1db4c7a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f272510f57df4e0741d6dd119df486495322724 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a72fad223f99cbc4199457fb83ed0de2eef5afa8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2504abeb6733136755801cb30af0fdf1afec59ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ae4c37a57d082a982c72eeb74515ef88af811e5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5364f69d43ca4867795ee3b1b7c70371adcbaee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e68d7e7223b19e2d07bd3b9d053630069ca7b19 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72e93368a798333b1e43ad1734852f5123a3ea85 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdb66ead8c7fb76e51cc125d10cb28fdae41e53f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67120278088749f0a2ac32deda4580d6eab344b5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffbcb29b38598cd3b796487f8b3f20175c303662 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1abea59b5d1bcb4c784e50f2eaf10eff3704c4d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9df4f49189f72f29dc8da457b85ccfd792824f1e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..addb6934d82bc1c332e1439f55dfd04ad176ea56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a198cb84cbb896d6d0d2217dcafdb62d500939f3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..047b61e00268e8b931c64b6de23f5ad1a77b24b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb74fefdd32dab13a2b4d1e1fe7907e491a6377e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c453bf7480f8ccd4aeca4fbd58243151872927f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/en/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_es_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_es_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4e1bb0834ea99e6908285121ec982c74e3d0473 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_es_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: es +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13d2eccf3a4deaac71e24bd06c35efb2b4d8061a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_es +task: + - global_mmlu_full_es_stem + - global_mmlu_full_es_other + - global_mmlu_full_es_social_sciences + - global_mmlu_full_es_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bda6944e1b855deb5296a730cfdd9836f8331ed0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_es_humanities +task: + - global_mmlu_full_es_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..610366ef7d1fe1e00c7ce77583cfccb0869c8cad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_es_other +task: + - global_mmlu_full_es_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0094869035f3e24f390107ebe9dce164594344fd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_es_social_sciences +task: + - global_mmlu_full_es_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..483a8fd6fd14575ab2bc2a87ae4767d6aa194480 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_es_stem +task: + - global_mmlu_full_es_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02fb72001e3fc6eea4e049b1d8504a680c51f97a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40f05e7beda0d8f224af0780790cee21fe6a9eb5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb688c13ca5c63973dce474461174eee2cd464fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aab858f1d2e638059b0d34a48ef37552164dc79e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3483f8d08bc1e764a357d1afd62637f62e4059e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36658ab6c40881f48980870f396ca8c32a1f5718 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47a4744497e564e5d3a199b2b9e9d2ee45c8a75e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4154324e4f77a2143aacbf7d76a417a05ce4a303 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85bc62614565b729aee3a7c875cfc5da5980f7f2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40e8d1291ec14e09a25bad922a56ca751c809415 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ebc5e950300544903aab2d4c4cca3b15a60ee24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b586eb2b80f65b2d18c5d4cde05ff9f95849eebe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4186cec69574e7a8aa916c31af316410a052ae10 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d61c8f9dc39a02b7fdc697395e8a049ac317281 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a454d79664c52ab09cb275eb45808f2d4dc9d97 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..772436e6b533d8dab172388e5b0bc87584fa203e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da6223fe38232995ac5409e98993c2d86faa565c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae3b5912bb5e10d4a70ed5c3f73f6d180498093e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79a72140de89f3adeaaaf8d13b2e8502e623faf7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27ba757082d345c4c0973a7039ce340951d89b68 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72ad45057bd8fa98b7f226790d9699943b7254d4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cec9d5f6b4ee527ad0ed01716de92b93c6e9628 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ee91f710cf4372f01de5d0ea60749707d2c8e5f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3f1031993168ec87fdf05bb9523ada6ae771935 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d555129a40a64ff90972be9ac8d8a1cdaa6f67ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1216336182fb78fea17a50a8ec1e70f7ff234e3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c2884439ea7c9f44156c5a29cb0455e0fe04fb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb83ad1e44df9554f4b956a678fa307b4c2e1a14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bcd53e48b13aa2d8b6357809ad142f415ad01e9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..900936eb1b27ad153842e7da9115325a27fb8b2f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d54acd65afd3a0cdac8316f6fc3e4b0ccaa43b9b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a654fe86568bf5c602d7be916450d54d6ad42a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_high_school_world_history