diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1825ec27a5eb16b4229385555d699d33338d7c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_yor diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dc72af6f3dbcc41ab83a2adcf384b952e97a4d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7ac8793976d7722375f01f43f587b74e9654ec2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97cf3cba569f25219a7cdccb2be7a1c6effae0c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..824bb17aa23f3a1a546d369a6d3118272ee05c54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d971c3e5fe6ca65e7568da81b647d2ff8f20696 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..832b51493a0f17996ebca680f7151e38b59168d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml @@ -0,0 +1,27 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7aec16a61ab04415327addd832893ab6e4e531c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5667b3d0624c569bf9e392a67780a80fdf5aee3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a74950cc34121f63d3dd944a310b7656bc2ff894 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27e88a5b7664a515e7d6cc901962f201913670d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63ff988c3aea7178e06d85ca20fea40c87e9dbcc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db1a3ea1cbeff8ddd8c5b2ba49d3d553410c1b06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa110774a6fa86e29253307163e22c82f35a9ac0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3133308b2810a3a61e1037ebe58c02ba0da0a2ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926f91321cfcade46ca52492cba83daa348bc746 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c229de3dd2b40b902e7ef93de0cc06a6da04d8af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87844c49639fa350e52877d967f64d5ea95cf28e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63fa3ffc1c69d89637dca73da5050c840593aa4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ad87afcd99a5c875ccf1f7d2761afdea2a118aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml @@ -0,0 +1,31 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dfc9bd6d46fb8982402dd85531d1d312a8a07b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0878c4e03e985eced95c234fa079c3d92999a982 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1ac19e19b2e855c957e75f1c778366dfbc7e55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fb06d0f1f7dc3c62e7c51a6395b1a79f1759878 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Given the following premise and hypothesis in Amharic, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ae3c21e2100b4b07428f67c23070d51d32761b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the following premise and hypothesis in Hausa, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28696f337841ae738354ae11bf537f07743c49f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the following premise and hypothesis in Igbo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6981da83291b927f5fd5908f62e039227ded3a9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Given the following premise and hypothesis in Kinyarwanda, identify\ + \ if the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1984416f0a828871d5e29ca46bde91c300440feb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Given the following premise and hypothesis in Lingala, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32a7ad2a161818c6d84b9002d6690b95cc86af3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the following premise and hypothesis in Luganda, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3923a80cdc78c59dabb63bb3fe9dda6e8e572a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Given the following premise and hypothesis in Oromo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4e89ec90608180f281765d321c6c1b0220171e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Given the following premise and hypothesis in Sesotho, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3c5243b35264d5b26065a63c9c3babb8c714ec7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the following premise and hypothesis in Swahili, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cf0b08eff851ad0832adad093f6338222f3bc74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given the following premise and hypothesis in Wolof, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ad718c708ddb9c82c3e0ac35c7b805c0c364a64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the French language.\nAnalyze the premise and hypothesis given in French, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e63f93eb89351af076593eff1623123301bbda9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Oromo language.\nAnalyze the premise and hypothesis given in Oromo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py new file mode 100644 index 0000000000000000000000000000000000000000..4aa0db7af761fd7ea8858383b4564130a374f223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py @@ -0,0 +1,5 @@ +from datasets import load_dataset + + +data = load_dataset("HausaNLP/AfriSenti-Twitter", "yor", trust_remote_code=True) +print(data) diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2645b72befa6b0048986f68aa14b4d5ed60027dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Amharic statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbcc88d70447a6257d1acefa94b65336d6b330c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Igbo statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0f96c24a290d33a3c16754f3c0412ac11cb285a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Tigrinya statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_tir_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7061ab63523b3b5cb34ec1bb0d35db11fdefd5d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_kin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f745ebb28e843d400ccba5973de312d841a2592 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_orm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5071134ac6d1645650edef34ee6dde2d3bdabce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_pcm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97b9e4f1a2fc87433bf511e6e511bd9f54284d5e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_swa_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02dfca854e166ca3d96a570e8db126e646fbd5b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tir_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa83c1378d15b8c3cedd8582025aea8ea795bb07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tso +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tso_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4366d1f27902651502a96adfc9c09fb8d82dfde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_twi_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8394706cdfbfb1124d4c25c987901d5f338cd5f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_yor_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py new file mode 100644 index 0000000000000000000000000000000000000000..4515053c4265ba0b6bc9afa9f876d20ef5fc5c2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py @@ -0,0 +1,5 @@ +from datasets import load_dataset + + +data = load_dataset("masakhane/afrisenti", "orm", trust_remote_code=True) +print(data) diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti new file mode 100644 index 0000000000000000000000000000000000000000..5107bb80d5333a462afda9a8efb62a6fd039a733 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti @@ -0,0 +1,39 @@ +tag: + - afrobench_sentiment_tasks + - afrisenti_prompt_5 +dataset_path: masakhane/afrisenti +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: label +doc_to_choice: + - "negative" + - "positive" + - "neutral" +should_decontaminate: true +doc_to_decontamination_query: 'Text: {{tweet}} \nlabel:' +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..783785c031aab91f8840df8e70316ab1d4e24606 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: arq +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Algerian Arabic text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_arq_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e601dc19ddd1bf960ee56e5345bbbcc8c6b84caf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ary +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Moroccan Arabic text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_ary_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ab2adc1aaaf15ce9dced631d20ec2efb428ace5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Hausa text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dba7b17964cdc320243578f9e9249a00caea4a44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Igbo text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16ea6f0c5734b4795acb06b5d854d5cb33b97c05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Kinyarwanda text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_kin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c61ea75ee6b448a7dcf30ecdfcef7328201379ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Oromo text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_orm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6de78061a142a0bd80d69619a5eba4ce3756d616 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Nigerian Pidgin text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_pcm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48b728d5dd38bb55632fd3fb22f37fbcbeb66eea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: por +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Mozambique Portuguese text. For each input, classify the sentiment as positive,\ + \ negative, or neutral. Use the following guidelines: \n\n Positive: The text expresses\ + \ happiness, satisfaction, or optimism. \nNegative: The text conveys disappointment,\ + \ dissatisfaction, or pessimism. \nNeutral: The text is factual, objective, or without\ + \ strong emotional undertones. \n\nIf the text contains both positive and negative\ + \ sentiments, choose the dominant sentiment. For ambiguous or unclear sentiments,\ + \ select the label that best reflects the overall tone. Please provide a single\ + \ classification for each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_por_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fee357ab47703dea57fe9b78a2d46da0f0212874 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Swahili text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47a67e1c9358682be41792e77ef9e85fb1735baf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Tigrinya text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tir_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f5705285e81a7d998dd77b44c9221697aaea435 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tso +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Xithonga text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tso_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0b4fe03da64d5f1940dc9aeebe29de2cea09227 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Twi text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_twi_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b96edb4104e85d01642b2ceca14fc59f2c296ecd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Yoruba text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py new file mode 100644 index 0000000000000000000000000000000000000000..375facffa5030cdde562a9ab4474193a9d45f597 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py @@ -0,0 +1,8 @@ +# data = load_dataset('HausaNLP/AfriSenti-Twitter', 'yor', trust_remote_code=True) +# print(data) + +import torch + + +print(torch.cuda.is_available()) # Should return True +print(torch.cuda.device_count()) diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/README.md new file mode 100644 index 0000000000000000000000000000000000000000..10d46a44e2098e7e8aaadf57dbdfe5eb52156d7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/README.md @@ -0,0 +1,41 @@ +# + +## Paper +Title: `The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants` + +Paper Link: https://aclanthology.org/2023.emnlp-main.862/ + +## Abstract +>Belebele is a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. This dataset enables the evaluation of mono- and multi-lingual models in high-, medium-, and low-resource languages. Each question has four multiple-choice answers and is linked to a short passage from the FLORES-200 dataset. The human annotation procedure was carefully curated to create questions that discriminate between different levels of generalizable language comprehension and is reinforced by extensive quality checks. While all questions directly relate to the passage, the English dataset on its own proves difficult enough to challenge state-of-the-art language models. Being fully parallel, this dataset enables direct comparison of model performance across all languages. Belebele opens up new avenues for evaluating and analyzing the multilingual abilities of language models and NLP systems. + +HomePage: https://github.com/facebookresearch/belebele + +### Citation + +``` +@inproceedings{bandarkar-etal-2024-belebele, + title = "The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants", + author = "Bandarkar, Lucas and + Liang, Davis and + Muller, Benjamin and + Artetxe, Mikel and + Shukla, Satya Narayan and + Husa, Donald and + Goyal, Naman and + Krishnan, Abhinandan and + Zettlemoyer, Luke and + Khabsa, Madian", + editor = "Ku, Lun-Wei and + Martins, Andre and + Srikumar, Vivek", + booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2024", + address = "Bangkok, Thailand", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.acl-long.44/", + doi = "10.18653/v1/2024.acl-long.44", + pages = "749--775", + abstract = "We present Belebele, a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. Significantly expanding the language coverage of natural language understanding (NLU) benchmarks, this dataset enables the evaluation of text models in high-, medium-, and low-resource languages. Each question is based on a short passage from the FLORES-200 dataset and has four multiple-choice answers. The questions were carefully curated to discriminate between models with different levels of general language comprehension. The English dataset on its own proves difficult enough to challenge state-of-the-art language models. Being fully parallel, this dataset enables direct comparison of model performance across all languages. We use this dataset to evaluate the capabilities of multilingual masked language models (MLMs) and large language models (LLMs). We present extensive results and findings, notably that despite significant cross-lingual transfer in English-centric LLMs, much smaller MLMs pretrained on balanced multilingual data still understand far more languages. Overall, Belebele opens up new avenues for evaluating and analyzing the multilingual capabilities of NLP systems." +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/belebele.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/belebele.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c7d3a9dc4450ffbb152abd6eac2655c2cf2199c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/belebele.yaml @@ -0,0 +1,13 @@ +group: belebele +task: + - belebele_prompt_1 + - belebele_prompt_2 + - belebele_prompt_3 + - belebele_prompt_4 + - belebele_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele new file mode 100644 index 0000000000000000000000000000000000000000..51553e0e077d968e1fca29e27783b225ccaf7323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_1 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3a7c2b97208d5ac2ced4bbacc3192a290fa6ea2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_afr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ee55e8eb291cd47abc402f7c4323d2f95caee1e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82f0d5230d4d24a77d381be67a22dd270255fea1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_ary_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38f8c3edc09bd33823c0a221bcfe8a6d7d758d91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_arz_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2bc2d49f78a094ad11711baee3df33923077c89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_bam_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef1f0463d4c1b7c6661c570f00f9beb855cfd534 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f2513826f2808d242ced3d0b7278878bb846649 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_fra_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b24422c03aa1f81b18bab14d8810c916afb8e367 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_fuv_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b999f4a85fab866f0cbc6ccab5bc78ddd7a65bc5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_gaz_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..933e90b50653a19529277c30e0c940130d656528 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa17935c3b331e347ddc13512b46c80e484e24d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad535d498447fde8ee74423a80858d42a85f558c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_kea_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de957a59b3cb665760f8c36c5968ce75c5b4271c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3247f065c8f5137f52af8965f18a2a23e942b64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b2ef7a14b8b1f566b1a55b6e8e02e12b327becb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b667c1d66cff764f6d57cfa6ff698796b608e446 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_luo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c220c7738f369cc8255efc2b96c4d9654a5c33f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_nya_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5c286e8517e6348798dc9c7807b1aec687c9ad1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_plt_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceba2310c5e483727eb5f4386058b666cd49620c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_por_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eec0b1e19fe1bdd53a5add1d16b6b9a89e12a213 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24af555865acd5d2f8c48c5aa2814e401891ccd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_som_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10cde5be846773e764b762157bc528a5acf3fc1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..032b4629b569fc340c12fc701dbc694fd8631d95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_ssw_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c4ae7b786261ef4e7b48bc35c077207a232bb31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b62e848daa5776384c780acce0b270cf88607c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tir_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..147a1c9857be7f9557759765a7a6d5d490564bf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tsn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..869c50153d289b01ee408656701d0fb14dcb679d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tso_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1aed1e5ac7954377f0e594e5e2f3b8260d140c62 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..549560ac1d4cd2181750d6e587f355f743d52cef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70c55eba1c99673595f00b297237391dd767c74c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..257396921dbe6be6b5954170511093b382e9ba9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele new file mode 100644 index 0000000000000000000000000000000000000000..75f673a425116056c7516d0b7e8f54844f0c9716 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_2 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7c7180959511b34bc8582976aa14f1b4327ad1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_afr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8f95cac2a0f27d8a1699215855167bffe3b38a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_amh_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a784902ad61589942794fe4661375154909999 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ary_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a975b485cd567a5f574add79bc5651e58d533219 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_arz_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..814d32b5483290ef00380b30ce0797492f696497 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_bam_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..510f1fbb0d2070f3c547fbe76f3c5a69e4ba31f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..157433727099d0bc545ddb1e821aa80fbb37f3a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_fra_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bf02ff0fa62983d5e4c9c8e63129b1e40da333a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_fuv_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc2b2704e382ea4eaa32276e369ad43b8fa298b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_gaz_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7af70e03fa6ab6f868205f2ad2b5abc4727a0dfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92d895803b9927d2a8288a7b9ff4493f23476c77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f1dcf9117db0c0ef3065975c7bce8abab626ae1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_kea_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e686e477dca84fe5fa1c666c553a0ac1c0efe1b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_kin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..544eb9ddec49e0fbca3c29d56a3108db039d8979 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_lin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe97881b310795720928c4c43241c076297903c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7bdde48a01bd9a08b75961f43bff0438889228c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_luo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..212c0635342943d90922467829854dd051c72298 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_nya_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57e71ac9c186e22ac235f85781ffb6400e41eb11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_plt_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9be02a8e500875dfc2f5b20c1d0f6d3767dce93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_por_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a5ad43a21f7a2bd32933b47e9d5aa10a623cff3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_sna_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d551d1d5c501ddc309b37b8688898359e44ecc3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_som_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18780c9017658355fc160320784f848867bf03cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_sot_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6bd0a69ac685b18d7d03eaa2deeb120913904ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ssw_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bdfd132cd8ac856e157d2dd502f7f030e95c0c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fba28cba714edf37a5169da7ba427543b1dd8e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tir_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..337e08ceab543cad55a4ca0206bbf70590422642 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tsn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a0e24e4a4118e26150288d0f6842ec2baeec12a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tso_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..439148106cf268746b7e41695a752a9ba8422a6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_wol_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2077614892f77249687bf7b8889a211212325e83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9684f05db11c39ea1e000f1b31ce04b3693cc01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c81180e17c7f71799e36536a4abcdb707ef7652d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_zul_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele new file mode 100644 index 0000000000000000000000000000000000000000..a27ea5fb3a06cf7d949c6bc464c721a697e8d981 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_3 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c296cc93f26c6bb91bb7a844b22c8827ebc57fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_afr_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e6f2fd77b913715ca3d24c8cf209a46dca1398b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..203bf1c9239b45803f91c601047567aaa01c8ed9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ary_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97f13672f57f52613728a862805273451e78b5c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_arz_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b5d3415a3008831a38f2f6e3abb09eb79ba072e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_bam_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceb5270ec60454b3322a5058307bfce92caa3f8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..affc5d12fd30c95380804fdeb538a50fd6bcf582 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_fra_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ff7bfdad8e19ae4d4f1bc3a94e6158721243b88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_fuv_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c067e7c1972edd8109312d4a313fa81d62f3effc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_gaz_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..689724b4f8593af87018cef13c7c596868df7c2b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5eaacad2b56cae67a575640a79df06e87624156 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c24b2ae7a28c976c7d3980ad86b61148aa0d1635 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_kea_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae0a821fc67dd16965c9e46dc0f8724c508dc7bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_kin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93e1a5b5438d97012207f12d9bc88a639252644f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..724947d41b746ee2507c1340003dbda425c89ddb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21b4b8f730f3f24abaf93b0a71badee8f6c16819 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_luo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db045f723e91ed1e04317ad29ff7c7ee5bafa274 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_nya_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..946e417946c1c066f2c552d42e266fac048bf795 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_plt_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72ca651b8c251e1dfd304c9e6312b8230c2e2b1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_por_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f5d810ac1ec1bca42f68bf73da7a9ba10a2315a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_sna_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d3a7c4e46094d0eaa70553ecf1c6ff95b557379 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_som_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3db32d81ad3f709a2889da57042411ce00c8f294 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_sot_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..888ecf8423b80a3ac57980a91658c0f0c6a83254 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ssw_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec8127aae7cb28ddcfae818b80c6d58bb71c70fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab3545445999b415cc5a81864672db07f4cf548d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tir_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..019a95fe9a49e4c3398d01d84af7fcfc72cf1214 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tsn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcc97c4f09bc6abc5129d37ef2dabbbbcd6b71b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tso_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20af7b3c57d384f4f63461f47240d9e4cadb91e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_wol_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a205da905597abd079e2a8131b37e63c2d1a13f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_xho_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdcbb8c244275b55670b542247d8db79c6b3bf54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da1ef4239b68c83c5ceaa1988a85329275db97fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_zul_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele new file mode 100644 index 0000000000000000000000000000000000000000..cc28101b1072e9ef48202c143592df0ff2f8286b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_4 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..325cb85f391fb709081167cb70ea97d5664d2c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_afr_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02eb0683fb6fdd1a603f3c7d864f58a3ada9c458 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_amh_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c7899d23cf3bb34c56bb5c1b866397deb6d96be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ary_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5acc3222b7bae84c471027e21afb2b6930ac7848 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_arz_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..466dddff4989fe2ba2e26959fd3f47d40ebba425 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_bam_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21dfa3ea83d46444712c98657b68cd2d4416ba1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_eng_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7fea6f1ba3e5792d2ebe755111c5d924ad07999 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_fra_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77fa7798b1b079a65146976e0ecbf68ce94504cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_fuv_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9e54eb9b9a74a929ecc7fff7313bb2783f7a8d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_gaz_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45dbfc5730a25b7945cc06efb258da867f86c690 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_hau_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb58d8a079a7c0fd1fbeb30976361190d91f067f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8ce83009df730072f0d40dd8090d103c2825a9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_kea_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..028de73a190394f6c7c7de22f24060294cf1da3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_kin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e7b6a20c34b08afb2d6f569a07a275203e48ae1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_lug_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce5ec04ac9e580585168a7a26b7f180ccba450f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_luo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26d2f699da8daad91726627cd37a6a8d92965faf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_nya_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffdf1460a464cdc0a14459b8d48178b4246e6ed4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_plt_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3869a67959fbff48a93174161750246ee0ccc510 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_sna_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d7be50ca6bd926924a79dc261156b7d00775966 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_som_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec30bccc32adc5a5d7a7d6789a4dfb3941d3cc4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_sot_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..510e7b8f2a6d7528fc8162f00bdbaa7b6a89ac4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ssw_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..827f1f3614be4b1945bc8495ef52729b6c0778c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_tir_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele new file mode 100644 index 0000000000000000000000000000000000000000..0d85bf5172c2e8b9408448196191f3b7d40367a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_5 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c0314a96d451eb0cbbe04ef2036fba63d7927f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_arz_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..704c41a5ec8a68081ac33ea39b72f962e75e130f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_bam_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff6493b711b0126ed2b32ad0d7d7f668c5c71482 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_lin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f81859aae5913c79df2bb245fe48016dba0920a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_luo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2e8b96f93a43c64e36d0f2b8199524344511b70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_ssw_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a44af344142293043db80d9f55140569b7fdebf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7654a6cfe7974b352446e8c71c5740fb9e45f9f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/utils.py @@ -0,0 +1,155 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "P: {{flores_passage}}\nQ: {{question.strip()}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nPlease choose the correct answer from the options above:", + "prompt_2": "Passage: {{flores_passage}}\nQuestion: {{question.strip()}}\n1: {{mc_answer1}}\n2: {{mc_answer2}}\n3: {{mc_answer3}}\n4: {{mc_answer4}}\nPlease select the correct answer from the given choices:", + "prompt_3": "Context: {{flores_passage}}\nQuery: {{question.strip()}}\nOption A: {{mc_answer1}}\nOption B: {{mc_answer2}}\nOption C: {{mc_answer3}}\nOption D: {{mc_answer4}}\nPlease indicate the correct option from the list above:", + "prompt_4": "{{flores_passage}}\nBased on the above passage, answer the following question:\n{{question.strip()}}\nChoices:\nA) {{mc_answer1}}\nB) {{mc_answer2}}\nC) {{mc_answer3}}\nD) {{mc_answer4}}\nPlease provide the correct answer from the choices given:", + "prompt_5": "Read the passage: {{flores_passage}}\nThen answer the question: {{question.strip()}}\nOptions:\nA. {{mc_answer1}}\nB. {{mc_answer2}}\nC. {{mc_answer3}}\nD. {{mc_answer4}}\nPlease choose the correct option from the above list:", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "afr": "Afrikaans", + "amh": "Amharic", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "bam": "Bambara", + "eng": "English", + "fra": "French", + "hau": "Hausa", + "ibo": "Igbo", + "lin": "Lingala", + "por": "Portuguese", + "sna": "Shona", + "swa": "Swahili", + "tir": "Tigrinya", + "tso": "Tsonga", + "tsn": "Tswana", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", + "ssw": "Swati", + "sot": "Southern Sotho", + "som": "Somali", + "plt": "Plateau Malagasy", + "nya": "Nyanja", + "luo": "Luo", + "lug": "Luganda", + "kin": "Kinyarwanda", + "kea": "Kabuverdianu", + "gaz": "Oromo", + "fuv": "Nigerian Fulfulde", + } + + lang_2_dataset_lang_code = { + "afr": "afr_Latn", + "amh": "amh_Ethi", + "ary": "ary_Arab", + "arz": "arz_Arab", + "bam": "bam_Latn", + "eng": "eng_Latn", + "fra": "fra_Latn", + "hau": "hau_Latn", + "ibo": "ibo_Latn", + "lin": "lin_Latn", + "por": "por_Latn", + "sna": "sna_Latn", + "swa": "swh_Latn", + "tir": "tir_Ethi", + "tso": "tso_Latn", + "tsn": "tsn_Latn", + "wol": "wol_Latn", + "xho": "xho_Latn", + "yor": "yor_Latn", + "zul": "zul_Latn", + "ssw": "ssw_Latn", + "sot": "sot_Latn", + "som": "som_Latn", + "plt": "plt_Latn", + "nya": "nya_Latn", + "luo": "luo_Latn", + "lug": "lug_Latn", + "kin": "kin_Latn", + "kea": "kea_Latn", + "gaz": "gaz_Latn", + "fuv": "fuv_Latn", + } + + for lang in languages.keys(): + try: + file_name = f"belebele_{lang}.yaml" + task_name = f"belebele_{lang}_{mode}" + yaml_template = "belebele" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang_2_dataset_lang_code[lang], + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_5", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/flores.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/flores.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09b6e39274a4a686d20f742927b9a2740c2ef59f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/flores.yaml @@ -0,0 +1,14 @@ +group: african_flores +task: + - flores_eng-afr_prompt_1 + - flores_eng-afr_prompt_2 + - flores_eng-afr_prompt_3 + - flores_afr-eng_prompt_1 + - flores_afr-eng_prompt_2 + - flores_afr-eng_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores new file mode 100644 index 0000000000000000000000000000000000000000..c25cf195cd032014435335eadf13e102f47598f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c558249774e6182755078db81154e3d88db656c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ace_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Acehnese (Arabic script): {{sentence_ace_Arab}} \nEnglish: " +include: flores +task: flores_ace_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f0a6ee27cfc218e297ccaf05ab7d0bcef5da57b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ace_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Acehnese (Latin script): {{sentence_ace_Latn}} \nEnglish: " +include: flores +task: flores_ace_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d63b4c6baf598d665509cadcaf1c1613f7f2c77d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Shona: {{sentence_sna_Latn}} \nEnglish: " +include: flores +task: flores_sna_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe3ceb9a874c035a06f3c9fdc46e254a544bc563 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ssw_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swati: {{sentence_ssw_Latn}} \nEnglish: " +include: flores +task: flores_ssw_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3f605f9400dacb782a0066b1f6559aaba6ed270 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sun_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Sundanese: {{sentence_sun_Latn}} \nEnglish: " +include: flores +task: flores_sun_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7651ac3159f59d94886dc97a8e854fb19e184115 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swh_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swahili: {{sentence_swh_Latn}} \nEnglish: " +include: flores +task: flores_swh_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3fca39004e66ae6c74858cea59e930527a41eff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: taq_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tamasheq: {{sentence_taq_Latn}} \nEnglish: " +include: flores +task: flores_taq_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7152867ee6e32951f630f2d24d615ec248090fde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: taq_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tamasheq (Tifinagh script): {{sentence_taq_Tfng}} \nEnglish: " +include: flores +task: flores_taq_Tfng-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc13ae0413578e722f0f7c7e1c724e565b7a10c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tigrinya: {{sentence_tir_Ethi}} \nEnglish: " +include: flores +task: flores_tir_Ethi-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a6c4e1c820c15ff4e1b3f2d8db43c102a4b206e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tsn_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Setswana: {{sentence_tsn_Latn}} \nEnglish: " +include: flores +task: flores_tsn_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d473ab03b198ff0691265f278a6aec6e688e967 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tsonga: {{sentence_tso_Latn}} \nEnglish: " +include: flores +task: flores_tso_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c491f25b514977bd8554a0d53212525b028a6e41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tum_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tumbuka: {{sentence_tum_Latn}} \nEnglish: " +include: flores +task: flores_tum_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d8ad29e375918e52cbd3337ac6965a461c7f7fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Twi: {{sentence_twi_Latn}} \nEnglish: " +include: flores +task: flores_twi_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba4624651a68c219eefd7c9157f3bce58d751c48 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tzm_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Central Atlas Tamazight: {{sentence_tzm_Tfng}} \nEnglish: " +include: flores +task: flores_tzm_Tfng-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0758003ad3bb766fad0fa545d39ba976d4443f26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: umb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Umbundu: {{sentence_umb_Latn}} \nEnglish: " +include: flores +task: flores_umb_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..914e6c128220583cb7f2e064e0eed56948bbdfd9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Wolof: {{sentence_wol_Latn}} \nEnglish: " +include: flores +task: flores_wol_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc130fb0447aa7e617debe3cf1086f55ac96aec6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Xhosa: {{sentence_xho_Latn}} \nEnglish: " +include: flores +task: flores_xho_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ea0fbc4e65b9320f1d4bd701a98819c17b69ab0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Yoruba: {{sentence_yor_Latn}} \nEnglish: " +include: flores +task: flores_yor_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea070b30e7822d9e55135f3842766e68f94c1f2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Zulu: {{sentence_zul_Latn}} \nEnglish: " +include: flores +task: flores_zul_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..252117ef888300c0dfaa64f2cacc55bf66292136 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nBemba: " +include: flores +task: flores_eng_Latn-bem_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb4e3566b8383d7bce70a88bd3b663fa984c5154 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-cjk_Latn +doc_to_target: sentence_cjk_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nChokwe: " +include: flores +task: flores_eng_Latn-cjk_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47b99a088c485bed46c51d1da0308ac569aaebd3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nFrench: " +include: flores +task: flores_eng_Latn-fra_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e124ae153091ed617f87388afb7d6c4c980d754 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-gaz_Latn +doc_to_target: sentence_gaz_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nOromo: " +include: flores +task: flores_eng_Latn-gaz_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9aaf537f1d1c491ae3de996d2180c9b32002647 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nHausa: " +include: flores +task: flores_eng_Latn-hau_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..802ae7dca4e9b4165f2028cfce4be8c08c1b0edd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kam_Latn +doc_to_target: sentence_kam_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKamba: " +include: flores +task: flores_eng_Latn-kam_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bee435fe5495364b08420772d1dfade8f9ac671d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kik_Latn +doc_to_target: sentence_kik_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKikuyu: " +include: flores +task: flores_eng_Latn-kik_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28530541f153ad52724b5d0aa13eca176fa73c29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKinyarwanda: " +include: flores +task: flores_eng_Latn-kin_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5619209f346582b64e51b904288181fc18bc34d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kmb_Latn +doc_to_target: sentence_kmb_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKimbundu: " +include: flores +task: flores_eng_Latn-kmb_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fba5e257d7c5154ef3d89ace61d5f7fe2397ccc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Arab +doc_to_target: sentence_knc_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nCentral Kanuri (Arabic script): " +include: flores +task: flores_eng_Latn-knc_Arab_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1f84d5753ff5e03d9d4d6463beeb9a390d5c2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Latn +doc_to_target: sentence_knc_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nCentral Kanuri (Latin script): " +include: flores +task: flores_eng_Latn-knc_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6d8ef32d897edfa3086e7b93c39efa41a907e75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kon_Latn +doc_to_target: sentence_kon_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKikongo: " +include: flores +task: flores_eng_Latn-kon_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f998f3590b5ebdc35388a9a875a6358366684260 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-lin_Latn +doc_to_target: sentence_lin_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLingala: " +include: flores +task: flores_eng_Latn-lin_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..246fc1354a71eadbe1ea6e058387859fd5c018c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-lua_Latn +doc_to_target: sentence_lua_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLuba-Kasai: " +include: flores +task: flores_eng_Latn-lua_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3416989fdba2777eb13664a7db4409f091bb0b75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-lug_Latn +doc_to_target: sentence_lug_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLuganda: " +include: flores +task: flores_eng_Latn-lug_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a56e1482037f8728945929f991a9217a9d91f05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-luo_Latn +doc_to_target: sentence_luo_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLuo: " +include: flores +task: flores_eng_Latn-luo_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..393862689847cd6d3c6701696f57bea6f190c564 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-mos_Latn +doc_to_target: sentence_mos_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nMossi: " +include: flores +task: flores_eng_Latn-mos_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86bd9c6bcdc72f750dd3ae245c992e754ec6b55b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: flores +task: flores_eng_Latn-nso_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ac9148958f11e460266f4ff55aab4b44263074c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-nus_Latn +doc_to_target: sentence_nus_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNuer: " +include: flores +task: flores_eng_Latn-nus_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e35d78e708d25ad57abf36bb3ef4230f1acd66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNyanja: " +include: flores +task: flores_eng_Latn-nya_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e07ffcd257e91028807b37bec7c259f04fd3adb9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-plt_Latn +doc_to_target: sentence_plt_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nPlateau Malagasy: " +include: flores +task: flores_eng_Latn-plt_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cad3666bdb35072af569f2405ea106c487121d57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-run_Latn +doc_to_target: sentence_run_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nRundi: " +include: flores +task: flores_eng_Latn-run_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eaa3c8995add71ccf7052a621d92e76cd06861c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sag_Latn +doc_to_target: sentence_sag_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSango: " +include: flores +task: flores_eng_Latn-sag_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16f70ba79f218b67b8e3efb730cde9d903ea38b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nShona: " +include: flores +task: flores_eng_Latn-sna_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b628b7a4eddf4a073c2b6a77c6ed295c0f9cca17 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSomali: " +include: flores +task: flores_eng_Latn-som_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62655dff56701879705027502b23986c3bd96f78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sot_Latn +doc_to_target: sentence_sot_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSouthern Sotho: " +include: flores +task: flores_eng_Latn-sot_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee3c4a5712ea83c5ab676c949c77192ba4f84735 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sun_Latn +doc_to_target: sentence_sun_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSundanese: " +include: flores +task: flores_eng_Latn-sun_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b464c16601d0e7e385a6df32b83fcde41d24c91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-swh_Latn +doc_to_target: sentence_swh_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwahili: " +include: flores +task: flores_eng_Latn-swh_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc50d54faa83f621f08241f59baf6a14e4b6c674 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Latn +doc_to_target: sentence_taq_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTamasheq: " +include: flores +task: flores_eng_Latn-taq_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8c0045338ad054dd48750ae88767f958f0b9e4f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Tfng +doc_to_target: sentence_taq_Tfng +doc_to_text: "English: {{sentence_eng_Latn}} \nTamasheq (Tifinagh script): " +include: flores +task: flores_eng_Latn-taq_Tfng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d3110696c8c1088d2ad2683d8d9d45b3415038d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nTigrinya: " +include: flores +task: flores_eng_Latn-tir_Ethi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85bca5e9ffdf083cca344501fb818d7e0e60b732 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tso_Latn +doc_to_target: sentence_tso_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTsonga: " +include: flores +task: flores_eng_Latn-tso_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28728f412fdbe74d85d062a316eeb385b04b94a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tzm_Tfng +doc_to_target: sentence_tzm_Tfng +doc_to_text: "English: {{sentence_eng_Latn}} \nCentral Atlas Tamazight: " +include: flores +task: flores_eng_Latn-tzm_Tfng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb6965245032f6821b4ca413d6ead9e892bdb407 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nWolof: " +include: flores +task: flores_eng_Latn-wol_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08480361c18c787ead563d02783982bd1ad8b8e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nXhosa: " +include: flores +task: flores_eng_Latn-xho_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29e9a9c859134f25abdc46ca44a256d473415a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nYoruba: " +include: flores +task: flores_eng_Latn-yor_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62de546051295ff6b413870f9eee5e806151f4cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nZulu: " +include: flores +task: flores_eng_Latn-zul_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/flores new file mode 100644 index 0000000000000000000000000000000000000000..74f9f33eb22662bec79709bd64d8d31f3fb8eae0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/flores @@ -0,0 +1,24 @@ +tag: +- flores_tasks +- flores_afr-eng +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores new file mode 100644 index 0000000000000000000000000000000000000000..e0fa69a2a441116ef15a4158cc366792d841f304 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_2 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd54b6c84dc428721616791f698ce55f5064aef4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Acehnese (Arabic\ + \ script) sentences to English \nAcehnese (Arabic script): {{sentence_ace_Arab}}\n\ + English: " +include: flores +task: flores_ace_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0814b27f80b3ac85e70179a124708ed5f9c3ac4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Acehnese (Latin\ + \ script) sentences to English \nAcehnese (Latin script): {{sentence_ace_Latn}}\n\ + English: " +include: flores +task: flores_ace_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1464b4965d566d628dcfa12583003a972af29aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: acq_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Ta’izzi-Adeni\ + \ Arabic sentences to English \nTa’izzi-Adeni Arabic: {{sentence_acq_Arab}}\nEnglish: " +include: flores +task: flores_acq_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bbded5ff0d48bf2bbd582dbfa97cfa4d222b5c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tunisian Arabic\ + \ sentences to English \nTunisian Arabic: {{sentence_aeb_Arab}}\nEnglish: " +include: flores +task: flores_aeb_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b5847d8a997310bd191a3e0009d24abdedb6e4f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Afrikaans sentences\ + \ to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: flores +task: flores_afr_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f9493c5ea18493d4b0a2f2070135b5fb01c0692 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Akan sentences\ + \ to English \nAkan: {{sentence_aka_Latn}}\nEnglish: " +include: flores +task: flores_aka_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d615bfc3d6f9ed19dce78c024fdaa45a53fdac8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: flores +task: flores_amh_Ethi-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..feecf4510ac45bf5d364cf1782942e388c2119eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Moroccan Arabic\ + \ sentences to English \nMoroccan Arabic: {{sentence_ary_Arab}}\nEnglish: " +include: flores +task: flores_ary_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13f3e18b5d6ecd8d911741e4fe1d3ee7720f81df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Egyptian Arabic\ + \ sentences to English \nEgyptian Arabic: {{sentence_arz_Arab}}\nEnglish: " +include: flores +task: flores_arz_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a258d264b26a9c02be0ca900f058500b19f0c256 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Bambara sentences\ + \ to English \nBambara: {{sentence_bam_Latn}}\nEnglish: " +include: flores +task: flores_bam_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c19cf00874080b980b18e811f61d31a332ca2a5b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ban_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Balinese sentences\ + \ to English \nBalinese: {{sentence_ban_Latn}}\nEnglish: " +include: flores +task: flores_ban_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9500a3b37c033a026795f47cc74c6a7df94325c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: flores +task: flores_bem_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58185199f7dcfcff5a9c5c4ba501ea7edaa4b26f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Chokwe sentences\ + \ to English \nChokwe: {{sentence_cjk_Latn}}\nEnglish: " +include: flores +task: flores_cjk_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c9090a56c686036f07c63b3b92bafad068d811e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Southwestern Dinka\ + \ sentences to English \nSouthwestern Dinka: {{sentence_dik_Latn}}\nEnglish: " +include: flores +task: flores_dik_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47187fb0817ddf7c04041f6b2b9ef358138dfdf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Dyula sentences\ + \ to English \nDyula: {{sentence_dyu_Latn}}\nEnglish: " +include: flores +task: flores_dyu_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8838bc3a03742e9e4da5a387f995dcc94018b5d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: flores +task: flores_ewe_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7874a87cecb89c402a0e0c1ffea473f4283cc58f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Fon sentences\ + \ to English \nFon: {{sentence_fon_Latn}}\nEnglish: " +include: flores +task: flores_fon_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb84246ef4b7942f1ecac940e1f80d1664faef3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: flores +task: flores_fra_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0686706d533616d7c1809313c1f5bd302b3c1a45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fuv_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Nigerian Fulfulde\ + \ sentences to English \nNigerian Fulfulde: {{sentence_fuv_Latn}}\nEnglish: " +include: flores +task: flores_fuv_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0ba07a6112f9dc2b740a868a5df7673dfba7650 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_gaz_Latn}}\nEnglish: " +include: flores +task: flores_gaz_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85647455d58aabfb824508bae282e15f63ccaa18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: flores +task: flores_hau_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c401f1e75be4f83b89a714d40eaaa97ffe677e36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: flores +task: flores_ibo_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c82946b9143ce0706f9768bb078d6bdc6541ebbc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kabyle sentences\ + \ to English \nKabyle: {{sentence_kab_Latn}}\nEnglish: " +include: flores +task: flores_kab_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8661bf6e5694498af0e29ca23e9e037ddd77adc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kamba sentences\ + \ to English \nKamba: {{sentence_kam_Latn}}\nEnglish: " +include: flores +task: flores_kam_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e20af3149dc1baad3b0edca477444a98ba078c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kabiyè sentences\ + \ to English \nKabiyè: {{sentence_kbp_Latn}}\nEnglish: " +include: flores +task: flores_kbp_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d078c293ab75a286b9ac717f9322d8fd92d0b585 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kabuverdianu sentences\ + \ to English \nKabuverdianu: {{sentence_kea_Latn}}\nEnglish: " +include: flores +task: flores_kea_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..346dcb98be91749b59747194a47cefe40ca3eef4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kikuyu sentences\ + \ to English \nKikuyu: {{sentence_kik_Latn}}\nEnglish: " +include: flores +task: flores_kik_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7210e7e6b21c0458983917f659518ba666d45d0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kinyarwanda sentences\ + \ to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: flores +task: flores_kin_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3dc8d5ac6d52ccfb84f6b8fc416ab61eaa7c007 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kimbundu sentences\ + \ to English \nKimbundu: {{sentence_kmb_Latn}}\nEnglish: " +include: flores +task: flores_kmb_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d60408cd39dc640f7db077186340de97aa4702f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Central Kanuri\ + \ (Latin script) sentences to English \nCentral Kanuri (Latin script): {{sentence_knc_Latn}}\n\ + English: " +include: flores +task: flores_knc_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63b3539cf9f361ba4b0786596bf43d934c85478c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kikongo sentences\ + \ to English \nKikongo: {{sentence_kon_Latn}}\nEnglish: " +include: flores +task: flores_kon_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af0796cc47138b67137e58ba44835ffa9ebf8596 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Luba-Kasai sentences\ + \ to English \nLuba-Kasai: {{sentence_lua_Latn}}\nEnglish: " +include: flores +task: flores_lua_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1f9ca54df695d83cc607b4409522ab459c28a99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Nuer sentences\ + \ to English \nNuer: {{sentence_nus_Latn}}\nEnglish: " +include: flores +task: flores_nus_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..983675b039a2ba51232dede065edec9dd7536c75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Umbundu sentences\ + \ to English \nUmbundu: {{sentence_umb_Latn}}\nEnglish: " +include: flores +task: flores_umb_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores new file mode 100644 index 0000000000000000000000000000000000000000..ab71d6563002c5deed46fb73f2b61bd585b7b9ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_eng-afr +- flores_eng-afr_prompt_2 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30890d9150172e44d453679cea878790d1153f95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Arab +doc_to_target: sentence_ace_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Acehnese (Arabic script) \nEnglish: {{sentence_eng_Latn}} \nAcehnese (Arabic\ + \ script): " +include: flores +task: flores_eng_Latn-ace_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4356785a7d7a7de55ea328e7957ba14764a26745 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Latn +doc_to_target: sentence_ace_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Acehnese (Latin script) \nEnglish: {{sentence_eng_Latn}} \nAcehnese (Latin\ + \ script): " +include: flores +task: flores_eng_Latn-ace_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..630c824e342e8032926204ca41cfdbc6472c35eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-acq_Arab +doc_to_target: sentence_acq_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ta’izzi-Adeni Arabic \nEnglish: {{sentence_eng_Latn}} \nTa’izzi-Adeni Arabic: " +include: flores +task: flores_eng_Latn-acq_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2769adf1ec72d54aab8dd1911b91f14c6c56db7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: flores +task: flores_eng_Latn-afr_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..624149c7fb2c031e4483050382e7fe12a09ad32f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-aka_Latn +doc_to_target: sentence_aka_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Akan \nEnglish: {{sentence_eng_Latn}} \nAkan: " +include: flores +task: flores_eng_Latn-aka_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0362666cb7d58d02ed5af9e9429f8b56e1a12d47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-arz_Arab +doc_to_target: sentence_arz_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Egyptian Arabic \nEnglish: {{sentence_eng_Latn}} \nEgyptian Arabic: " +include: flores +task: flores_eng_Latn-arz_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b38459211670064398a117bdb4b5a63c342471d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-bam_Latn +doc_to_target: sentence_bam_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Bambara \nEnglish: {{sentence_eng_Latn}} \nBambara: " +include: flores +task: flores_eng_Latn-bam_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38c4ea6ff6d0358ddf49b934b4f21549fb7b14d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-cjk_Latn +doc_to_target: sentence_cjk_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Chokwe \nEnglish: {{sentence_eng_Latn}} \nChokwe: " +include: flores +task: flores_eng_Latn-cjk_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b33033ff00dce959d37f6b7fe8f0440a6cd1577 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: flores +task: flores_eng_Latn-kin_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/flores new file mode 100644 index 0000000000000000000000000000000000000000..74f9f33eb22662bec79709bd64d8d31f3fb8eae0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/flores @@ -0,0 +1,24 @@ +tag: +- flores_tasks +- flores_afr-eng +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..887344c67c0fe6cd1658c8172b55a252bfc9f12b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-tzm_Tfng +doc_to_target: sentence_tzm_Tfng +doc_to_text: "As a Central Atlas Tamazight and English linguist, translate the following\ + \ English sentences to Central Atlas Tamazight \nEnglish: {{sentence_eng_Latn}}\ + \ \nCentral Atlas Tamazight: " +include: flores +task: flores_eng_Latn-tzm_Tfng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15b41952e52b3acfee85294c372ecb954f8b37cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "As a Zulu and English linguist, translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: flores +task: flores_eng_Latn-zul_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a569b3cec8f6858d041e9d0c0c876720558b808b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'You are a Twi linguistic analyst trained to understand Twi user intent. + Based on the Twitext: ''{{text}}'', choose the intent that best matches from this + list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_twi_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a55398ab4161ec54e971863cd1b4bfb41330b093 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'You are a Wolof linguistic analyst trained to understand Wolof user + intent. Based on the Woloftext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_wol_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b6e5aace303095eccb19b8a011559555cee7eb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'You are a Zulu linguistic analyst trained to understand Zulu user intent. + Based on the Zulutext: ''{{text}}'', choose the intent that best matches from this + list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_zul_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..518ec898411de5d141ac16137578e710fc7bb60e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'The following text is in Igbo: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72a7686934892b2f21f3098b302dc601050b394b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'The following text is in Oromo: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_orm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a8d0328e75acaead01f4e08fb4c6ed26eed6314 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'The following text is in Sotho: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_sot_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1da6be32a6c64ec6583c70a2f0cbaa3d6aa3d435 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'The following text is in Swahili: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc78ae4f60d64d9d30a5bf52064fda54e0bce9ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'The following text is in Twi: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_twi_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c71483ec36660b62bc945b4ebf89fdb77034e85 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'The following text is in Wolof: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_wol_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d8b543fc5cac462b31a9ae806aa3ce0f60b6ba9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'The following text is in Xhosa: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_xho_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbe285688f78a2ec210740e0294e4a3c6fde8dd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'The following text is in Yoruba: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ba384db5a8b2f7f17601ad7fff0bed8327ecc1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'The following text is in Zulu: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_zul_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/mafand.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/mafand.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef8619addf5df6be4d59e31540c6ed6663e6b189 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/mafand.yaml @@ -0,0 +1,14 @@ +group: mafand +task: + - mafand_eng-afr_prompt_1 + - mafand_eng-afr_prompt_2 + - mafand_eng-afr_prompt_3 + - mafand_afr-eng_prompt_1 + - mafand_afr-eng_prompt_2 + - mafand_afr-eng_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand new file mode 100644 index 0000000000000000000000000000000000000000..4f2047be0877cfda1e41acc76e491478c1f8f8f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_afr-eng +- mafand_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target +doc_to_text: !function utils.create_text_prompt_1 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95e87fd8aeb3df35fd529338e719683805a78f18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_amh-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abe64f9b2a1dbf14bcf60f9f4f80df24f65821ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_bbj-fr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecd9b38bcdfda7354569bc002e3ed10ff573449f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_ewe-fr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..705cfbb855261a2cc14d841bde89661fa5d6be75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fon-fr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d78c91bb1fc61797c9554b9cf8acb7c099e53919 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_ibo-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..954c036e80de22ad705888d66705e58b2a15f689 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_kin-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..671c072adc79858fc9eb1f2c32b99420b26fcb95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_lug-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1d5965f08d62f2a8147a39878f844d714800ade --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_luo-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da085707bc8ec37730535867ad2faa80e23bfa20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_mos-fr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bc2426687b5784602b1bc1bcb64c453d79d39ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_nya-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6fb5dee6d393fbe6a6172e347e382cee2aecfb5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_pcm-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..283517d6ac4f9f4c8e462a63002d11a681c438b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_sna-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..476bba42d82461b241b2b8a396baefe20c154206 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_swa-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a94c5b6ea0f870db19b4fc3c692be95a3fc6d455 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_tsn-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f5883b12d6875133699a8158dd2151c8c784981 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_twi-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb887188000da8e83aeed287619ed1cd2375e18a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_wol-fr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0561b4157c15f0fa5f8f3876ab86d07087a5a0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_xho-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec97ae7d3c4abc88a4774dc69cb32e2d7dced32d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_yor-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9649d772d259a5519e87ae522f80b3d591f1b2be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_zul-en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0df3a329824d44fa94eb830ae943fa30dd32bab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand new file mode 100644 index 0000000000000000000000000000000000000000..1d004556267924e0120e95fb516fee72a5d3eb1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_eng-afr +- mafand_eng-afr_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target_reverse +doc_to_text: !function utils.create_reverse_prompt_1 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ef9ab288615c42f1c4c04538f0b51e3d2245ac1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_en-amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ea577781deda41a7217dbd757158789a80dafc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_en-hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88af221fb31e583d9238c95c05dffb44306a617e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_en-ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c415f511ad246eb99509f4bedb979810dfcf20d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_en-kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94070e8986b0f44eecb1097c673c7daf4aec8067 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_en-lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc6b15c67f2919dca92ef84d058a93da6e0271a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_en-luo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..225a46474ff2c694da98558bfaf665bb8e102248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_en-nya_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69380c7182bdfc8f8e25c2ee16b3ced4a5c05d49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_en-pcm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..634d988fd46f0a98b6e5b76e4a42f897edb4871d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_en-sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfbf259cbbe8d6d740dc7522f6d8bd6542c790a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_en-swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..faa99ddf5186515253279585aa1439569d7de9b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_en-tsn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9294975da5385d196db9abc9f22d087a2c9cce0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_en-twi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..244f5cabd9901020a136322842ec47fb181faed6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_en-xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa3189779577ecbc58f80ff4973a63d81ea1c6a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_en-yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6afdc0c5648d8fec0be6d2d2fca0b9416dccffab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_en-zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c21d96f275a14b81225b3979044a23909bfb023 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_fr-bam_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76cf07507625677c85772e988ac66741508efbc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_fr-bbj_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c7bd6671b7ba6da2a869cf12862046b632396e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_fr-ewe_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..737d68eba79405276a96151a33786387c33cc148 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fr-fon_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9186a5b9f6670f5187284981d547d91631cc94f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_fr-mos_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e29f5fb98a138f3a131c62b685abc25b60e3f69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_fr-wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0df3a329824d44fa94eb830ae943fa30dd32bab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand new file mode 100644 index 0000000000000000000000000000000000000000..eb7ad9883115b42626fd30d223fa90ff6f133384 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_afr-eng +- mafand_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target +doc_to_text: !function utils.create_text_prompt_2 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6db544cb5e17e87e6ebcf606ad59ad9a7435f338 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_amh-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a9f3b3ac09958dc4253d982dfe1c872aefafd7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_bam-fr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0b42b23787e88bd5ff0d000c1806655e26f65cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_bbj-fr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..457c0d1945bfa65f5d5f0e1ccffc30c11aadd452 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_ewe-fr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84263d5a9ca41877f02d680660706eaeba28fea2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fon-fr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05c31a4670a9300c1175dc2f7e109c024b146301 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_hau-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cb4a5b897c2d143ef31d667c9d87fc35f40caa0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_ibo-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3e1acf9a92a63ab80dbc0314adebfa59e449490 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_kin-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb68279d783b71ff94742507d4ed1571a03b6b51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_lug-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12f199473ee6e4ab1f6a2d3ae6e69eb04ba6a399 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_luo-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a723701d3250aa78f7631a3dcfc450301201bf73 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_mos-fr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24569f00825bb4a2e0419e290ae4ff1bc7e0d312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_nya-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ec0c66a0a5c7fadbc7b9ec715fe53596d4c2b51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_pcm-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bf99b9955378aa6a5930d9783ed33dfea96ac95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_sna-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb2ada0bba76e9c83a99d8060ac8f4146a1462c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_swa-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d16e7e94c5b251eff2adece451f89c7af71fbc30 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_tsn-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..267337c177fc06e777d62c3a2286084d79266a4c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_twi-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6c67bd8d6df8d81d4d4088b6bbf8e04498afef6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_wol-fr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d44db7a2eeefcc7dc0218f08327ceaee4a6e351a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_zul-en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0df3a329824d44fa94eb830ae943fa30dd32bab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand new file mode 100644 index 0000000000000000000000000000000000000000..35548392e118f0625d9adae32849389d5239cd3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_eng-afr +- mafand_eng-afr_prompt_2 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target_reverse +doc_to_text: !function utils.create_reverse_prompt_2 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09c21d215e4cab74dda2b30ee110f55dcf2cbcbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_en-amh_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9a91c76499bf37863e6b55c71ff88205b1e2599 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_en-hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..568a845e403663c3d24e30383cab8eebe0d37151 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_en-ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09425f64afdb68817d0f462545e67f5a1e2d5f07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_en-kin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13c91d36a6e5e0037a301c88fcf814309e29590f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_en-lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41bb09363d1c100ffea32d38eb2085400bec018b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_en-luo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90a728107edf37bddd1d4eb80bcc6ddfaa49572e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_en-nya_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d9fc0e71589cb2e1c832a6b82e6d8fb5288b89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_en-swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3dd43626d00774a8e9bd6e432322e4d608887ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_en-tsn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8c1ffee3415a1373a1b583819e874f968b5cfee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_en-xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15c6e981d1cb878e26a08755f5bf1fd7629f2525 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_fr-bam_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f5101a752865bf641a150be2aff3b449239d3f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_fr-bbj_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..682fb19c479f7f4a49a810c1f665c20439e4ee3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_fr-mos_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand new file mode 100644 index 0000000000000000000000000000000000000000..eb7ad9883115b42626fd30d223fa90ff6f133384 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_afr-eng +- mafand_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target +doc_to_text: !function utils.create_text_prompt_2 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bed4252375b82e34d27c96adf368217957e33063 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_bam-fr_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f78f55a3b27a4461efd85e741503ead06e4bcce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_ibo-en_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3a367493d5da3804d2eda16eba676886765939e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_luo-en_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95ad3380334ced0554dc21ab4e6a07f23352f51d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_pcm-en_prompt_3