diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2d709e9ccfdb0251096156eecea495576aca1c13
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimgsm_yaml
+task: afrimgsm_fra_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fce1d51899bd307d460f312cbc60c8820295db6f
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ibo
+include: afrimgsm_yaml
+task: afrimgsm_ibo_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8e27f832a3b24b39d7515d657c01c54587fa3ac7
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sna
+include: afrimgsm_yaml
+task: afrimgsm_sna_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2eaceade5342296cc43a1d343ee9af582792922e
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml
@@ -0,0 +1,34 @@
+tag:
+    - afrimgsm_tasks
+    - afrimgsm_tasks_prompt_1
+dataset_path: masakhane/afrimgsm
+output_type: generate_until
+test_split: test
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: "Give direct numerical answers for the question provided. \n\nQuestion: {{question}} \nAnswer: "
+target_delimiter: ""
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6faf98bdd2ebe70aa1fbe7a0bc955eedce4c726a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimgsm_yaml
+task: afrimgsm_yor_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..eb3ed8309c481abfa483761fefa9f8792f9a600a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: amh
+include: afrimgsm_yaml
+task: afrimgsm_amh_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b1faf85c8be5594d03e73c57fcea2fcfba1508a9
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimgsm_yaml
+task: afrimgsm_fra_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fc451b94feac4afc4ba0405ee9643e0ab6790080
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ibo
+include: afrimgsm_yaml
+task: afrimgsm_ibo_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..86ccf1e44be09e34aa5b597f1db171a1c6b7f9a5
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lin
+include: afrimgsm_yaml
+task: afrimgsm_lin_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ff6b6de36aa85f43da7c26b0bccf33b92a4f5d9a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimgsm_yaml
+task: afrimgsm_lug_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a9941fce3105d32d2610c4e41b938360ac8b961c
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimgsm_yaml
+task: afrimgsm_sot_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..abd8358024d6cc567f6044b7e0dfa69cd9d1a5eb
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: swa
+include: afrimgsm_yaml
+task: afrimgsm_swa_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aae0b5f51a347f8b52841b0e0fc49197a8c8490b
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: vai
+include: afrimgsm_yaml
+task: afrimgsm_vai_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..067e88d709cd0f06c217d6823ae2cd99ebd5f003
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: xho
+include: afrimgsm_yaml
+task: afrimgsm_xho_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2d64481065491583c980a83d9ea40507b8830552
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimgsm_yaml
+task: afrimgsm_yor_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4b94a2c13dae19e6ed50ab49a321b1f73d326a97
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: zul
+include: afrimgsm_yaml
+task: afrimgsm_zul_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cda1994b4dd2b18fb18e0200342a58c1e42d47b8
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: ewe
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_ewe_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e6ad1a00dce65e79adac55593e53e9683b48decd
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_hau_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2c7cb9ffe60577d49d6aa87c191e0fb81c974ac
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_kin_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d8a12945fd73da0abe42bc7504ec5a861ef86a8d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: lug
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_lug_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..dcf7123f8e45f9d5d32ffc97f5038c29d946958d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: orm
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_orm_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ac0017a36b03a55102fab40f5c6a3cb2d6fffa84
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: sot
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_sot_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..043311d3e634759e7b08892126dfc64b94e39310
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_swa_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2dbe04c16eab6ea207a384c01860cc2555ffb6bb
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: vai
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_vai_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..55f546ab101fb6ee26158d5803fe1ad80f3cb8e4
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_wol_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5f34f774e9bc1cf7fb925eac9fd55b123389b738
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml
@@ -0,0 +1,33 @@
+tag:
+    - afrimgsm_tasks
+    - afrimgsm_tasks_prompt_4
+dataset_path: masakhane/afrimgsm
+output_type: generate_until
+test_split: test
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+target_delimiter: ""
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fd4d01bbf4f5ddb22b8057f3ffd99996c82b92d1
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_yor_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e7ad215fac134e19c62187205f141a73d1978e22
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: amh
+doc_to_text: "For mathematical questions provided in Amharic language. Supply the\
+  \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\
+  Answer: "
+include: afrimgsm_yaml
+task: afrimgsm_amh_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a4de5e95948fd28d990f224c9cb0f6e82b6f5cf5
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: eng
+doc_to_text: "For mathematical questions provided in English language. Supply the\
+  \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\
+  Answer: "
+include: afrimgsm_yaml
+task: afrimgsm_eng_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bdb3c4f278e1e31e5c20cb399df2a4a374aaa253
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: ewe
+doc_to_text: "For mathematical questions provided in Ewe language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_ewe_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a93e79ec90099beb8c50baac1503bf824cb9bcda
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: fra
+doc_to_text: "For mathematical questions provided in French language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_fra_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..74fabdee06aa0669ada4ba5fe46bd9a6ad6bdd32
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: "For mathematical questions provided in Hausa language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_hau_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c2dd77f238af5a67eccad3e05f6add96c3d1f321
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "For mathematical questions provided in Igbo language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_ibo_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ba3f3c2f2b264b333a6fe1903c6f84d6d6000692
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: "For mathematical questions provided in Kinyarwanda language. Supply\
+  \ the accurate numeric answer to the provided question. \n\nQuestion: {{question}}\
+  \ \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_kin_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0c33dd44cc0019f3485ca72368c767b9161ec983
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: orm
+doc_to_text: "For mathematical questions provided in Oromo language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_orm_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2518c36981f50157890029d3fb49d596ea688b6
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "For mathematical questions provided in chiShona language. Supply the\
+  \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\
+  Answer: "
+include: afrimgsm_yaml
+task: afrimgsm_sna_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..54de3ce507d4f5f4b0f098ec8d9a71e37fb75fc4
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: "For mathematical questions provided in Twi language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_twi_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e711e8efa4b2e854d9924f953dce931ea3a0ba1
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "For mathematical questions provided in Wolof language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_wol_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..728cacf881aef3014669a43c2ffc5316664decad
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml
@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: xho
+doc_to_text: "For mathematical questions provided in isiXhosa language. Supply the\
+  \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\
+  Answer: "
+include: afrimgsm_yaml
+task: afrimgsm_xho_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ca8bb03f88f2a98928fd21794753f0ff990bb309
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml
@@ -0,0 +1,33 @@
+tag:
+    - afrimgsm_tasks
+    - afrimgsm_tasks_prompt_5
+dataset_path: masakhane/afrimgsm
+output_type: generate_until
+test_split: test
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+target_delimiter: ""
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fb8474e31f328120b855c6e45d5340de8bd6c266
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml
@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_zul_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d07832b4ddb9d7b1306e36cc5c39f11ee840661b
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml
@@ -0,0 +1,9 @@
+group: afrimgsm_cot-irokobench
+task:
+  - afrimgsm_cot_tasks
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c9f0d9311aa462f2f4c3e9d38c5d0407ae7b72d7
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: amh
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_amh_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..717a45d98beabb763984f030350252b6d3424747
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_fra_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..55d20e011aa15f290f292411604391e2c761053f
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: kin
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_kin_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..033cbce071fed9d337fecfc002b3988671cafe9a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_lug_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7ce25aee61aa076b7c250c00a3a49f552c82f9d3
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: orm
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_orm_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fae029f19280d7bf4f0928a63022c6deeae09f6d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sna
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_sna_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d0d1207791575accf84f4f6839e8b41f688ba0d9
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_sot_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..da4e39cfc0a7b28c341ba8557971bcedd944e9d4
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: swa
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_swa_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c6858ab21d02d14d797ce3704c940fd627feb2be
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_yor_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e7ef2907f54397390ff29aa44612d6a24a3a9380
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ewe
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_ewe_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..75e8b89276ae9db785fbc75cf403bffb6c6a5f1c
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_lug_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..22fc718bc20db26210333151faf22c5a5c0bbef7
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_sot_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..833edbb1a704f59fc39000ccb8447fec46d55849
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: zul
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_zul_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..76501f53d90f79d70f86ec52b8cbe3d329db7b7e
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_sot_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d1f94eea1e47bd9442dbc76181069bf47ac29da1
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimmlu_direct
+task: afrimmlu_direct_fra_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..52f317982b39dd21a3e11ce6695b95af9ef8f8df
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eng
+include: afrimmlu_direct
+task: afrimmlu_direct_eng_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cce0e4607d45d5e302f7642b2343a7b1a1dcf991
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimmlu_direct
+task: afrimmlu_direct_sot_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1c1507260fd7c79cfde49981a52fffa5b5cc89d2
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: xho
+include: afrimmlu_direct
+task: afrimmlu_direct_xho_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bbbf9387e5563914e7b89a06540d73156c8fb1b9
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml
@@ -0,0 +1,9 @@
+group: afrimmlu_tt-irokobench
+task:
+  - afrimmlu_tt_tasks
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate
new file mode 100644
index 0000000000000000000000000000000000000000..7a974279a3918de90369c391b09de818cb1b483d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate
@@ -0,0 +1,32 @@
+tag: afrimmlu_tt_tasks
+dataset_path: masakhane/afrimmlu-translate-test
+dataset_name: null
+output_type: multiple_choice
+test_split: test
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
+doc_to_choice: !function utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
+    # aggregation: mean
+    average: weighted
+    hf_evaluate: true
+    higher_is_better: True
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6ac43a8060bf6e3dff94cd141c2bcb02014e8abd
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimmlu_translate
+task: afrimmlu_translate_fra_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c09424d3812e26e2c8ff8a2bc08eecec7690f9fd
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: hau
+include: afrimmlu_translate
+task: afrimmlu_translate_hau_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bcbac5f65455134c60b873710d7c2e66cf0ac7ca
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimmlu_translate
+task: afrimmlu_translate_lug_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..84b3d2c35042fa836f7450fd48e4654876c0473e
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: orm
+include: afrimmlu_translate
+task: afrimmlu_translate_orm_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9fb5f3709d4bfb2702903eb6b50f36bd9b02c34a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: wol
+include: afrimmlu_translate
+task: afrimmlu_translate_wol_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1a06af041863759d796db54f8276c7722d4abf92
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: xho
+include: afrimmlu_translate
+task: afrimmlu_translate_xho_prompt_1
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f1bb9162f0fbc68807db68134970ae2636980cbf
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py
@@ -0,0 +1,32 @@
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence
+                model answers multiple-choice questions about {subject}
+
+                Question: {question}
+
+                Choices:
+                        A: {choice1}
+                        B: {choice2}
+                        C: {choice3}
+                        D: {choice4}
+
+                Answer:  """
+
+    choices = eval(doc["choices"])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
+    return text
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..283b6ff1798eba1b331edc46036b5cf38e482443
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: amh
+include: afrimmlu_translate
+task: afrimmlu_translate_amh_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..39e9f7355ace85592c87ad626303f3d4cc89a16b
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ewe
+include: afrimmlu_translate
+task: afrimmlu_translate_ewe_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ced80282a8addde381a1b719e10e871adb2cf533
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimmlu_translate
+task: afrimmlu_translate_fra_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0d687cac47605f507f67c4a30d39e975a2a794cb
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: hau
+include: afrimmlu_translate
+task: afrimmlu_translate_hau_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d1edfaa3999a264b903295abfc34d39cdcece572
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ibo
+include: afrimmlu_translate
+task: afrimmlu_translate_ibo_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..48fa15181a1c6e031581cb7e80da907788b90750
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: kin
+include: afrimmlu_translate
+task: afrimmlu_translate_kin_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d6b709ec0b5d967579097b2251f16e278db940e5
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimmlu_translate
+task: afrimmlu_translate_lug_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2a58330c01ee2da2d2b76b3f700ccdad665450fa
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: orm
+include: afrimmlu_translate
+task: afrimmlu_translate_orm_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0edd502ec7d909dcde2026f7e2bc0d7747032728
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sna
+include: afrimmlu_translate
+task: afrimmlu_translate_sna_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2f499691fb172c8ddc1a89abc1d3ac64c4303286
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimmlu_translate
+task: afrimmlu_translate_sot_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b145669ebb274b39da4826cf7a62c4dd0d72dd35
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: twi
+include: afrimmlu_translate
+task: afrimmlu_translate_twi_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c87a8d61d88ab5109759ac57cc18c1935f9596b0
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: wol
+include: afrimmlu_translate
+task: afrimmlu_translate_wol_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b5cdbc6b28f26ffa7d227618e1adf04aa13f0d8d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimmlu_translate
+task: afrimmlu_translate_yor_prompt_2
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0cfb334c27cfe4c5bbb1ff7126215c0ea9130c9
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py
@@ -0,0 +1,30 @@
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """As an expert in {subject}, choose the most accurate answer to the question below.
+Your goal is to select the correct option 'A', 'B', 'C', or 'D' by understanding the nuances of the topic.
+
+Question: {question}
+Choices:
+        A: {choice1}
+        B: {choice2}
+        C: {choice3}
+        D: {choice4}
+Answer: """
+
+    choices = eval(doc["choices"])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
+    return text
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate
new file mode 100644
index 0000000000000000000000000000000000000000..7a974279a3918de90369c391b09de818cb1b483d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate
@@ -0,0 +1,32 @@
+tag: afrimmlu_tt_tasks
+dataset_path: masakhane/afrimmlu-translate-test
+dataset_name: null
+output_type: multiple_choice
+test_split: test
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
+doc_to_choice: !function utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
+    # aggregation: mean
+    average: weighted
+    hf_evaluate: true
+    higher_is_better: True
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0377257387ded41038b9d77532536cd5383a02b0
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: amh
+include: afrimmlu_translate
+task: afrimmlu_translate_amh_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aa924284fc298a2171f6cd13e69684d1e26cc166
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ewe
+include: afrimmlu_translate
+task: afrimmlu_translate_ewe_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7bef6e78de5ed0f69cda1e26383e05cc5735c23d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimmlu_translate
+task: afrimmlu_translate_fra_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d9841db93cb047f4b5a21118a1c7ce5f036ea1d4
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: hau
+include: afrimmlu_translate
+task: afrimmlu_translate_hau_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..20c58b27ee0e4c3d73affc18d621a8db348f278e
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ibo
+include: afrimmlu_translate
+task: afrimmlu_translate_ibo_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ed3cfd16498db252b6c48eda56079b253595cfa7
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: kin
+include: afrimmlu_translate
+task: afrimmlu_translate_kin_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9eeb66eaadf15cbeec8cfcbf6664fa6eb74da889
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lin
+include: afrimmlu_translate
+task: afrimmlu_translate_lin_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..46f722b757ee9acb3520868d428bb51735343687
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimmlu_translate
+task: afrimmlu_translate_lug_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aa031e5c8a875c5fe75f11efa01dc044a003b1a4
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sna
+include: afrimmlu_translate
+task: afrimmlu_translate_sna_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e55ce671f15df0c1478357f4f8dad2fa4a4f09ed
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimmlu_translate
+task: afrimmlu_translate_sot_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9f507772fb75616c8fe9a44ccd4a9549125046b8
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: swa
+include: afrimmlu_translate
+task: afrimmlu_translate_swa_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1ea25d8b7f3ff1a3ce70f0a4ed68d2e999ff2143
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: twi
+include: afrimmlu_translate
+task: afrimmlu_translate_twi_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..815d219ba75659cf564b691a6c859042b6a397b6
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimmlu_translate
+task: afrimmlu_translate_yor_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f294a873804351a29f63276958d10a963e461519
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: zul
+include: afrimmlu_translate
+task: afrimmlu_translate_zul_prompt_3
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4609d97afdbe27cb5b55da9f057a08a2f73d649
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py
@@ -0,0 +1,32 @@
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """You are a subject matter expert in {subject}.
+
+  Utilizing your expertise in {subject}, answer the following multiple-choice question
+  by picking ''A'', ''B'', ''C'', or ''D''.
+
+Question: {question}
+Choices:
+        A: {choice1}
+        B: {choice2}
+        C: {choice3}
+        D: {choice4}
+Answer: """
+
+    choices = eval(doc["choices"])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
+    return text
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate
new file mode 100644
index 0000000000000000000000000000000000000000..7a974279a3918de90369c391b09de818cb1b483d
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate
@@ -0,0 +1,32 @@
+tag: afrimmlu_tt_tasks
+dataset_path: masakhane/afrimmlu-translate-test
+dataset_name: null
+output_type: multiple_choice
+test_split: test
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
+doc_to_choice: !function utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
+    # aggregation: mean
+    average: weighted
+    hf_evaluate: true
+    higher_is_better: True
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..90c880241df63265e7c8e7a60228163024394e9c
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: amh
+include: afrimmlu_translate
+task: afrimmlu_translate_amh_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c63ccfb49c04b4bfbbbf54c2d6a85d3308a64708
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ewe
+include: afrimmlu_translate
+task: afrimmlu_translate_ewe_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..740f87adbe66021a281f516836558a767ab91c68
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: fra
+include: afrimmlu_translate
+task: afrimmlu_translate_fra_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4f73a2edd89efa810435fa4363fe69a7b0855425
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: hau
+include: afrimmlu_translate
+task: afrimmlu_translate_hau_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cafcae600b0f4f74160353609f68c6fecf055068
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: kin
+include: afrimmlu_translate
+task: afrimmlu_translate_kin_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..618f4aaf77e7694b2265bb1084630c7927460daa
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lin
+include: afrimmlu_translate
+task: afrimmlu_translate_lin_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4f25d96c45abc75c963effa1a0dcba504fdb81ce
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: orm
+include: afrimmlu_translate
+task: afrimmlu_translate_orm_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0943eec102650255d5955fde8b995eea2274c83a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sna
+include: afrimmlu_translate
+task: afrimmlu_translate_sna_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..882117e4c7c10e6b0eabd80531d136189891188f
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: sot
+include: afrimmlu_translate
+task: afrimmlu_translate_sot_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..92928aae81f1bc2f27e99b6e5ad20401990a102f
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: swa
+include: afrimmlu_translate
+task: afrimmlu_translate_swa_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..66161c7dd6a149ab713ae6f92f595b097c6ba794
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: wol
+include: afrimmlu_translate
+task: afrimmlu_translate_wol_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..25ff91f0004a504f48b1ed9c0ea3b000a7c7fcf2
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: xho
+include: afrimmlu_translate
+task: afrimmlu_translate_xho_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..54743fddbc83195064abc456b8dca0ff750c6fe5
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimmlu_translate
+task: afrimmlu_translate_yor_prompt_4
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4860b79c529b23f7827c6b25527e0070b32c36bc
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ewe
+include: afrimmlu_translate
+task: afrimmlu_translate_ewe_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a6cca83cd7eb6b765b5512827044547e01810813
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: ibo
+include: afrimmlu_translate
+task: afrimmlu_translate_ibo_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6acd743a77e5b04b5df5ddf8861984f206697542
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: kin
+include: afrimmlu_translate
+task: afrimmlu_translate_kin_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..24a3b78e2d1c5b1323a335cbe20034a7a7e4b0b7
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: swa
+include: afrimmlu_translate
+task: afrimmlu_translate_swa_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ab3ea80c0610b333c11b67712e6bc5284994bbed
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: twi
+include: afrimmlu_translate
+task: afrimmlu_translate_twi_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2cd495e81df06383612c1278b18faeb0ac5c567f
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: wol
+include: afrimmlu_translate
+task: afrimmlu_translate_wol_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6e227431566f0c5187e0baeeaeb1e82838db0469
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: yor
+include: afrimmlu_translate
+task: afrimmlu_translate_yor_prompt_5
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..147225bb70653d67663ac1762a7cd6246c4e9f22
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py
@@ -0,0 +1,28 @@
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """Given your proficiency in {subject}, please answer the subsequent multiple-choice question with 'A', 'B', 'C', or 'D'.
+Question: {question}
+Choices:
+        A: {choice1}
+        B: {choice2}
+        C: {choice3}
+        D: {choice4}
+Answer: """
+
+    choices = eval(doc["choices"])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
+    return text
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ac18bf6a47de6adbfa36f0bfa92d014f988de358
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lin.yaml	
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lin
+include: afrixnli_en_direct_yaml
+task: afrixnli_en_direct_lin
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6375309f9b72c5eed77057bcb5a08416002647f0
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lug.yaml	
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrixnli_en_direct_yaml
+task: afrixnli_en_direct_lug
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_orm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..eb1c1c4fa09a1528118c84ae18334d5a9492cd8a
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_orm.yaml	
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: orm
+include: afrixnli_en_direct_yaml
+task: afrixnli_en_direct_orm
diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/gen_utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/gen_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..338b4f9daf9e0724b0d87a7f2182fac956e245ca
--- /dev/null
+++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/gen_utils.py
@@ -0,0 +1,129 @@
+import argparse
+import os
+
+import yaml
+
+
+class FunctionTag:
+    def __init__(self, value):
+        self.value = value
+
+
+def prompt_func(mode, lang):
+    prompt_map = {
+        "prompt_1": "Please identify whether the premise entails or contradicts the hypothesis in the following premise "
+        "and hypothesis. The answer should be exact entailment, contradiction, or neutral.\n\nPremise: {premise}\nHypothesis: {hypothesis}\n\n"
+        "Is it entailment, contradiction, or neutral?",
+        "prompt_3": f"Given the following premise and hypothesis in {lang}, identify if the premise entails, contradicts, "
+        f"or is neutral towards the hypothesis. Please respond with exact 'entailment', 'contradiction', or 'neutral'. \n\n"
+        "Premise: {{premise}} \nHypothesis: {{hypothesis}}",
+        "prompt_4": f"You are an expert in Natural Language Inference (NLI) specializing in the {lang} language.\n"
+        f"Analyze the premise and hypothesis given in {lang}, and determine the relationship between them.\n "
+        f"Respond with one of the following options: 'entailment', 'contradiction', or 'neutral'. \n\n"
+        "Premise: {{premise}} \nHypothesis: {{hypothesis}}",
+        "prompt_5": "Based on the given statement, is the following claim 'true', 'false', or 'inconclusive'. \n"
+        "Statement: {{premise}} \nClaim: {{hypothesis}}",
+    }
+    return prompt_map[mode]
+
+
+def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
+    """
+    Generate a yaml file for each language.
+
+    :param output_dir: The directory to output the files to.
+    :param overwrite: Whether to overwrite files if they already exist.
+    """
+    err = []
+    languages = {
+        "eng": "English",
+        "amh": "Amharic",
+        "ibo": "Igbo",
+        "fra": "French",
+        "sna": "chiShona",
+        "wol": "Wolof",
+        "ewe": "Ewe",
+        "lin": "Lingala",
+        "lug": "Luganda",
+        "xho": "isiXhosa",
+        "kin": "Kinyarwanda",
+        "twi": "Twi",
+        "zul": "Zulu",
+        "orm": "Oromo",
+        "yor": "Yoruba",
+        "hau": "Hausa",
+        "sot": "Sesotho",
+        "swa": "Swahili",
+    }
+
+    for lang in languages.keys():
+        try:
+            file_name = f"afrixnli_{lang}.yaml"
+            task_name = f"afrixnli_{lang}_{mode}"
+            yaml_template = "afrixnli_yaml"
+            if output_dir.split("/")[-1] == "translate":
+                file_name = f"afrixnli_translate_{lang}.yaml"
+                task_name = f"afrixnli_translate_{lang}_{mode}"
+                yaml_template = "afrixnli_translate_yaml"
+            if int(mode.split("_")[-1]) == 1 or int(mode.split("_")[-1]) > 2:
+                yaml_details = {
+                    "include": yaml_template,
+                    "task": task_name,
+                    "dataset_name": lang,
+                    "doc_to_text": prompt_func(mode, languages[lang]),
+                }
+            else:
+                yaml_details = {
+                    "include": yaml_template,
+                    "task": task_name,
+                    "dataset_name": lang,
+                }
+            os.makedirs(f"{output_dir}/{mode}", exist_ok=True)
+            with open(
+                f"{output_dir}/{mode}/{file_name}",
+                "w" if overwrite else "x",
+                encoding="utf8",
+            ) as f:
+                f.write("# Generated by utils.py\n")
+                yaml.dump(
+                    yaml_details,
+                    f,
+                    allow_unicode=True,
+                )
+        except FileExistsError:
+            err.append(file_name)
+
+    if len(err) > 0:
+        raise FileExistsError(
+            "Files were not created because they already exist (use --overwrite flag):"
+            f" {', '.join(err)}"
+        )
+
+
+def main() -> None:
+    """Parse CLI args and generate language-specific yaml files."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--overwrite",
+        default=True,
+        action="store_true",
+        help="Overwrite files if they already exist",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="./translate",
+        help="Directory to write yaml files to",
+    )
+    parser.add_argument(
+        "--mode",
+        default="prompt_5",
+        choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"],
+        help="Prompt number",
+    )
+    args = parser.parse_args()
+
+    gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode)
+
+
+if __name__ == "__main__":
+    main()