Add files using upload-large-folder tool

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Russian/include_base_44_russian_health_oriented_education.yaml

@@ -0,0 +1,5 @@
+include: _russian_few_shot_og_template_yaml
+description: Ниже приведены вопросы с несколькими вариантами ответов (с ответами)
+  по теме Health oriented education.
+process_docs: !function 'utils.process_health_oriented_education'
+task: include_base_44_russian_few_shot_og_health_oriented_education

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Russian/include_base_44_russian_social_science.yaml

@@ -0,0 +1,5 @@
+include: _russian_few_shot_og_template_yaml
+description: Ниже приведены вопросы с несколькими вариантами ответов (с ответами)
+  по теме Social Science.
+process_docs: !function 'utils.process_social_science'
+task: include_base_44_russian_few_shot_og_social_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Spanish/_include_base_44_spanish.yaml

@@ -0,0 +1,11 @@
+group: include_base_44_spanish
+task:
+- include_base_44_spanish_few_shot_og_stem
+- include_base_44_spanish_few_shot_og_social_science
+- include_base_44_spanish_few_shot_og_arts_humanities
+- include_base_44_spanish_few_shot_og_health_oriented_education
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Spanish/include_base_44_spanish_arts_humanities.yaml

@@ -0,0 +1,5 @@
+include: _spanish_few_shot_og_template_yaml
+description: Las siguientes son preguntas de opción múltiple (con respuestas) sobre
+  Arts & Humanities.
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_spanish_few_shot_og_arts_humanities

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Spanish/include_base_44_spanish_social_science.yaml

@@ -0,0 +1,5 @@
+include: _spanish_few_shot_og_template_yaml
+description: Las siguientes son preguntas de opción múltiple (con respuestas) sobre
+  Social Science.
+process_docs: !function 'utils.process_social_science'
+task: include_base_44_spanish_few_shot_og_social_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Spanish/include_base_44_spanish_stem.yaml

@@ -0,0 +1,5 @@
+include: _spanish_few_shot_og_template_yaml
+description: Las siguientes son preguntas de opción múltiple (con respuestas) sobre
+  STEM.
+process_docs: !function 'utils.process_stem'
+task: include_base_44_spanish_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Spanish/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tagalog/_include_base_44_tagalog.yaml

@@ -0,0 +1,9 @@
+group: include_base_44_tagalog
+task:
+- include_base_44_tagalog_few_shot_og_arts_humanities
+- include_base_44_tagalog_few_shot_og_driving_license
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tagalog/_tagalog_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Tagalog
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tamil/_include_base_44_tamil.yaml

@@ -0,0 +1,9 @@
+group: include_base_44_tamil
+task:
+- include_base_44_tamil_few_shot_og_stem
+- include_base_44_tamil_few_shot_og_general_knowledge
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tamil/_tamil_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Tamil
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tamil/include_base_44_tamil_general_knowledge.yaml

@@ -0,0 +1,5 @@
+include: _tamil_few_shot_og_template_yaml
+description: பின்வருபவை General knowledge பற்றிய பல தேர்வு கேள்விகள் (பதில்களுடன்)
+  உள்ளன.
+process_docs: !function 'utils.process_general_knowledge'
+task: include_base_44_tamil_few_shot_og_general_knowledge

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tamil/include_base_44_tamil_stem.yaml

@@ -0,0 +1,4 @@
+include: _tamil_few_shot_og_template_yaml
+description: பின்வருபவை STEM பற்றிய பல தேர்வு கேள்விகள் (பதில்களுடன்) உள்ளன.
+process_docs: !function 'utils.process_stem'
+task: include_base_44_tamil_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Tamil/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/_include_base_44_telugu.yaml

@@ -0,0 +1,11 @@
+group: include_base_44_telugu
+task:
+- include_base_44_telugu_few_shot_og_arts_humanities
+- include_base_44_telugu_few_shot_og_applied_science
+- include_base_44_telugu_few_shot_og_stem
+- include_base_44_telugu_few_shot_og_social_science
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/_telugu_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Telugu
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/include_base_44_telugu_applied_science.yaml

@@ -0,0 +1,4 @@
+include: _telugu_few_shot_og_template_yaml
+description: Applied Science గురించిన బహుళ-ఎంపిక ప్రశ్నలు (సమాధానాలతో) క్రిందివి.
+process_docs: !function 'utils.process_applied_science'
+task: include_base_44_telugu_few_shot_og_applied_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/include_base_44_telugu_arts_humanities.yaml

@@ -0,0 +1,4 @@
+include: _telugu_few_shot_og_template_yaml
+description: Arts & Humanities గురించిన బహుళ-ఎంపిక ప్రశ్నలు (సమాధానాలతో) క్రిందివి.
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_telugu_few_shot_og_arts_humanities

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/include_base_44_telugu_social_science.yaml

@@ -0,0 +1,4 @@
+include: _telugu_few_shot_og_template_yaml
+description: Social Science గురించిన బహుళ-ఎంపిక ప్రశ్నలు (సమాధానాలతో) క్రిందివి.
+process_docs: !function 'utils.process_social_science'
+task: include_base_44_telugu_few_shot_og_social_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/include_base_44_telugu_stem.yaml

@@ -0,0 +1,4 @@
+include: _telugu_few_shot_og_template_yaml
+description: STEM గురించిన బహుళ-ఎంపిక ప్రశ్నలు (సమాధానాలతో) క్రిందివి.
+process_docs: !function 'utils.process_stem'
+task: include_base_44_telugu_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Telugu/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/_include_base_44_turkish.yaml

@@ -0,0 +1,11 @@
+group: include_base_44_turkish
+task:
+- include_base_44_turkish_few_shot_og_business_commerce
+- include_base_44_turkish_few_shot_og_stem
+- include_base_44_turkish_few_shot_og_arts_humanities
+- include_base_44_turkish_few_shot_og_social_science
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/_turkish_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Turkish
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/include_base_44_turkish_arts_humanities.yaml

@@ -0,0 +1,5 @@
+include: _turkish_few_shot_og_template_yaml
+description: Aşağıda Arts & Humanities ile ilgili çoktan seçmeli sorular (cevaplarıyla
+  birlikte) bulunmaktadır.
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_turkish_few_shot_og_arts_humanities

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/include_base_44_turkish_business_commerce.yaml

@@ -0,0 +1,5 @@
+include: _turkish_few_shot_og_template_yaml
+description: Aşağıda Business & Commerce ile ilgili çoktan seçmeli sorular (cevaplarıyla
+  birlikte) bulunmaktadır.
+process_docs: !function 'utils.process_business_commerce'
+task: include_base_44_turkish_few_shot_og_business_commerce

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/include_base_44_turkish_social_science.yaml

@@ -0,0 +1,5 @@
+include: _turkish_few_shot_og_template_yaml
+description: Aşağıda Social Science ile ilgili çoktan seçmeli sorular (cevaplarıyla
+  birlikte) bulunmaktadır.
+process_docs: !function 'utils.process_social_science'
+task: include_base_44_turkish_few_shot_og_social_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/include_base_44_turkish_stem.yaml

@@ -0,0 +1,5 @@
+include: _turkish_few_shot_og_template_yaml
+description: Aşağıda STEM ile ilgili çoktan seçmeli sorular (cevaplarıyla birlikte)
+  bulunmaktadır.
+process_docs: !function 'utils.process_stem'
+task: include_base_44_turkish_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Turkish/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Ukrainian/_include_base_44_ukrainian.yaml

@@ -0,0 +1,10 @@
+group: include_base_44_ukrainian
+task:
+- include_base_44_ukrainian_few_shot_og_arts_humanities
+- include_base_44_ukrainian_few_shot_og_social_science
+- include_base_44_ukrainian_few_shot_og_stem
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Ukrainian/_ukrainian_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Ukrainian
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Ukrainian/include_base_44_ukrainian_arts_humanities.yaml

@@ -0,0 +1,5 @@
+include: _ukrainian_few_shot_og_template_yaml
+description: Нижче наведено запитання з кількома варіантами відповідей (з відповідями)
+  про Arts & Humanities.
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_ukrainian_few_shot_og_arts_humanities

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Ukrainian/include_base_44_ukrainian_social_science.yaml

@@ -0,0 +1,5 @@
+include: _ukrainian_few_shot_og_template_yaml
+description: Нижче наведено запитання з кількома варіантами відповідей (з відповідями)
+  про Social Science.
+process_docs: !function 'utils.process_social_science'
+task: include_base_44_ukrainian_few_shot_og_social_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Ukrainian/include_base_44_ukrainian_stem.yaml

@@ -0,0 +1,5 @@
+include: _ukrainian_few_shot_og_template_yaml
+description: Нижче наведено запитання з кількома варіантами відповідей (з відповідями)
+  про STEM.
+process_docs: !function 'utils.process_stem'
+task: include_base_44_ukrainian_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Ukrainian/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Urdu/_include_base_44_urdu.yaml

@@ -0,0 +1,10 @@
+group: include_base_44_urdu
+task:
+- include_base_44_urdu_few_shot_og_stem
+- include_base_44_urdu_few_shot_og_health_oriented_education
+- include_base_44_urdu_few_shot_og_arts_humanities
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Urdu/_urdu_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Urdu
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Urdu/include_base_44_urdu_arts_humanities.yaml

@@ -0,0 +1,5 @@
+include: _urdu_few_shot_og_template_yaml
+description: ذیل میں Arts & Humanities کے بارے میں متعدد انتخابی سوالات (جوابات کے
+  ساتھ) ہیں۔
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_urdu_few_shot_og_arts_humanities

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Urdu/include_base_44_urdu_health_oriented_education.yaml

@@ -0,0 +1,5 @@
+include: _urdu_few_shot_og_template_yaml
+description: ذیل میں Health oriented education کے بارے میں متعدد انتخابی سوالات (جوابات
+  کے ساتھ) ہیں۔
+process_docs: !function 'utils.process_health_oriented_education'
+task: include_base_44_urdu_few_shot_og_health_oriented_education

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Urdu/include_base_44_urdu_stem.yaml

@@ -0,0 +1,4 @@
+include: _urdu_few_shot_og_template_yaml
+description: ذیل میں STEM کے بارے میں متعدد انتخابی سوالات (جوابات کے ساتھ) ہیں۔
+process_docs: !function 'utils.process_stem'
+task: include_base_44_urdu_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Urdu/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/_include_base_44_uzbek.yaml

@@ -0,0 +1,11 @@
+group: include_base_44_uzbek
+task:
+- include_base_44_uzbek_few_shot_og_arts_humanities
+- include_base_44_uzbek_few_shot_og_medical_license
+- include_base_44_uzbek_few_shot_og_social_science
+- include_base_44_uzbek_few_shot_og_stem
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/_uzbek_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Uzbek
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/include_base_44_uzbek_arts_humanities.yaml

@@ -0,0 +1,5 @@
+include: _uzbek_few_shot_og_template_yaml
+description: Quyida Arts & Humanities boʻyicha koʻp tanlovli savollar (javoblari bilan)
+  keltirilgan.
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_uzbek_few_shot_og_arts_humanities

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/include_base_44_uzbek_medical_license.yaml

@@ -0,0 +1,5 @@
+include: _uzbek_few_shot_og_template_yaml
+description: Quyida Medical License boʻyicha koʻp tanlovli savollar (javoblari bilan)
+  keltirilgan.
+process_docs: !function 'utils.process_medical_license'
+task: include_base_44_uzbek_few_shot_og_medical_license

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/include_base_44_uzbek_social_science.yaml

@@ -0,0 +1,5 @@
+include: _uzbek_few_shot_og_template_yaml
+description: Quyida Social Science boʻyicha koʻp tanlovli savollar (javoblari bilan)
+  keltirilgan.
+process_docs: !function 'utils.process_social_science'
+task: include_base_44_uzbek_few_shot_og_social_science

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/include_base_44_uzbek_stem.yaml

@@ -0,0 +1,4 @@
+include: _uzbek_few_shot_og_template_yaml
+description: Quyida STEM boʻyicha koʻp tanlovli savollar (javoblari bilan) keltirilgan.
+process_docs: !function 'utils.process_stem'
+task: include_base_44_uzbek_few_shot_og_stem

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Uzbek/utils.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+
+CATEGORIES = [
+    "Applied Science",
+    "Arts & Humanities",
+    "Business & Commerce",
+    "Driving License",
+    "General knowledge",
+    "Health oriented education",
+    "Marine License",
+    "Medical License",
+    "Professional certification",
+    "STEM",
+    "Social Science",
+]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["domain"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Vietnamese/_include_base_44_vietnamese.yaml

@@ -0,0 +1,10 @@
+group: include_base_44_vietnamese
+task:
+- include_base_44_vietnamese_few_shot_og_stem
+- include_base_44_vietnamese_few_shot_og_arts_humanities
+- include_base_44_vietnamese_few_shot_og_social_science
+aggregate_metric_list:
+- metric: acc
+  weight_by_size: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Vietnamese/_vietnamese_few_shot_og_template_yaml

@@ -0,0 +1,18 @@
+dataset_path: CohereForAI/include-base-44
+dataset_name: Vietnamese
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n
+  D. {{option_d}}\nAnswer:"
+doc_to_choice:
+  - A
+  - B
+  - C
+  - D
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-evaluation-harness/lm_eval/tasks/include/few_shot_og/Vietnamese/include_base_44_vietnamese_arts_humanities.yaml

@@ -0,0 +1,4 @@
+include: _vietnamese_few_shot_og_template_yaml
+description: Sau đây là các câu hỏi trắc nghiệm (có đáp án) về Arts & Humanities.
+process_docs: !function 'utils.process_arts_humanities'
+task: include_base_44_vietnamese_few_shot_og_arts_humanities