diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..890595c135aca9dc9c4da7622003321aa8732cc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_high_school_psychology.yaml @@ -0,0 +1,147 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u090F\u0915 \u092E\u091C\u092C\u0942\u0924 \u0938\u0941\u092A\u0930\u0907\ + \u0917\u094B." + B: "\u0915\u092E \u0906\u0924\u094D\u092E \u0938\u092E\u094D\u092E\u093E\u0928\ + \u0964" + C: "\u0915\u092E \u0906\u0924\u094D\u092E-\u092A\u094D\u0930\u092D\u093E\u0935\ + \u0915\u093E\u0930\u093F\u0924\u093E." + D: "\u0928\u093F\u092F\u0902\u0924\u094D\u0930\u0923 \u0915\u093E \u090F\u0915\ + \ \u0906\u0902\u0924\u0930\u093F\u0915 \u0938\u094D\u0925\u093E\u0928." + input_correct_responses: + - D + input_question: "\u0905\u0928\u0940 \u0915\u093E \u092E\u093E\u0928\u0928\u093E\ + \ \u0939\u0948 \u0915\u093F \u0909\u0938\u0915\u0947 \u0938\u093E\u0925 \u091C\ + \u094B \u0939\u094B\u0924\u093E \u0939\u0948 \u0909\u0938\u092E\u0947\u0902\ + \ \u0909\u0938\u0915\u093E \u0926\u0943\u0937\u094D\u091F\u093F\u0915\u094B\u0923\ + \ \u0914\u0930 \u0935\u094D\u092F\u0935\u0939\u093E\u0930 \u0915\u0947\u0902\ + \u0926\u094D\u0930\u0940\u092F \u092D\u0942\u092E\u093F\u0915\u093E \u0928\u093F\ + \u092D\u093E\u0924\u093E \u0939\u0948\u0964 \u0910\u0938\u0940 \u092E\u093E\u0928\ + \u094D\u092F\u0924\u093E \u091C\u0941\u0921\u093C\u0940 \u0939\u094B\u0928\u0947\ + \ \u0915\u0940 \u0938\u0902\u092D\u093E\u0935\u0928\u093E \u0939\u0948" + - input_choice_list: + A: "\u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u0940 \u0935\u0930\u094D\u0924\ + \u092E\u093E\u0928 \u0938\u092E\u0938\u094D\u092F\u093E\u0913\u0902 \u0915\ + \u0947 \u0915\u093E\u0930\u0923\u094B\u0902 \u0914\u0930 \u0938\u092E\u093E\ + \u0927\u093E\u0928\u094B\u0902 \u0915\u0940 \u092A\u0939\u091A\u093E\u0928\ + \ \u0915\u0930\u0928\u093E" + B: "\u0915\u093F\u0938\u0940 \u0938\u092E\u0938\u094D\u092F\u093E \u0938\u0947\ + \ \u0928\u093F\u092A\u091F\u0928\u0947 \u092E\u0947\u0902 \u092A\u0930\u093E\ + \u092E\u0930\u094D\u0936\u0926\u093E\u0924\u093E \u0915\u0940 \u0915\u0920\ + \u093F\u0928\u093E\u0907\u092F\u094B\u0902 \u0915\u0947 \u0915\u093E\u0930\ + \u0923\u094B\u0902 \u0915\u0940 \u092A\u0939\u091A\u093E\u0928 \u0915\u0930\ + \u0928\u093E \u0914\u0930 \u0909\u0928\u094D\u0939\u0947\u0902 \u0926\u0942\ + \u0930 \u0915\u0930\u0928\u093E" + C: "\u092A\u094D\u0930\u092D\u093E\u0935\u0940 \u0928\u093F\u0930\u094D\u0923\ + \u092F \u0932\u0947\u0928\u0947 \u092E\u0947\u0902 \u0938\u0915\u094D\u0937\ + \u092E \u092C\u0928\u093E\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u092A\ + \u094D\u0930\u093E\u0927\u093F\u0915\u093E\u0930 \u0915\u093E \u090F\u0915\ + \ \u092A\u0926\u093E\u0928\u0941\u0915\u094D\u0930\u092E \u0938\u094D\u0925\ + \u093E\u092A\u093F\u0924 \u0915\u0930\u0928\u093E" + D: "\u0915\u094C\u0936\u0932 \u0915\u0940 \u0915\u092E\u0940 \u0915\u094B \u0926\ + \u0942\u0930 \u0915\u0930\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u0938\ + \u0932\u093E\u0939\u0915\u093E\u0930 \u0915\u0947 \u0932\u093F\u090F \u090F\ + \u0915\u0932, \u0905\u091A\u094D\u091B\u0940 \u0924\u0930\u0939 \u0938\u0947\ + \ \u092A\u0930\u093F\u092D\u093E\u0937\u093F\u0924 \u0914\u0930 \u0938\u094D\ + \u092A\u0937\u094D\u091F \u0915\u093E\u0930\u094D\u0930\u0935\u093E\u0908\ + \ \u0915\u093E \u092A\u093E\u0920\u094D\u092F\u0915\u094D\u0930\u092E \u092A\ + \u094D\u0930\u0938\u094D\u0924\u0941\u0924 \u0915\u0930\u0928\u093E" + input_correct_responses: + - B + input_question: "\u092A\u0930\u093E\u092E\u0930\u094D\u0936\u0926\u093E\u0924\u093E\ + -\u0915\u0947\u0902\u0926\u094D\u0930\u093F\u0924 \u0915\u0947\u0938 \u092A\u0930\ + \u093E\u092E\u0930\u094D\u0936 \u0915\u0947 \u0915\u0948\u092A\u0932\u0928 \u092E\ + \u0949\u0921\u0932 \u0915\u0947 \u0905\u0928\u0941\u0938\u093E\u0930, \u0938\ + \u0932\u093E\u0939\u0915\u093E\u0930 \u092E\u0941\u0916\u094D\u092F \u0930\u0942\ + \u092A \u0938\u0947 \u0907\u0938\u092E\u0947\u0902 \u0930\u0941\u091A\u093F\ + \ \u0930\u0916\u0924\u093E \u0939\u0948" + - input_choice_list: + A: "\u0938\u0902\u0926\u0947\u0936 \u0925\u0948\u0932\u0947\u092E\u0938 \u0938\ + \u0947 \u0938\u0940\u0927\u0947 \u0905\u092E\u093F\u0917\u0921\u093E\u0932\ + \u093E \u0915\u094B \u092D\u0947\u091C\u0947 \u091C\u093E\u0924\u0947 \u0939\ + \u0948\u0902\u0964" + B: "\u0938\u0902\u0926\u0947\u0936 \u0925\u0948\u0932\u0947\u092E\u0938 \u0938\ + \u0947 "\u0915\u094D\u092F\u093E" \u0914\u0930 "\u0915\u0939\ + \u093E\u0902" \u092E\u093E\u0930\u094D\u0917 \u092A\u0930 \u092D\u0947\ + \u091C\u0947 \u091C\u093E\u0924\u0947 \u0939\u0948\u0902\u0964" + C: "\u0938\u0902\u0926\u0947\u0936 \u092A\u0948\u0930\u093E\u0938\u093F\u092E\ + \u094D\u092A\u0947\u0925\u0947\u091F\u093F\u0915 \u0924\u0902\u0924\u094D\u0930\ + \u093F\u0915\u093E \u0924\u0902\u0924\u094D\u0930 \u0938\u0947 \u0938\u0947\ + \u0930\u0947\u092C\u094D\u0930\u0932 \u0915\u0949\u0930\u094D\u091F\u0947\u0915\ + \u094D\u0938 \u0924\u0915 \u092D\u0947\u091C\u0947 \u091C\u093E\u0924\u0947\ + \ \u0939\u0948\u0902\u0964" + D: "\u0938\u0902\u0926\u0947\u0936 \u0932\u0932\u093E\u091F \u0932\u094B\u092C\ + \ \u0938\u0947 \u092A\u093F\u091F\u094D\u092F\u0942\u091F\u0930\u0940 \u0917\ + \u094D\u0930\u0902\u0925\u093F \u0924\u0915 \u092D\u0947\u091C\u0947 \u091C\ + \u093E\u0924\u0947 \u0939\u0948\u0902\u0964" + input_correct_responses: + - A + input_question: "\u0938\u092E\u0941\u0926\u094D\u0930 \u092E\u0947\u0902 \u0924\ + \u0948\u0930\u0924\u0947 \u0938\u092E\u092F, \u0907\u0935\u093E\u0928 \u092A\ + \u093E\u0928\u0940 \u092E\u0947\u0902 \u090F\u0915 \u0905\u0902\u0927\u0947\u0930\ + \u0940 \u091B\u093E\u092F\u093E \u0938\u0947 \u0921\u0930 \u091C\u093E\u0924\ + \u093E \u0939\u0948, \u0907\u0938\u0938\u0947 \u092A\u0939\u0932\u0947 \u0915\ + \u093F \u0909\u0938\u0947 \u092F\u0939 \u092A\u0939\u091A\u093E\u0928\u0928\u0947\ + \ \u0915\u093E \u092E\u094C\u0915\u093E \u092E\u093F\u0932\u0947 \u0915\u093F\ + \ \u0935\u0939 \u091B\u093E\u092F\u093E \u0915\u094D\u092F\u093E \u0939\u0948\ + \u0964 \u092D\u092F \u0915\u0940 \u0907\u0938 \u0918\u091F\u0928\u093E \u0915\ + \u0947 \u0926\u094C\u0930\u093E\u0928 \u0939\u094B\u0928\u0947 \u0935\u093E\u0932\ + \u0947 \u0938\u093F\u0928\u0948\u092A\u094D\u091F\u093F\u0915 \u0915\u0928\u0947\ + \u0915\u094D\u0936\u0928\u094B\u0902 \u0915\u094B \u0928\u093F\u092E\u094D\u0928\ + \u0932\u093F\u0916\u093F\u0924 \u092E\u0947\u0902 \u0938\u0947 \u0915\u093F\u0938\ + \u0915\u0947 \u0926\u094D\u0935\u093E\u0930\u093E \u0938\u092C\u0938\u0947 \u0905\ + \u091A\u094D\u091B\u093E \u0935\u0930\u094D\u0923\u0928 \u0915\u093F\u092F\u093E\ + \ \u0917\u092F\u093E \u0939\u0948?" + - input_choice_list: + A: "\u092C\u091A\u094D\u091A\u0947 \u0915\u094B \u0928\u090F \u0935\u093E\u0924\ + \u093E\u0935\u0930\u0923 \u092E\u0947\u0902 \u092A\u0930\u0940\u0915\u094D\ + \u0937\u0923 \u0905\u0935\u0927\u093F \u0926\u0947\u0902" + B: "\u092E\u093E\u0924\u093E-\u092A\u093F\u0924\u093E \u0915\u094B \u0932\u093F\ + \u0916\u093F\u0924 \u0930\u0942\u092A \u0938\u0947 \u0938\u0942\u091A\u093F\ + \u0924 \u0915\u0930\u0947\u0902" + C: "\u0938\u094D\u0915\u0942\u0932 \u092C\u094B\u0930\u094D\u0921 \u0915\u0940\ + \ \u092E\u0902\u091C\u0942\u0930\u0940 \u092A\u094D\u0930\u093E\u092A\u094D\ + \u0924 \u0915\u0930\u0947\u0902" + D: "\u092E\u093E\u0924\u093E-\u092A\u093F\u0924\u093E \u0915\u0940 \u0938\u0939\ + \u092E\u0924\u093F \u092A\u094D\u0930\u093E\u092A\u094D\u0924 \u0915\u0930\ + \u0947\u0902" + input_correct_responses: + - B + input_question: "\u0935\u093F\u0915\u0932\u093E\u0902\u0917 \u0935\u094D\u092F\ + \u0915\u094D\u0924\u093F \u0936\u093F\u0915\u094D\u0937\u093E \u0938\u0941\u0927\ + \u093E\u0930 \u0905\u0927\u093F\u0928\u093F\u092F\u092E \u0915\u0947 \u0905\u0928\ + \u0941\u0938\u093E\u0930, \u090F\u0915 \u0936\u0948\u0915\u094D\u0937\u093F\u0915\ + \ \u090F\u091C\u0947\u0902\u0938\u0940 \u0915\u094B \u0935\u093F\u0915\u0932\ + \u093E\u0902\u0917 \u091B\u093E\u0924\u094D\u0930 \u0915\u0947 \u0936\u0948\u0915\ + \u094D\u0937\u093F\u0915 \u092A\u094D\u0932\u0947\u0938\u092E\u0947\u0902\u091F\ + \ \u0915\u094B \u092C\u0926\u0932\u0928\u0947 \u0938\u0947 \u092A\u0939\u0932\ + \u0947 \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u092E\u0947\ + \u0902 \u0938\u0947 \u0915\u094D\u092F\u093E \u0915\u0930\u0928\u093E \u091A\ + \u093E\u0939\u093F\u090F?" + - input_choice_list: + A: "\u0938\u093E\u092E\u093E\u091C\u093F\u0915-\u0938\u093E\u0902\u0938\u094D\ + \u0915\u0943\u0924\u093F\u0915" + B: "\u0915\u094D\u0932\u0940\u0928\u093F\u0915\u0932" + C: "\u0938\u0902\u091C\u094D\u091E\u093E\u0928\u093E\u0924\u094D\u092E\u0915" + D: "\u0935\u094D\u092F\u0935\u0939\u093E\u0930\u0935\u093E\u0926\u0940" + input_correct_responses: + - C + input_question: "\u092A\u093E\u0938\u094D\u0915\u0932 \u0909\u0928 \u092A\u094D\ + \u0930\u0938\u0902\u0938\u094D\u0915\u0930\u0923 \u0930\u0923\u0928\u0940\u0924\ + \u093F\u092F\u094B\u0902 \u092E\u0947\u0902 \u0930\u0941\u091A\u093F \u0930\u0916\ + \u0924\u093E \u0939\u0948 \u091C\u093F\u0928\u0915\u093E \u0909\u092A\u092F\u094B\ + \u0917 \u092C\u091A\u094D\u091A\u0947 \u0928\u0908 \u091C\u093E\u0928\u0915\u093E\ + \u0930\u0940 \u0938\u0940\u0916\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F\ + \ \u0915\u0930\u0924\u0947 \u0939\u0948\u0902\u0964 \u092A\u093E\u0938\u094D\ + \u0915\u0932 \u0915\u094B \u0915\u093F\u0938 \u092A\u094D\u0930\u0915\u093E\u0930\ + \ \u0915\u0947 \u092E\u0928\u094B\u0935\u0948\u091C\u094D\u091E\u093E\u0928\u093F\ + \u0915 \u0915\u0947 \u0930\u0942\u092A \u092E\u0947\u0902 \u0935\u0930\u094D\ + \u0917\u0940\u0915\u0943\u0924 \u0915\u093F\u092F\u093E \u091C\u093E\u090F\u0917\ + \u093E?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_psychology +tag: mmlu_hi_llama_social_sciences_tasks +task: mmlu_hi_llama_high_school_psychology +task_alias: high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4896a5a0ac9267513614033aeb450fdd8dba821e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_prehistory.yaml @@ -0,0 +1,142 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0936\u0915\u094D\u0924\u093F\u0936\u093E\u0932\u0940 \u092E\u093E\u092F\ + \u093E \u0916\u0917\u094B\u0932\u0936\u093E\u0938\u094D\u0924\u094D\u0930\u0940\ + \ \u092A\u0941\u091C\u093E\u0930\u093F\u092F\u094B\u0902 \u0915\u094B \u0938\ + \u0902\u0924\u0941\u0937\u094D\u091F \u0915\u0930\u0947\u0902\u0964" + B: "\u0906\u092E \u0932\u094B\u0917\u094B\u0902 \u0915\u0947 \u092A\u094D\u0930\ + \u0924\u093F \u0905\u092A\u0928\u0940 \u0909\u0926\u093E\u0930\u0924\u093E\ + \ \u092A\u094D\u0930\u0926\u0930\u094D\u0936\u093F\u0924 \u0915\u0930\u0947\ + \u0902, \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u0909\u0928\u094D\u0939\ + \u0947\u0902 \u092E\u0902\u0926\u093F\u0930\u094B\u0902 \u092E\u0947\u0902\ + \ \u0930\u0939\u0928\u0947 \u0915\u0940 \u0905\u0928\u0941\u092E\u0924\u093F\ + \ \u0925\u0940\u0964" + C: "\u0936\u0924\u094D\u0930\u0941\u0913\u0902 \u0915\u094B, \u0935\u093F\u0936\ + \u0947\u0937\u0915\u0930 \u0938\u094D\u092A\u0947\u0928\u093F\u092F\u094B\u0902\ + \ \u0915\u094B, \u0921\u0930\u093E \u0926\u0947\u0902\u0964" + D: "\u0909\u0938\u0915\u0947 \u0930\u093E\u091C\u0924\u094D\u0935 \u0915\u094B\ + \ \u0935\u0948\u0927 \u092C\u0928\u093E\u0928\u093E, \u0915\u094D\u092F\u094B\ + \u0902\u0915\u093F \u0909\u0938\u0915\u0947 \u092A\u093F\u0924\u093E \u0930\ + \u093E\u091C\u0938\u0940 \u0928\u0939\u0940\u0902 \u0925\u0947\u0964" + input_correct_responses: + - D + input_question: "\u092E\u0939\u093E\u0928 \u092E\u093E\u092F\u093E \u0930\u093E\ + \u091C\u093E \u092A\u0948\u0915\u0932 \u0928\u0947 \u092A\u0948\u0932\u0947\u0928\ + \u0915 \u0936\u0939\u0930 \u092E\u0947\u0902 \u092E\u0902\u0926\u093F\u0930\u094B\ + \u0902 \u0915\u093E \u0928\u093F\u0930\u094D\u092E\u093E\u0923 \u0915\u0930\u0935\ + \u093E\u092F\u093E:" + - input_choice_list: + A: "\u092A\u094D\u0930\u093E\u0930\u0902\u092D\u093F\u0915 \u0930\u093E\u091C\ + \u094D\u092F\u094B\u0902 \u0915\u0947 \u0909\u0926\u092F \u0915\u0947 \u0938\ + \u092E\u093E\u0928 \u0938\u094D\u0925\u093F\u0924\u093F\u092F\u094B\u0902\ + \ \u0935\u093E\u0932\u093E \u092E\u093F\u0938\u093F\u0938\u093F\u092A\u093F\ + \u092F\u093E\u0908 \u0938\u092D\u094D\u092F\u0924\u093E \u0915\u093E \u090F\ + \u0915 \u0915\u0947\u0902\u0926\u094D\u0930\u0964" + B: "\u0938\u092E\u0924\u093E\u0935\u093E\u0926\u0940 \u0935\u0928\u0935\u093E\ + \u0938\u093F\u092F\u094B\u0902 \u0915\u0947 \u092E\u0942\u0932 \u0905\u092E\ + \u0947\u0930\u093F\u0915\u0940 \u0938\u092E\u093E\u091C \u092E\u0947\u0902\ + \ \u0905\u0927\u093F\u0915\u093E\u0930 \u0915\u0940 \u0938\u0940\u092E\u093E\ + \u090F\u0901\u0964" + C: "1500 \u0908. \u0924\u0915 \u090F\u0915 \u0938\u093E\u0927\u093E\u0930\u0923\ + \ \u092E\u0941\u0916\u093F\u092F\u093E\u0924\u0902\u0924\u094D\u0930 \u092F\ + \u093E \u0936\u093E\u092F\u0926 \u090F\u0915 \u091C\u091F\u093F\u0932 \u092E\ + \u0941\u0916\u093F\u092F\u093E\u0924\u0902\u0924\u094D\u0930 \u0935\u093F\u0915\ + \u0938\u093F\u0924 \u0939\u094B \u091A\u0941\u0915\u093E \u0925\u093E\u0964" + D: "\u092E\u093F\u0938\u093F\u0938\u093F\u092A\u093F\u092F\u093E\u0908 \u0938\ + \u092D\u094D\u092F\u0924\u093E \u0915\u093E \u090F\u0915 \u0915\u0947\u0902\ + \u0926\u094D\u0930 \u091C\u093F\u0938\u0915\u0940 \u0938\u094D\u0925\u093F\ + \u0924\u093F\u092F\u093E\u0901 \u0909\u0924\u094D\u0924\u0930\u0940 \u0905\ + \u092E\u0947\u0930\u093F\u0915\u093E \u0915\u0947 \u0909\u0924\u094D\u0924\ + \u0930-\u092A\u0936\u094D\u091A\u093F\u092E\u0940 \u0924\u091F \u0915\u0947\ + \ \u0938\u092E\u093E\u091C\u094B\u0902 \u0915\u0947 \u0938\u092E\u093E\u0928\ + \ \u0939\u0948\u0902\u0964" + input_correct_responses: + - A + input_question: "\u091F\u093F\u092E\u094B\u0925\u0940 \u092A\u0949\u0915\u0947\ + \u091F\u091F \u0915\u0947 \u0905\u0928\u0941\u0938\u093E\u0930, \u0915\u093E\ + \u0939\u094B\u0915\u093F\u092F\u093E \u092E\u0947\u0902 \u0938\u093E\u092E\u093E\ + \u091C\u093F\u0915 \u0938\u094D\u0924\u0930\u0940\u0915\u0930\u0923 \u0914\u0930\ + \ \u0930\u093E\u091C\u0928\u0940\u0924\u093F\u0915 \u0936\u0915\u094D\u0924\u093F\ + \ \u0915\u0947 \u0938\u093E\u0915\u094D\u0937\u094D\u092F \u092C\u0924\u093E\ + \u0924\u0947 \u0939\u0948\u0902:" + - input_choice_list: + A: "\u0915\u093F\u0938\u0940 \u092A\u094D\u0930\u0915\u093E\u0930 \u0915\u0940\ + \ \u092A\u094D\u0930\u0932\u092F, \u091C\u0948\u0938\u0947 \u092D\u0942\u0915\ + \u0902\u092A, \u091C\u094D\u0935\u093E\u0932\u093E\u092E\u0941\u0916\u0940\ + \ \u092F\u093E \u0938\u0941\u0928\u093E\u092E\u0940\u0964" + B: "\u0915\u093E\u091F \u0915\u0930 \u091C\u0932\u093E\u0913 \u0915\u0943\u0937\ + \u093F \u0924\u0915\u0928\u0940\u0915\u094B\u0902 \u0915\u0947 \u092A\u0930\ + \u093F\u0923\u093E\u092E\u0938\u094D\u0935\u0930\u0942\u092A \u092A\u093E\u0930\ + \u093F\u0938\u094D\u0925\u093F\u0924\u093F\u0915 \u0915\u094D\u0937\u0930\u0923\ + \u0964" + C: "\u092A\u0921\u093C\u094B\u0938\u0940 \u092E\u093E\u092F\u093E \u0936\u0939\ + \u0930-\u0930\u093E\u091C\u094D\u092F\u094B\u0902 \u0915\u0947 \u092C\u0940\ + \u091A \u0905\u0902\u0924\u0939\u0940\u0928 \u092F\u0941\u0926\u094D\u0927\ + \u0964" + D: "\u0905\u0902\u0924\u0930-\u092A\u094D\u0930\u091C\u0928\u0928 \u0915\u0940\ + \ \u092A\u094D\u0930\u0925\u093E\u0913\u0902 \u0915\u0947 \u0915\u093E\u0930\ + \u0923 \u091C\u0928\u094D\u092E\u091C\u093E\u0924 \u0935\u093F\u0915\u093E\ + \u0930\u094B\u0902 \u092E\u0947\u0902 \u092D\u093E\u0930\u0940 \u0935\u0943\ + \u0926\u094D\u0927\u093F \u0939\u0941\u0908\u0964" + input_correct_responses: + - B + input_question: "\u0936\u094B\u0927\u0915\u0930\u094D\u0924\u093E \u0905\u092C\ + \ \u092E\u093E\u0928\u0924\u0947 \u0939\u0948\u0902 \u0915\u093F \u092E\u093E\ + \u092F\u093E \u0915\u093E \u092A\u0924\u0928 \u092E\u0941\u0916\u094D\u092F\u0924\ + \u0903 \u0928\u093F\u092E\u094D\u0928 \u0915\u093E\u0930\u0923\u094B\u0902 \u0938\ + \u0947 \u0939\u0941\u0906:" + - input_choice_list: + A: "\u092C\u0921\u093C\u0940 \u092E\u093E\u0924\u094D\u0930\u093E \u092E\u0947\ + \u0902 \u092A\u094D\u0930\u091C\u093E\u0924\u093F\u092F\u094B\u0902 \u0915\ + \u0940 \u0935\u093F\u0935\u093F\u0927\u0924\u093E, \u092F\u093E \u090F\u0915\ + \ \u0939\u0940 \u092A\u094D\u0930\u091C\u093E\u0924\u093F \u091C\u093F\u0938\ + \u0928\u0947 \u092C\u0939\u0941\u0924 \u0905\u0927\u093F\u0915 \u0935\u093F\ + \u0935\u093F\u0927\u0924\u093E \u092A\u094D\u0930\u0926\u0930\u094D\u0936\u093F\ + \u0924 \u0915\u0940\u0964" + B: "\u0907\u0938 \u0905\u0935\u0927\u093F \u0915\u0947 \u0926\u094C\u0930\u093E\ + \u0928 \u092A\u094D\u0930\u091C\u093E\u0924\u093F\u092F\u094B\u0902 \u0915\ + \u0940 \u0935\u093F\u0935\u093F\u0927\u0924\u093E \u092C\u0939\u0941\u0924\ + \ \u0915\u092E \u0925\u0940 \u0914\u0930 \u0939\u094B\u092E\u093F\u0928\u093F\ + \u0921 \u092D\u0940 \u092C\u0939\u0941\u0924 \u0915\u092E \u0925\u0947\u0964" + C: "\u0932\u0902\u092C\u0947 \u0938\u092E\u092F \u0924\u0915 \u0939\u093F\u092E\ + \u092F\u0941\u0917 \u0914\u0930 \u0909\u0938\u0915\u0947 \u092C\u093E\u0926\ + \ \u092D\u092F\u0902\u0915\u0930 \u0938\u0942\u0916\u0947 \u0915\u0947 \u0915\ + \u093E\u0930\u0923 \u092A\u094D\u0930\u091C\u093E\u0924\u093F\u092F\u094B\u0902\ + \ \u0915\u0940 \u0935\u093F\u0935\u093F\u0927\u0924\u093E \u092E\u0947\u0902\ + \ \u0915\u092E\u0940 \u0906\u0908\u0964" + D: "\u092A\u094D\u0930\u091C\u093E\u0924\u093F\u092F\u094B\u0902 \u0915\u0940\ + \ \u0935\u093F\u0935\u093F\u0927\u0924\u093E \u092E\u0947\u0902 \u0915\u092E\ + \u0940 \u0906\u0908 \u0932\u0947\u0915\u093F\u0928 \u0939\u0925\u094C\u0921\ + \u093C\u0947 \u0915\u0947 \u092A\u0924\u094D\u0925\u0930\u094B\u0902 \u0914\ + \u0930 \u0917\u0941\u091A\u094D\u091B\u094B\u0902 \u0915\u0940 \u0938\u0902\ + \u0916\u094D\u092F\u093E \u092E\u0947\u0902 \u0935\u0943\u0926\u094D\u0927\ + \u093F \u0939\u0941\u0908, \u091C\u094B \u092A\u0924\u094D\u0925\u0930 \u0915\ + \u0947 \u0914\u091C\u093E\u0930\u094B\u0902 \u0915\u0947 \u0928\u093F\u0930\ + \u094D\u092E\u093E\u0923 \u0915\u093E \u0938\u0902\u0915\u0947\u0924 \u0939\ + \u0948\u0964" + input_correct_responses: + - A + input_question: "\u092E\u0927\u094D\u092F \u092A\u094D\u0932\u093F\u092F\u094B\ + \u0938\u0940\u0928 \u0938\u0947 \u0938\u0902\u092C\u0902\u0927\u093F\u0924 \u0939\ + \u094B\u092E\u093F\u0928\u093F\u0921 \u092A\u094D\u0930\u091C\u093E\u0924\u093F\ + \u092F\u094B\u0902 \u092A\u0930 \u0939\u093E\u0932\u093F\u092F\u093E \u0936\u094B\ + \u0927 \u0938\u0947 \u0938\u0902\u0915\u0947\u0924 \u092E\u093F\u0932\u0924\u093E\ + \ \u0939\u0948 \u0915\u093F (2020 \u0924\u0915):" + - input_choice_list: + A: "650 \u0938\u0940\u0938\u0940 \u0938\u0947 \u0915\u092E" + B: "\u0932\u0917\u092D\u0917 800 \u0938\u0940.\u0938\u0940" + C: "\u092C\u0938 1000 \u0938\u0940\u0938\u0940 \u0938\u0947 \u0915\u092E" + D: "1200 \u0938\u0940.\u0938\u0940" + input_correct_responses: + - C + input_question: "\u0939\u094B\u092E\u094B \u0907\u0930\u0947\u0915\u094D\u091F\ + \u0938 \u0915\u0940 \u0905\u0928\u0941\u092E\u093E\u0928\u093F\u0924 \u0914\u0938\ + \u0924 \u0915\u092A\u093E\u0932 \u0915\u094D\u0937\u092E\u0924\u093E \u0915\u094D\ + \u092F\u093E \u0939\u0948?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_prehistory +tag: mmlu_hi_llama_humanities_tasks +task: mmlu_hi_llama_prehistory +task_alias: prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffd2a0e2e40873e77e5411aebb49248871c06458 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_accounting.yaml @@ -0,0 +1,152 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: $70,000 + B: $75,000 + C: $80,000 + D: '100000' + input_correct_responses: + - D + input_question: "\u092C\u0949\u0915\u094D\u0938 \u090F\u0915 \u0917\u0948\u0930\ + -\u0938\u0930\u0915\u093E\u0930\u0940 \u0917\u0948\u0930-\u0932\u093E\u092D\u0915\ + \u093E\u0930\u0940 \u0938\u0902\u0917\u0920\u0928 \u0928\u0947 \u0935\u0930\u094D\ + \u0937 \u0915\u0947 \u0926\u094C\u0930\u093E\u0928 \u0928\u093F\u092E\u094D\u0928\ + \u0932\u093F\u0916\u093F\u0924 \u0932\u0947\u0928\u0926\u0947\u0928 \u0915\u093F\ + \u090F: \u0928\u093F\u0935\u0947\u0936 \u0915\u0940 \u092C\u093F\u0915\u094D\ + \u0930\u0940 \u0938\u0947 \u092A\u094D\u0930\u093E\u092A\u094D\u0924 \u0906\u092F\ + \ $80000 \u0938\u0902\u092A\u0924\u094D\u0924\u093F \u0938\u0902\u092F\u0902\ + \u0924\u094D\u0930 \u0914\u0930 \u0909\u092A\u0915\u0930\u0923 \u0915\u0940\ + \ \u0916\u0930\u0940\u0926 $10000 \u0926\u0940\u0930\u094D\u0918\u0915\u093E\ + \u0932\u093F\u0915 \u090B\u0923 \u0938\u0947 \u0906\u092F $100000 \u0928\u093F\ + \u0935\u0947\u0936 \u0915\u0940 \u092C\u093F\u0915\u094D\u0930\u0940 \u092A\u0930\ + \ \u0939\u093E\u0928\u093F $5000 \u0915\u093F\u0924\u0928\u0940 \u0930\u093E\ + \u0936\u093F \u0915\u094B \u0936\u0941\u0926\u094D\u0927 \u0915\u0947 \u0930\ + \u0942\u092A \u092E\u0947\u0902 \u0930\u093F\u092A\u094B\u0930\u094D\u091F \u0915\ + \u093F\u092F\u093E \u091C\u093E\u0928\u093E \u091A\u093E\u0939\u093F\u090F \u092C\ + \u0949\u0915\u094D\u0938 \u0915\u0947 \u0928\u0915\u0926\u0940 \u092A\u094D\u0930\ + \u0935\u093E\u0939 \u0935\u093F\u0935\u0930\u0923 \u092E\u0947\u0902 \u0935\u093F\ + \u0924\u094D\u0924\u0940\u092F \u0917\u0924\u093F\u0935\u093F\u0927\u093F\u092F\ + \u094B\u0902 \u0926\u094D\u0935\u093E\u0930\u093E \u092A\u094D\u0930\u0926\u093E\ + \u0928 \u0915\u0940 \u0917\u0908 \u0928\u0915\u0926\u0940?" + - input_choice_list: + A: $13,000 + B: $600 + C: $15,000 + D: $28,000 + input_correct_responses: + - A + input_question: "\u0938\u094C \u0938\u093E\u0932 \u092A\u0939\u0932\u0947, \u0906\ + \u092A\u0915\u0940 \u092A\u0930\u0926\u093E\u0926\u0940 \u0928\u0947 5% \u0935\ + \u093E\u0930\u094D\u0937\u093F\u0915 \u092C\u094D\u092F\u093E\u091C \u092A\u0930\ + \ 100 \u0921\u0949\u0932\u0930 \u0915\u093E \u0928\u093F\u0935\u0947\u0936 \u0915\ + \u093F\u092F\u093E \u0925\u093E\u0964 \u0906\u091C \u0928\u093F\u0935\u0947\u0936\ + \ \u0915\u093E \u092E\u0942\u0932\u094D\u092F \u0915\u094D\u092F\u093E \u0939\ + \u0948?" + - input_choice_list: + A: $0 + B: $500 + C: $1,650 + D: $16,500 + input_correct_responses: + - A + input_question: "\u0915\u094D\u0930\u0947\u0924\u0947 \u090F\u0915 \u0905\u0935\ + \u093F\u0935\u093E\u0939\u093F\u0924 \u0915\u0930\u0926\u093E\u0924\u093E \u0939\ + \u0948 \u091C\u093F\u0938\u0915\u0940 \u0906\u092F \u0935\u093F\u0936\u0947\u0937\ + \ \u0930\u0942\u092A \u0938\u0947 \u092E\u091C\u0926\u0942\u0930\u0940 \u0938\ + \u0947 \u0939\u094B\u0924\u0940 \u0939\u0948\u0964 31 \u0926\u093F\u0938\u0902\ + \u092C\u0930, \u0935\u0930\u094D\u0937 1 \u0924\u0915, \u0915\u094D\u0930\u0947\ + \u0924\u0947 \u0915\u0947 \u0928\u093F\u092F\u094B\u0915\u094D\u0924\u093E \u0928\ + \u0947 \u0938\u0902\u0918\u0940\u092F \u0906\u092F \u0915\u0930\u094B\u0902\ + \ \u092E\u0947\u0902 $16,000 \u0930\u094B\u0915 \u0932\u093F\u092F\u093E \u0939\ + \u0948 \u0914\u0930 \u0915\u094D\u0930\u0947\u0924\u0947 \u0928\u0947 \u0915\ + \u094B\u0908 \u0905\u0928\u0941\u092E\u093E\u0928\u093F\u0924 \u0915\u0930 \u092D\ + \u0941\u0917\u0924\u093E\u0928 \u0928\u0939\u0940\u0902 \u0915\u093F\u092F\u093E\ + \ \u0939\u0948\u0964 15 \u0905\u092A\u094D\u0930\u0948\u0932, \u0935\u0930\u094D\ + \u0937 2 \u0915\u094B, \u0915\u094D\u0930\u0947\u0924\u0947 \u0928\u0947 \u0905\ + \u092A\u0928\u093E \u0935\u094D\u092F\u0915\u094D\u0924\u093F\u0917\u0924 \u0915\ + \u0930 \u0930\u093F\u091F\u0930\u094D\u0928 \u0926\u093E\u0916\u093F\u0932 \u0915\ + \u0930\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u0938\u092E\u092F \u092A\ + \u0930 \u0935\u093F\u0938\u094D\u0924\u093E\u0930 \u0905\u0928\u0941\u0930\u094B\ + \u0927 \u0926\u093E\u092F\u0930 \u0915\u093F\u092F\u093E, \u0914\u0930 $300\ + \ \u0905\u0924\u093F\u0930\u093F\u0915\u094D\u0924 \u0915\u0930 \u0915\u093E\ + \ \u092D\u0941\u0917\u0924\u093E\u0928 \u0915\u093F\u092F\u093E\u0964 \u0915\ + \u094D\u0930\u0947\u0924\u0947 \u0915\u0940 \u0935\u0930\u094D\u0937 1 \u0915\ + \u0940 \u0915\u0930 \u0926\u0947\u092F\u0924\u093E $16,500 \u0925\u0940 \u091C\ + \u092C \u0909\u0938\u0928\u0947 \u0938\u092E\u092F \u092A\u0930 30 \u0905\u092A\ + \u094D\u0930\u0948\u0932, \u0935\u0930\u094D\u0937 2 \u0915\u094B \u0905\u092A\ + \u0928\u093E \u0930\u093F\u091F\u0930\u094D\u0928 \u0926\u093E\u0916\u093F\u0932\ + \ \u0915\u093F\u092F\u093E \u0914\u0930 \u0936\u0947\u0937 \u0915\u0930 \u0926\ + \u0947\u092F\u0924\u093E \u0915\u093E \u092D\u0941\u0917\u0924\u093E\u0928 \u0915\ + \u093F\u092F\u093E\u0964 \u0905\u0928\u0941\u092E\u093E\u0928\u093F\u0924 \u0915\ + \u0930\u094B\u0902 \u0915\u0947 \u0915\u092E \u092D\u0941\u0917\u0924\u093E\u0928\ + \ \u0915\u0947 \u0932\u093F\u090F \u0915\u093F\u0924\u0928\u0940 \u0930\u093E\ + \u0936\u093F \u091C\u0941\u0930\u094D\u092E\u093E\u0928\u0947 \u0915\u0947 \u0905\ + \u0927\u0940\u0928 \u0939\u094B\u0917\u0940?" + - input_choice_list: + A: $5,000 + B: $13,500 + C: $16,000 + D: $20,000 + input_correct_responses: + - B + input_question: "\u091C\u0928\u0935\u0930\u0940 1, \u0935\u0930\u094D\u0937 1\ + \ \u0915\u094B, \u0905\u0932\u094D\u092B\u093E \u0915\u0902\u092A\u0928\u0940\ + \ \u0928\u0947 \u090F\u0915 \u0938\u0949\u092B\u094D\u091F\u0935\u0947\u092F\ + \u0930 \u092A\u094D\u0930\u0926\u093E\u0924\u093E \u0915\u0947 \u0938\u093E\u0925\ + \ 15,000 \u0921\u0949\u0932\u0930 \u0915\u0947 \u0935\u093E\u0930\u094D\u0937\ + \u093F\u0915 \u0930\u0916\u0930\u0916\u093E\u0935 \u0938\u092E\u091D\u094C\u0924\ + \u0947 \u092A\u0930 \u0939\u0938\u094D\u0924\u093E\u0915\u094D\u0937\u0930 \u0915\ + \u093F\u090F \u0914\u0930 \u0930\u0916\u0930\u0916\u093E\u0935 \u0915\u0940\ + \ \u0905\u0935\u0927\u093F 1 \u092E\u093E\u0930\u094D\u091A, \u0935\u0930\u094D\ + \u0937 2 \u0938\u0947 \u0936\u0941\u0930\u0942 \u0939\u094B\u0924\u0940 \u0939\ + \u0948\u0964 \u0905\u0932\u094D\u092B\u093E \u0928\u0947 1 \u091C\u0928\u0935\ + \u0930\u0940, \u0935\u0930\u094D\u0937 1 \u0915\u094B \u0938\u0949\u092B\u094D\ + \u091F\u0935\u0947\u092F\u0930 \u0938\u0902\u0936\u094B\u0927\u0928 \u0938\u0947\ + \ \u0938\u0902\u092C\u0902\u0927\u093F\u0924 $5,000 \u0915\u0940 \u0932\u093E\ + \u0917\u0924 \u092D\u0940 \u0916\u0930\u094D\u091A \u0915\u0940\u0964 \u0905\ + \u0928\u0941\u0930\u094B\u0927 \u091C\u094B \u0938\u0949\u092B\u093C\u094D\u091F\ + \u0935\u0947\u092F\u0930 \u0915\u0940 \u0915\u093E\u0930\u094D\u092F\u0915\u094D\ + \u0937\u092E\u0924\u093E \u092C\u0922\u093C\u093E\u090F\u0902\u0917\u0947\u0964\ + \ \u0905\u0932\u094D\u092B\u093E \u0938\u094D\u091F\u094D\u0930\u0947\u091F\ + -\u0932\u093E\u0907\u0928 \u0935\u093F\u0927\u093F \u0915\u093E \u0909\u092A\ + \u092F\u094B\u0917 \u0915\u0930\u0915\u0947 \u092A\u093E\u0902\u091A \u0935\u0930\ + \u094D\u0937\u094B\u0902 \u092E\u0947\u0902 \u0905\u092A\u0928\u0947 \u0915\u0902\ + \u092A\u094D\u092F\u0942\u091F\u0930 \u0914\u0930 \u0938\u0949\u092B\u094D\u091F\ + \u0935\u0947\u092F\u0930 \u092A\u0930\u093F\u0938\u0902\u092A\u0924\u094D\u0924\ + \u093F\u092F\u094B\u0902 \u0915\u093E \u092E\u0942\u0932\u094D\u092F\u0939\u094D\ + \u0930\u093E\u0938 \u0914\u0930 \u092A\u0930\u093F\u0936\u094B\u0927\u0928 \u0915\ + \u0930\u0924\u093E \u0939\u0948\u0964 31 \u0926\u093F\u0938\u0902\u092C\u0930\ + , \u0935\u0930\u094D\u0937 1 \u0915\u094B \u0938\u092E\u093E\u092A\u094D\u0924\ + \ \u0935\u0930\u094D\u0937 \u0915\u0947 \u0932\u093F\u090F \u0930\u0916\u0930\ + \u0916\u093E\u0935 \u0938\u092E\u091D\u094C\u0924\u0947 \u0914\u0930 \u0938\u0949\ + \u092B\u093C\u094D\u091F\u0935\u0947\u092F\u0930 \u0938\u0902\u0936\u094B\u0927\ + \u0928\u094B\u0902 \u0938\u0947 \u0938\u0902\u092C\u0902\u0927\u093F\u0924 \u0905\ + \u0932\u094D\u092B\u093C\u093E \u0915\u094B \u0915\u0941\u0932 \u0935\u094D\u092F\ + \u092F \u0915\u093F\u0924\u0928\u0940 \u0930\u093E\u0936\u093F \u092E\u093E\u0928\ + \u0928\u0940 \u091A\u093E\u0939\u093F\u090F?" + - input_choice_list: + A: "\u092E\u0942\u0932\u094D\u092F\u093E\u0902\u0915\u0928 \u090F\u0935\u0902\ + \ \u0906\u0935\u0902\u091F\u0928" + B: "\u0938\u0902\u092A\u0942\u0930\u094D\u0923\u0924\u093E" + C: "\u0905\u0927\u093F\u0915\u093E\u0930 \u0906\u0948\u0930 \u0926\u093E\u092F\ + \u093F\u0924\u094D\u0935" + D: "\u092A\u094D\u0930\u0938\u094D\u0924\u0941\u0924\u093F \u090F\u0935\u0902\ + \ \u092A\u094D\u0930\u0915\u091F\u0940\u0915\u0930\u0923" + input_correct_responses: + - B + input_question: "\u090F\u0915 \u0911\u0921\u093F\u091F\u0930 \u0915\u093F\u0938\ + \u0940 \u0917\u0948\u0930-\u091C\u093E\u0930\u0940\u0915\u0930\u094D\u0924\u093E\ + \ \u0915\u0947 \u0938\u092C\u0932\u0947\u091C\u0930 \u0915\u094B \u0909\u092A\ + \u0915\u0930\u0923 \u092A\u0930 \u0938\u0940\u0930\u093F\u092F\u0932 \u0928\u0902\ + \u092C\u0930 \u0915\u093E \u092A\u0924\u093E \u0932\u0917\u093E\u0924\u093E\ + \ \u0939\u0948\u0964 \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924\ + \ \u092E\u0947\u0902 \u0938\u0947 \u0915\u094C\u0928 \u0938\u093E \u092A\u094D\ + \u0930\u092C\u0902\u0927\u0928 \u0926\u093E\u0935\u093E \u0907\u0938 \u092A\u0930\ + \u0940\u0915\u094D\u0937\u0923 \u0926\u094D\u0935\u093E\u0930\u093E \u0938\u092E\ + \u0930\u094D\u0925\u093F\u0924 \u0939\u0948?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_accounting +tag: mmlu_hi_llama_other_tasks +task: mmlu_hi_llama_professional_accounting +task_alias: professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77651a89b16eb1c2cb8f4b6be01e10256dd8c0b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_law.yaml @@ -0,0 +1,434 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u091A\u094C\u0926\u0939\u0935\u0947\u0902 \u0938\u0902\u0936\u094B\u0927\ + \u0928 \u0915\u0947 \u0928\u093F\u092F\u0924 \u092A\u094D\u0930\u0915\u094D\ + \u0930\u093F\u092F\u093E \u0916\u0902\u0921 \u0915\u0947 \u0924\u0939\u0924\ + \ \u0905\u0938\u094D\u092A\u0937\u094D\u091F\u0924\u093E \u0915\u0947 \u0915\ + \u093E\u0930\u0923 \u0915\u093C\u093E\u0928\u0942\u0928 \u0936\u0942\u0928\ + \u094D\u092F \u0939\u0948\u0964" + B: "\u0915\u093C\u093E\u0928\u0942\u0928 \u0905\u092E\u093E\u0928\u094D\u092F\ + \ \u0939\u0948 \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u092F\u0939 \u092A\ + \u094D\u0930\u0925\u092E \u0938\u0902\u0936\u094B\u0927\u0928 \u0915\u0947\ + \ \u0924\u0939\u0924 \u092F\u093E\u091A\u093F\u0915\u093E\u0915\u0930\u094D\ + \u0924\u093E \u0915\u0940 \u092C\u094B\u0932\u0928\u0947 \u0915\u0940 \u0938\ + \u094D\u0935\u0924\u0902\u0924\u094D\u0930\u0924\u093E \u0915\u093E \u0909\ + \u0932\u094D\u0932\u0902\u0918\u0928 \u0915\u0930\u0924\u093E \u0939\u0948\ + \u0964" + C: "\u092F\u0939 \u0915\u093C\u093E\u0928\u0942\u0928 \u092A\u094D\u0930\u0925\ + \u092E \u0938\u0902\u0936\u094B\u0927\u0928 \u0915\u0947 \u0924\u0939\u0924\ + \ \u092C\u094B\u0932\u0928\u0947 \u0915\u0940 \u0938\u094D\u0935\u0924\u0902\ + \u0924\u094D\u0930\u0924\u093E \u0915\u093E \u0939\u0928\u0928 \u0939\u0948\ + \ \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u0938\u092E\u093E\u0928 \u0909\ + \u0926\u094D\u0926\u0947\u0936\u094D\u092F \u0915\u094B \u092A\u094D\u0930\ + \u093E\u092A\u094D\u0924 \u0915\u0930\u0928\u0947 \u0915\u0947 \u0932\u093F\ + \u090F \u0915\u092E \u092A\u094D\u0930\u0924\u093F\u092C\u0902\u0927\u093E\ + \u0924\u094D\u092E\u0915 \u0938\u093E\u0927\u0928 \u0909\u092A\u0932\u092C\ + \u094D\u0927 \u0939\u0948\u0902\u0964" + D: "\u0915\u093C\u093E\u0928\u0942\u0928 \u0935\u094D\u092F\u093E\u092A\u0915\ + \ \u0939\u0948 \u0914\u0930 \u092A\u0930\u093F\u0923\u093E\u092E\u0938\u094D\ + \u0935\u0930\u0942\u092A \u092A\u0939\u0932\u0947 \u0914\u0930 \u091A\u094C\ + \u0926\u0939\u0935\u0947\u0902 \u0938\u0902\u0936\u094B\u0927\u0928 \u0915\ + \u0947 \u0924\u0939\u0924 \u0905\u092E\u093E\u0928\u094D\u092F \u0939\u0948\ + \u0964" + input_correct_responses: + - D + input_question: "\u090F\u0915 \u0930\u093E\u091C\u094D\u092F \u0935\u093F\u0927\ + \u093E\u092F\u093F\u0915\u093E \u0928\u0947 \u0939\u093E\u0932 \u0939\u0940\ + \ \u092E\u0947\u0902 \u090F\u0915 \u0915\u093C\u093E\u0928\u0942\u0928 \u092C\ + \u0928\u093E\u092F\u093E \u0939\u0948, \u091C\u093F\u0938\u0938\u0947 \u0905\ + \u092A\u0928\u0947 \u0915\u0930\u094D\u0924\u0935\u094D\u092F\u094B\u0902 \u0915\ + \u093E \u092A\u093E\u0932\u0928 \u0915\u0930 \u0930\u0939\u0947 \u0915\u093F\ + \u0938\u0940 \u092A\u0941\u0932\u093F\u0938 \u0905\u0927\u093F\u0915\u093E\u0930\ + \u0940 \u0915\u094B \u0936\u094D\u0930\u093E\u092A \u0926\u0947\u0928\u093E\ + , \u0928\u093F\u0902\u0926\u093E \u0915\u0930\u0928\u093E \u092F\u093E \u0909\ + \u0938\u0915\u0947 \u092A\u094D\u0930\u0924\u093F \u092F\u093E \u0909\u0938\u0915\ + \u0947 \u0938\u0902\u0926\u0930\u094D\u092D \u092E\u0947\u0902 \u0905\u0936\u094D\ + \u0932\u0940\u0932 \u092F\u093E \u0905\u092A\u092E\u093E\u0928\u091C\u0928\u0915\ + \ \u092D\u093E\u0937\u093E \u0915\u093E \u0909\u092A\u092F\u094B\u0917 \u0915\ + \u0930\u0928\u093E \u090F\u0915 \u0926\u0941\u0937\u094D\u0915\u0930\u094D\u092E\ + \ \u0939\u0948\u0964 \u0930\u093E\u091C\u094D\u092F \u0935\u093F\u0936\u094D\ + \u0935\u0935\u093F\u0926\u094D\u092F\u093E\u0932\u092F \u0915\u0947 \u090F\u0915\ + \ \u091B\u093E\u0924\u094D\u0930 \u0928\u0947 \u092F\u0941\u0926\u094D\u0927\ + \ \u0915\u0947 \u0935\u093F\u0930\u094B\u0927 \u092E\u0947\u0902 \u092A\u0930\ + \u093F\u0938\u0930 \u092E\u0947\u0902 \u090F\u0915 \u092A\u094D\u0930\u0926\u0930\ + \u094D\u0936\u0928 \u0915\u093E \u0906\u092F\u094B\u091C\u0928 \u0915\u093F\u092F\ + \u093E\u0964 \u0930\u0948\u0932\u0940 \u092E\u0947\u0902 50 \u091B\u093E\u0924\ + \u094D\u0930\u094B\u0902 \u0915\u0947 \u090F\u0915 \u0938\u092E\u0942\u0939\ + \ \u0928\u0947 \u092D\u093E\u0917 \u0932\u093F\u092F\u093E, \u091C\u093F\u0928\ + \u094D\u0939\u094B\u0902\u0928\u0947 \u092A\u093E\u0938 \u0938\u0947 \u0917\u0941\ + \u091C\u0930\u0928\u0947 \u0935\u093E\u0932\u0940 \u0915\u093E\u0930\u094B\u0902\ + \ \u092A\u0930 \u092F\u0941\u0926\u094D\u0927-\u0935\u093F\u0930\u094B\u0927\ + \u0940 \u0938\u0902\u0926\u0947\u0936 \u091A\u093F\u0932\u094D\u0932\u093E\u090F\ + \u0964 \u0938\u0902\u092F\u0941\u0915\u094D\u0924 \u0930\u093E\u091C\u094D\u092F\ + \ \u0905\u092E\u0947\u0930\u093F\u0915\u093E \u0915\u0947 \u092A\u094D\u0930\ + \u0924\u093F \u0905\u092A\u0928\u0940 \u0905\u0935\u092E\u093E\u0928\u0928\u093E\ + \ \u0926\u093F\u0916\u093E\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F, \u091B\ + \u093E\u0924\u094D\u0930 \u0928\u0947 \u0905\u092A\u0928\u0940 \u091C\u0940\u0902\ + \u0938 \u0915\u0947 \u092A\u093F\u091B\u0932\u0947 \u0939\u093F\u0938\u094D\u0938\ + \u0947 \u092A\u0930 \u0905\u092E\u0947\u0930\u093F\u0915\u0940 \u0927\u094D\u0935\ + \u091C \u0938\u093F\u0932 \u0926\u093F\u092F\u093E\u0964 \u091C\u092C \u090F\ + \u0915 \u092A\u0941\u0932\u093F\u0938 \u0905\u0927\u093F\u0915\u093E\u0930\u0940\ + \ \u0928\u0947 \u091B\u093E\u0924\u094D\u0930 \u0915\u0940 \u091C\u0940\u0902\ + \u0938 \u092A\u0930 \u091D\u0902\u0921\u093E \u0932\u0917\u093E \u0939\u0941\ + \u0906 \u0926\u0947\u0916\u093E, \u0924\u094B \u0935\u0939 \u0909\u0938\u0915\ + \u0947 \u092A\u093E\u0938 \u0906\u092F\u093E \u0914\u0930 \u0909\u0938\u0938\ + \u0947 \u0915\u0939\u093E \u0915\u093F \u0935\u0939 \u091D\u0902\u0921\u093E\ + \ \u0939\u091F\u093E \u0926\u0947 \u0905\u0928\u094D\u092F\u0925\u093E \u0909\ + \u0938\u0947 \u0917\u093F\u0930\u092B\u094D\u0924\u093E\u0930 \u0915\u0930 \u0932\ + \u093F\u092F\u093E \u091C\u093E\u090F\u0917\u093E\u0964 \u091B\u093E\u0924\u094D\ + \u0930 \u0915\u094D\u0930\u094B\u0927\u093F\u0924 \u0939\u094B \u0917\u092F\u093E\ + \ \u0914\u0930 \u092A\u0941\u0932\u093F\u0938 \u0905\u0927\u093F\u0915\u093E\ + \u0930\u0940 \u092A\u0930 \u091A\u093F\u0932\u094D\u0932\u093E\u092F\u093E,\ + \ "\u0938\u0941\u0928\u094B, \u0915\u092E\u0940\u0928\u0947, \u092E\u0948\ + \u0902 \u092F\u0939 \u0915\u092A\u0921\u093C\u093E \u091C\u0939\u093E\u0902\ + \ \u092D\u0940 \u091A\u093E\u0939\u0942\u0902 \u092A\u0939\u0928\u0942\u0902\ + \u0917\u093E\u0964" \u092C\u093E\u0926 \u092E\u0947\u0902 \u091B\u093E\u0924\ + \u094D\u0930 \u0915\u094B \u0917\u093F\u0930\u092B\u094D\u0924\u093E\u0930 \u0915\ + \u0930 \u0932\u093F\u092F\u093E \u0917\u092F\u093E \u0914\u0930 \u0930\u093E\ + \u091C\u094D\u092F \u0915\u093C\u093E\u0928\u0942\u0928 \u0915\u093E \u0909\u0932\ + \u094D\u0932\u0902\u0918\u0928 \u0915\u0930\u0928\u0947 \u0915\u093E \u0906\u0930\ + \u094B\u092A \u0932\u0917\u093E\u092F\u093E \u0917\u092F\u093E\u0964 \u091B\u093E\ + \u0924\u094D\u0930 \u092C\u093E\u0926 \u092E\u0947\u0902 \u0915\u093C\u093E\u0928\ + \u0942\u0928 \u0915\u0940 \u0938\u0902\u0935\u0948\u0927\u093E\u0928\u093F\u0915\ + \u0924\u093E \u0915\u094B \u091A\u0941\u0928\u094C\u0924\u0940 \u0926\u0947\u0924\ + \u0947 \u0939\u0941\u090F \u0930\u093E\u091C\u094D\u092F \u0905\u0926\u093E\u0932\ + \u0924 \u092E\u0947\u0902 \u092E\u0941\u0915\u0926\u092E\u093E \u0926\u093E\u092F\ + \u0930 \u0915\u0930\u0924\u093E \u0939\u0948\u0964 \u091B\u093E\u0924\u094D\u0930\ + \ \u0915\u0947 \u0932\u093F\u090F \u0938\u092C\u0938\u0947 \u092E\u091C\u092C\ + \u0942\u0924 \u0938\u0902\u0935\u0948\u0927\u093E\u0928\u093F\u0915 \u0924\u0930\ + \u094D\u0915 \u092F\u0939\u0940 \u0939\u0948" + - input_choice_list: + A: "\u0935\u093E\u0923\u093F\u091C\u094D\u092F \u0916\u0902\u0921." + B: "\u091A\u094C\u0926\u0939\u0935\u0947\u0902 \u0938\u0902\u0936\u094B\u0927\ + \u0928 \u0915\u093E \u0938\u092E\u093E\u0928 \u0938\u0902\u0930\u0915\u094D\ + \u0937\u0923 \u0916\u0902\u0921\u0964" + C: "\u0905\u0928\u0941\u091A\u094D\u091B\u0947\u0926 IV, \u0927\u093E\u0930\u093E\ + \ 2 \u0915\u0947 \u0935\u093F\u0936\u0947\u0937\u093E\u0927\u093F\u0915\u093E\ + \u0930 \u0914\u0930 \u0909\u0928\u094D\u092E\u0941\u0915\u094D\u0924\u093F\ + \u092F\u093E\u0901 \u0916\u0902\u0921\u0964" + D: "\u0905\u0928\u0941\u092C\u0902\u0927 \u0916\u0902\u0921." + input_correct_responses: + - A + input_question: "\u090F\u0915 \u0930\u093E\u091C\u094D\u092F \u0928\u0947 \u0939\ + \u093E\u0932 \u0939\u0940 \u092E\u0947\u0902 \u0930\u093E\u091C\u094D\u092F\ + \ \u0915\u0947 \u092D\u0940\u0924\u0930 \u0915\u093F\u0938\u0940 \u092D\u0940\ + \ \u092A\u0930\u092E\u093E\u0923\u0941 \u0915\u091A\u0930\u0947 \u0915\u0947\ + \ \u0928\u093F\u092A\u091F\u093E\u0928 \u092A\u0930 \u0930\u094B\u0915 \u0932\ + \u0917\u093E\u0928\u0947 \u0935\u093E\u0932\u093E \u090F\u0915 \u0915\u093C\u093E\ + \u0928\u0942\u0928 \u092C\u0928\u093E\u092F\u093E \u0939\u0948\u0964 \u092F\u0939\ + \ \u0915\u093E\u0928\u0942\u0928 \u0915\u093F\u0938\u0940 \u092D\u0940 \u0938\ + \u0902\u0918\u0940\u092F \u0915\u093C\u093E\u0928\u0942\u0928 \u0915\u093E \u0909\ + \u0932\u094D\u0932\u0902\u0918\u0928 \u092F\u093E \u091F\u0915\u0930\u093E\u0935\ + \ \u0928\u0939\u0940\u0902 \u0915\u0930\u0924\u093E \u0939\u0948\u0964 \u090F\ + \u0915 \u0906\u0926\u092E\u0940 \u0930\u093E\u091C\u094D\u092F \u092E\u0947\u0902\ + \ \u090F\u0915 \u0915\u0902\u092A\u0928\u0940 \u091A\u0932\u093E\u0924\u093E\ + \ \u0939\u0948 \u091C\u094B \u092A\u0930\u092E\u093E\u0923\u0941 \u0915\u091A\ + \u0930\u0947 \u0915\u0947 \u0928\u093F\u092A\u091F\u093E\u0928 \u092E\u0947\u0902\ + \ \u0932\u0917\u0940 \u0939\u0941\u0908 \u0939\u0948\u0964 \u0930\u093E\u091C\ + \u094D\u092F \u0915\u093C\u093E\u0928\u0942\u0928 \u0915\u0947 \u092A\u093E\u0930\ + \u093F\u0924 \u0939\u094B\u0928\u0947 \u0915\u0947 \u092C\u093E\u0926, \u0909\ + \u0938 \u0935\u094D\u092F\u0915\u094D\u0924\u093F \u0915\u094B, \u091C\u094B\ + \ \u0905\u092D\u0940 \u0924\u0915 \u0928\u090F \u0915\u093E\u0928\u0942\u0928\ + \ \u0938\u0947 \u0905\u0935\u0917\u0924 \u0928\u0939\u0940\u0902 \u0925\u093E\ + , \u0930\u093E\u091C\u094D\u092F \u092E\u0947\u0902 \u0905\u092A\u0928\u0947\ + \ \u092A\u0930\u092E\u093E\u0923\u0941 \u0915\u091A\u0930\u0947 \u0915\u0947\ + \ \u0928\u093F\u092A\u091F\u093E\u0928 \u0915\u0947 \u0932\u093F\u090F \u0915\ + \u0908 \u0930\u093E\u091C\u094D\u092F-\u092C\u093E\u0939\u0930 \u0915\u0940\ + \ \u0915\u0902\u092A\u0928\u093F\u092F\u094B\u0902 \u0915\u0947 \u0938\u093E\ + \u0925 \u0905\u0928\u0941\u092C\u0902\u0927 \u092E\u0947\u0902 \u092A\u094D\u0930\ + \u0935\u0947\u0936 \u0915\u093F\u092F\u093E\u0964 \u0939\u093E\u0932\u093E\u0901\ + \u0915\u093F, \u0907\u0938 \u0928\u090F \u0915\u093E\u0928\u0942\u0928 \u0915\ + \u0947 \u0915\u093E\u0930\u0923, \u0906\u0926\u092E\u0940 \u0907\u0928 \u0905\ + \u0928\u0941\u092C\u0902\u0927\u094B\u0902 \u0915\u094B \u092A\u0942\u0930\u093E\ + \ \u0915\u0930\u0928\u0947 \u092E\u0947\u0902 \u0905\u0938\u092E\u0930\u094D\ + \u0925 \u0939\u094B\u0917\u093E\u0964 \u092E\u093E\u0928 \u0932\u0940\u091C\u093F\ + \u090F \u0915\u093F \u0935\u0939 \u0935\u094D\u092F\u0915\u094D\u0924\u093F\ + \ \u0907\u0938 \u0930\u093E\u091C\u094D\u092F \u0915\u093E\u0928\u0942\u0928\ + \ \u0915\u094B \u091A\u0941\u0928\u094C\u0924\u0940 \u0926\u0947\u0928\u0947\ + \ \u0915\u0947 \u0932\u093F\u090F \u0916\u0921\u093C\u093E \u0939\u0948\u0964\ + \ \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u092E\u0947\u0902\ + \ \u0938\u0947 \u0915\u094C\u0928 \u0930\u093E\u091C\u094D\u092F \u0915\u0947\ + \ \u092D\u0940\u0924\u0930 \u092A\u0930\u092E\u093E\u0923\u0941 \u0915\u091A\ + \u0930\u0947 \u0915\u0947 \u0928\u093F\u092A\u091F\u093E\u0928 \u092A\u0930\ + \ \u0930\u094B\u0915 \u0932\u0917\u093E\u0928\u0947 \u0935\u093E\u0932\u0947\ + \ \u0930\u093E\u091C\u094D\u092F \u0915\u0947 \u0915\u093E\u0928\u0942\u0928\ + \ \u0915\u094B \u091A\u0941\u0928\u094C\u0924\u0940 \u0926\u0947\u0928\u0947\ + \ \u0915\u0947 \u0932\u093F\u090F \u0905\u092A\u0928\u093E \u0938\u092C\u0938\ + \u0947 \u092E\u091C\u092C\u0942\u0924 \u0938\u0902\u0935\u0948\u0927\u093E\u0928\ + \u093F\u0915 \u0906\u0927\u093E\u0930 \u092A\u094D\u0930\u0938\u094D\u0924\u0941\ + \u0924 \u0915\u0930\u0924\u093E \u0939\u0948?" + - input_choice_list: + A: "\u0928\u093F\u0930\u094D\u0935\u093F\u0935\u093E\u0926 \u0924\u0925\u094D\ + \u092F." + B: "\u0935\u0947 \u0924\u0925\u094D\u092F \u091C\u094B \u0935\u094D\u092F\u0915\ + \u094D\u0924\u093F\u0917\u0924 \u0930\u093E\u091C\u0928\u0940\u0924\u093F\u0915\ + \ \u0938\u0902\u0917\u0920\u0928\u094B\u0902 \u0926\u094D\u0935\u093E\u0930\ + \u093E \u0926\u093E\u0935\u093E \u0915\u093F\u090F \u0917\u090F \u0939\u0948\ + \u0902\u0964" + C: "\u0938\u093E\u092E\u093E\u0928\u094D\u092F \u091C\u094D\u091E\u093E\u0928\ + \ \u0926\u094D\u0935\u093E\u0930\u093E \u0938\u0924\u094D\u092F \u092E\u093E\ + \u0928\u0947 \u091C\u093E\u0928\u0947 \u0935\u093E\u0932\u0947 \u0924\u0925\ + \u094D\u092F\u0964" + D: "\u0935\u0948\u091C\u094D\u091E\u093E\u0928\u093F\u0915 \u0938\u0924\u094D\ + \u092F\u093E\u092A\u0928 \u092E\u0947\u0902 \u0938\u0915\u094D\u0937\u092E\ + \ \u0924\u0925\u094D\u092F." + input_correct_responses: + - B + input_question: "\u0928\u094D\u092F\u093E\u092F\u093E\u0927\u0940\u0936 \u0928\ + \u0947 \u092E\u0941\u0915\u0926\u092E\u0947 \u0915\u0940 \u0936\u0941\u0930\u0941\ + \u0906\u0924 \u092E\u0947\u0902 \u0915\u0941\u091B \u0924\u0925\u094D\u092F\u094B\ + \u0902 \u092A\u0930 \u0928\u094D\u092F\u093E\u092F\u093F\u0915 \u0938\u0902\u091C\ + \u094D\u091E\u093E\u0928 \u0932\u093F\u092F\u093E\u0964 \u0928\u093F\u092E\u094D\ + \u0928\u0932\u093F\u0916\u093F\u0924 \u092E\u0947\u0902 \u0938\u0947 \u0915\u094C\ + \u0928 \u0938\u093E \u0928\u094D\u092F\u093E\u092F\u093F\u0915 \u0928\u094B\u091F\ + \u093F\u0938 \u0915\u0947 \u0932\u093F\u090F \u0909\u092A\u092F\u0941\u0915\u094D\ + \u0924 \u092A\u094D\u0930\u0915\u093E\u0930 \u0915\u093E \u0924\u0925\u094D\u092F\ + \ \u0928\u0939\u0940\u0902 \u0939\u0948?" + - input_choice_list: + A: "\u0930\u093E\u0939\u0924 \u092A\u094D\u0930\u0926\u093E\u0928 \u0915\u0930\ + \u0947\u0902, \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u092C\u093E\u0921\ + \u093C \u0938\u0941\u0916 \u0938\u0941\u0935\u093F\u0927\u093E \u092A\u094D\ + \u0930\u0924\u093F\u092C\u0902\u0927 \u0915\u093E \u0909\u0932\u094D\u0932\ + \u0902\u0918\u0928 \u0915\u0930 \u0930\u0939\u0940 \u0925\u0940\u0964" + B: "\u0930\u093E\u0939\u0924 \u092A\u094D\u0930\u0926\u093E\u0928 \u0915\u0930\ + \u0947\u0902, \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u092C\u093E\u0921\ + \u093C \u0915\u0947 \u0905\u0924\u093F\u0915\u094D\u0930\u092E\u0923 \u0928\ + \u0947 \u092E\u0942\u0932 \u092F\u094B\u091C\u0928\u093E \u092E\u0947\u0902\ + \ \u092A\u094D\u0930\u0924\u093F\u092C\u0902\u0927 \u0915\u093E \u0909\u0932\ + \u094D\u0932\u0902\u0918\u0928 \u0915\u093F\u092F\u093E \u0939\u0948\u0964" + C: "\u0930\u093E\u0939\u0924 \u0938\u0947 \u0907\u0928\u0915\u093E\u0930 \u0915\ + \u0930\u0947\u0902, \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u0936\u093F\ + \u0915\u094D\u0937\u0915 \u0938\u0947\u0935\u093E\u0928\u093F\u0935\u0943\u0924\ + \u094D\u0924 \u0915\u0947 \u0916\u093F\u0932\u093E\u092B \u092A\u094D\u0930\ + \u0924\u093F\u092C\u0902\u0927 \u0932\u093E\u0917\u0942 \u0915\u0930\u0928\ + \u0947 \u092E\u0947\u0902 \u0935\u093F\u092B\u0932 \u0930\u0939\u093E\u0964" + D: "\u0930\u093E\u0939\u0924 \u0938\u0947 \u0907\u0928\u0915\u093E\u0930 \u0915\ + \u0930\u0947\u0902, \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u092A\u094D\ + \u0930\u0924\u093F\u092C\u0902\u0927 \u0915\u0940 \u0936\u0930\u094D\u0924\ + \u094B\u0902 \u0915\u0947 \u0924\u0939\u0924 \u092C\u093E\u0921\u093C \u0915\ + \u094B "\u090F\u0915 \u0938\u0902\u0930\u091A\u0928\u093E" \u0915\ + \u0947 \u0930\u0942\u092A \u092E\u0947\u0902 \u0928\u0939\u0940\u0902 \u0938\ + \u092E\u091D\u093E \u091C\u093E\u090F\u0917\u093E\u0964" + input_correct_responses: + - B + input_question: "1 \u0905\u0915\u094D\u091F\u0942\u092C\u0930 1980 \u0915\u094B\ + , \u090F\u0915 \u0917\u094D\u0930\u093E\u092E\u0940\u0923 \u0915\u093E\u0909\ + \u0902\u091F\u0940 \u092E\u0947\u0902 \u0915\u0908 \u0938\u094C \u090F\u0915\ + \u0921\u093C \u091C\u092E\u0940\u0928 \u0915\u0947 \u092E\u093E\u0932\u093F\u0915\ + , \u090F\u0915 \u0921\u0947\u0935\u0932\u092A\u0930 \u0928\u0947 \u0915\u094D\ + \u0937\u0947\u0924\u094D\u0930 \u0915\u0947 \u0932\u093F\u090F \u090F\u0915\ + \ \u0938\u093E\u092E\u093E\u0928\u094D\u092F \u0935\u093F\u0915\u093E\u0938\ + \ \u092F\u094B\u091C\u0928\u093E \u0915\u093E \u092E\u0938\u094C\u0926\u093E\ + \ \u0924\u0948\u092F\u093E\u0930 \u0915\u093F\u092F\u093E\u0964 \u0935\u093F\ + \u0927\u093F\u0935\u0924 \u0926\u0930\u094D\u091C \u092F\u094B\u091C\u0928\u093E\ + \ \u092E\u0947\u0902 \u092F\u094B\u091C\u0928\u093E \u092E\u0947\u0902 \u092D\ + \u0942\u092E\u093F \u092A\u0930 \u0935\u093F\u0938\u094D\u0924\u0943\u0924 \u0938\ + \u0940\u092E\u093E\u090F\u0902 \u0914\u0930 \u092A\u094D\u0930\u0924\u093F\u092C\ + \u0902\u0927 \u0932\u0917\u093E\u090F \u0917\u090F \u0925\u0947, \u091C\u093F\ + \u0938\u0947 \u090F\u0915 \u0906\u0935\u093E\u0938\u0940\u092F \u091C\u093F\u0932\ + \u0947 \u0915\u0947 \u0930\u0942\u092A \u092E\u0947\u0902 \u0935\u093F\u0915\ + \u0938\u093F\u0924 \u0915\u093F\u092F\u093E \u091C\u093E\u0928\u093E \u0925\u093E\ + \u0964 \u092A\u094D\u0930\u0924\u093F\u092C\u0902\u0927 \u0915\u093F\u0938\u0940\ + \ \u092D\u0940 \u0932\u0949\u091F \u0915\u094B \u092A\u094D\u0930\u093E\u092A\ + \u094D\u0924 \u0915\u0930\u0928\u0947 \u0935\u093E\u0932\u0947 \u0938\u092D\u0940\ + \ \u0935\u094D\u092F\u0915\u094D\u0924\u093F\u092F\u094B\u0902 \u0914\u0930\ + \ \u0909\u0928\u0915\u0947 \u0909\u0924\u094D\u0924\u0930\u093E\u0927\u093F\u0915\ + \u093E\u0930\u093F\u092F\u094B\u0902, \u0928\u093F\u092F\u0941\u0915\u094D\u0924\ + \u093F\u092F\u094B\u0902 \u0914\u0930 \u092A\u091F\u094D\u091F\u0947\u0926\u093E\ + \u0930\u094B\u0902 \u092A\u0930 \u0932\u093E\u0917\u0942 \u0939\u094B\u0928\u0947\ + \ \u0925\u0947\u0964 \u0906\u0917\u0947 \u092F\u0939 \u092A\u094D\u0930\u093E\ + \u0935\u0927\u093E\u0928 \u0915\u093F\u092F\u093E \u0917\u092F\u093E \u0915\u093F\ + \ \u0938\u092D\u0940 \u092C\u093E\u0926 \u0915\u0947 \u092E\u093E\u0932\u093F\ + \u0915\u094B\u0902 \u0938\u0947 \u092A\u094D\u0930\u0924\u093F\u092C\u0902\u0927\ + \u094B\u0902 \u0915\u0940 \u0909\u091A\u093F\u0924 \u0938\u0942\u091A\u0928\u093E\ + \ \u0932\u0940 \u091C\u093E\u090F\u0917\u0940\u0964 \u0938\u093E\u092E\u093E\ + \u0928\u094D\u092F \u092F\u094B\u091C\u0928\u093E \u092E\u0947\u0902 \u0909\u0928\ + \ \u092A\u094D\u0930\u0924\u093F\u092C\u0902\u0927\u094B\u0902 \u092E\u0947\u0902\ + \ \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u0925\u0947\ + : (22) \u092A\u094D\u0930\u0935\u0947\u0936 \u0914\u0930 \u0928\u093F\u0915\u093E\ + \u0938 \u0915\u0947 \u0905\u0927\u093F\u0915\u093E\u0930 \u0915\u0947 \u0938\ + \u093E\u0925 \u0938\u093E\u0930\u094D\u0935\u091C\u0928\u093F\u0915 \u0909\u092A\ + \u092F\u094B\u0917\u093F\u0924\u093E \u0915\u0902\u092A\u0928\u093F\u092F\u094B\ + \u0902 \u0915\u0947 \u0909\u092A\u092F\u094B\u0917 \u0915\u0947 \u0932\u093F\ + \u090F \u092A\u094D\u0930\u0924\u094D\u092F\u0947\u0915 \u0932\u0949\u091F \u0915\ + \u0947 \u092A\u0940\u091B\u0947 10 \u092B\u0940\u091F \u091A\u094C\u0921\u093C\ + \u0940 \u092D\u0942\u092E\u093F \u0915\u0940 \u090F\u0915 \u092A\u091F\u094D\ + \u091F\u0940 \u092E\u0947\u0902 \u090F\u0915 \u092E\u0924\u093E\u0927\u093F\u0915\ + \u093E\u0930 \u0905\u0927\u093F\u0915\u093E\u0930 \u092C\u0928\u093E\u092F\u093E\ + \ \u091C\u093E\u0924\u093E \u0939\u0948\u0964 (23) \u0909\u0915\u094D\u0924\ + \ \u092C\u094D\u0932\u0949\u0915\u094B\u0902 \u0938\u0947 \u0917\u0941\u091C\ + \u0930\u0928\u0947 \u0935\u093E\u0932\u0940 \u092D\u0942\u092E\u093F \u0915\u0940\ + \ \u0909\u092A\u0930\u094B\u0915\u094D\u0924 \u092A\u091F\u094D\u091F\u0940\ + \ \u092A\u0930 \u0915\u093F\u0938\u0940 \u092D\u0940 \u092A\u094D\u0930\u0915\ + \u093E\u0930 \u0915\u093E \u0915\u094B\u0908 \u0918\u0930 \u092F\u093E \u0938\ + \u0902\u0930\u091A\u0928\u093E \u0928\u0939\u0940\u0902 \u092C\u0928\u093E\u0908\ + \ \u091C\u093E\u090F\u0917\u0940\u0964 2000 \u092E\u0947\u0902, \u090F\u0915\ + \ \u0938\u0947\u0935\u093E\u0928\u093F\u0935\u0943\u0924\u094D\u0924 \u0935\u094D\ + \u092F\u0915\u094D\u0924\u093F \u0928\u0947 \u0932\u0949\u091F \u092E\u0947\u0902\ + \ \u0938\u0947 \u090F\u0915 \u0916\u0930\u0940\u0926\u093E, \u090F\u0915 \u0918\ + \u0930 \u092C\u0928\u093E\u092F\u093E, \u0914\u0930 \u092A\u094D\u0930\u0924\ + \u093F\u092C\u0902\u0927\u093F\u0924 \u0915\u094D\u0937\u0947\u0924\u094D\u0930\ + \ \u0915\u0947 \u092D\u0940\u0924\u0930 \u0905\u092A\u0928\u0940 \u0938\u0902\ + \u092A\u0924\u094D\u0924\u093F \u0915\u0947 \u092A\u0940\u091B\u0947 \u090F\u0915\ + \ \u092C\u093E\u0921\u093C \u0932\u0917\u093E \u0926\u0940\u0964 2004 \u092E\ + \u0947\u0902, \u090F\u0915 \u0936\u093F\u0915\u094D\u0937\u0915 \u0928\u0947\ + \ \u0938\u0947\u0935\u093E\u0928\u093F\u0935\u0943\u0924\u094D\u0924 \u0935\u094D\ + \u092F\u0915\u094D\u0924\u093F \u0915\u0940 \u0938\u0902\u092A\u0924\u094D\u0924\ + \u093F \u0915\u0947 \u092A\u093E\u0938 \u092C\u0939\u0941\u0924 \u0915\u0941\ + \u091B \u0916\u0930\u0940\u0926\u093E \u0914\u0930 \u090F\u0915 \u0928\u092F\ + \u093E \u0918\u0930 \u092C\u0928\u093E\u092F\u093E\u0964 \u0926\u094B \u0938\ + \u093E\u0932 \u092C\u093E\u0926, \u090F\u0915 \u0932\u093E\u0907\u092C\u094D\ + \u0930\u0947\u0930\u093F\u092F\u0928 \u0928\u0947 \u0936\u093F\u0915\u094D\u0937\ + \u0915 \u0915\u0940 \u0938\u0902\u092A\u0924\u094D\u0924\u093F \u0938\u0947\ + \ \u091C\u0941\u0921\u093C\u0940 \u0939\u0941\u0908 \u091C\u092E\u0940\u0928\ + \ \u0916\u0930\u0940\u0926 \u0932\u0940\u0964 \u0909\u0928 \u0938\u0902\u092A\ + \u0924\u094D\u0924\u093F\u092F\u094B\u0902 \u0915\u0947 \u0924\u0940\u0928 \u0915\ + \u093E\u0930\u094D\u092F\u094B\u0902 \u092E\u0947\u0902 \u0938\u0947 \u092A\u094D\ + \u0930\u0924\u094D\u092F\u0947\u0915 \u092E\u0947\u0902 \u0921\u0940\u0921 \u092C\ + \u0941\u0915 \u0915\u093E \u0938\u0902\u0926\u0930\u094D\u092D \u0936\u093E\u092E\ + \u093F\u0932 \u0925\u093E \u091C\u0939\u093E\u0902 \u0938\u093E\u092E\u093E\u0928\ + \u094D\u092F \u092F\u094B\u091C\u0928\u093E \u0926\u0930\u094D\u091C \u0915\u0940\ + \ \u0917\u0908 \u0925\u0940\u0964 2008 \u092E\u0947\u0902, \u0932\u093E\u0907\ + \u092C\u094D\u0930\u0947\u0930\u093F\u092F\u0928 \u0928\u0947 \u0936\u093F\u0915\ + \u094D\u0937\u0915 \u0915\u0947 \u0938\u093E\u0925 \u0905\u092A\u0928\u0947\ + \ \u0939\u093F\u0938\u094D\u0938\u0947 \u0915\u094B \u0935\u093F\u092D\u093E\ + \u091C\u093F\u0924 \u0915\u0930\u0928\u0947 \u0935\u093E\u0932\u0940 \u0930\u0947\ + \u0916\u093E \u0915\u0947 \u0938\u093E\u0925-\u0938\u093E\u0925, \u0914\u0930\ + \ \u092E\u0924\u093E\u0927\u093F\u0915\u093E\u0930 \u0905\u0927\u093F\u0915\u093E\ + \u0930 \u0915\u0947 \u0905\u0927\u0940\u0928 \u0915\u094D\u0937\u0947\u0924\u094D\ + \u0930 \u0915\u0947 \u0915\u0947\u0902\u0926\u094D\u0930 \u0915\u0947 \u0938\ + \u093E\u0925 \u0938\u093E\u0924 \u092B\u0941\u091F \u0915\u0940 \u092A\u094B\ + \u0938\u094D\u091F-\u090F\u0902\u0921-\u0930\u0947\u0932 \u092C\u093E\u0921\u093C\ + \ \u0915\u093E \u0928\u093F\u0930\u094D\u092E\u093E\u0923 \u0936\u0941\u0930\ + \u0942 \u0915\u093F\u092F\u093E\u0964 \u0939\u093E\u0932\u093E\u0901\u0915\u093F\ + \ \u0936\u093F\u0915\u094D\u0937\u0915 \u0928\u0947 \u0907\u0938\u0915\u0947\ + \ \u0928\u093F\u0930\u094D\u092E\u093E\u0923 \u092A\u0930 \u0906\u092A\u0924\ + \u094D\u0924\u093F \u091C\u0924\u093E\u0908, \u0932\u0947\u0915\u093F\u0928\ + \ \u092C\u093E\u0921\u093C \u0915\u093E \u0928\u093F\u0930\u094D\u092E\u093E\ + \u0923 \u092A\u0942\u0930\u093E \u0939\u094B \u0917\u092F\u093E\u0964 \u092F\ + \u0926\u093F \u0936\u093F\u0915\u094D\u0937\u0915 \u0932\u093E\u0907\u092C\u094D\ + \u0930\u0947\u0930\u093F\u092F\u0928 \u0915\u0940 \u092C\u093E\u0921\u093C \u0915\ + \u094B \u0939\u091F\u093E\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u092C\ + \u093E\u0927\u094D\u092F \u0915\u0930\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F\ + \ \u0905\u0928\u093F\u0935\u093E\u0930\u094D\u092F \u0928\u093F\u0937\u0947\u0927\ + \u093E\u091C\u094D\u091E\u093E \u0915\u0940 \u092E\u093E\u0902\u0917 \u0915\u0930\ + \u0924\u093E \u0939\u0948, \u0924\u094B \u0905\u0926\u093E\u0932\u0924 \u0938\ + \u092C\u0938\u0947 \u0905\u0927\u093F\u0915 \u0938\u0902\u092D\u093E\u0935\u0928\ + \u093E \u0939\u094B\u0917\u0940" + - input_choice_list: + A: "\u092A\u093F\u0924\u093E \u0915\u093E \u0935\u093E\u0926\u093E \u0914\u0930\ + \ \u0909\u0938 \u092A\u0930 \u090B\u0923\u0926\u093E\u0924\u093E \u0915\u0940\ + \ \u0928\u093F\u0930\u094D\u092D\u0930\u0924\u093E, \u092F\u0926\u093F \u0938\ + \u093E\u092C\u093F\u0924 \u0939\u094B \u091C\u093E\u0924\u0940 \u0939\u0948\ + , \u0924\u094B \u0935\u091A\u0928\u092C\u0902\u0927\u0928 \u0915\u0947 \u0938\ + \u093F\u0926\u094D\u0927\u093E\u0902\u0924 \u0915\u0947 \u0906\u0927\u093E\ + \u0930 \u092A\u0930 \u090B\u0923\u0926\u093E\u0924\u093E \u0926\u094D\u0935\ + \u093E\u0930\u093E \u092A\u093F\u0924\u093E \u0915\u0947 \u0916\u093F\u0932\ + \u093E\u092B \u090F\u0915 \u0935\u0948\u0927 \u0926\u093E\u0935\u0947 \u0915\ + \u094B \u091C\u0928\u094D\u092E \u0926\u093F\u092F\u093E \u091C\u093E\u0924\ + \u093E \u0939\u0948\u0964" + B: "\u0915\u094D\u092F\u094B\u0902\u0915\u093F \u092F\u0939 \u0905\u0928\u0941\ + \u092E\u093E\u0928 \u0932\u0917\u093E\u092F\u093E \u091C\u093E \u0938\u0915\ + \u0924\u093E \u0925\u093E \u0915\u093F \u092A\u093F\u0924\u093E \u0915\u093E\ + \ \u0935\u093E\u0926\u093E \u0932\u0947\u0928\u0926\u093E\u0930 \u0915\u094B\ + \ \u092C\u0947\u091F\u0947 \u0915\u0947 \u0916\u093F\u0932\u093E\u092B \u0915\ + \u094B\u0908 \u092D\u0940 \u0915\u093E\u0930\u094D\u0930\u0935\u093E\u0908\ + \ \u0915\u0930\u0928\u0947 \u0938\u0947 \u092E\u0928\u093E \u0915\u0930\u0928\ + \u0947 \u0915\u0947 \u0932\u093F\u090F \u092A\u094D\u0930\u0947\u0930\u093F\ + \u0924 \u0915\u0930\u0947\u0917\u093E, \u0907\u0938 \u0924\u0930\u0939 \u0915\ + \u0940 \u0938\u0939\u0928\u0936\u0940\u0932\u0924\u093E, \u0915\u093E\u0928\ + \u0942\u0928 \u0915\u0947 \u092E\u093E\u092E\u0932\u0947 \u092E\u0947\u0902\ + , \u092A\u093F\u0924\u093E \u0915\u0947 \u0935\u093E\u0926\u0947 \u0915\u0947\ + \ \u0932\u093F\u090F \u090F\u0915 \u0938\u094C\u0926\u093E \u0925\u0940\u0964" + C: "\u092A\u093F\u0924\u093E \u0926\u094D\u0935\u093E\u0930\u093E \u0932\u0947\ + \u0928\u0926\u093E\u0930 \u0915\u094B \u0915\u0941\u0932 2,500 \u0921\u0949\ + \u0932\u0930 \u0915\u0947 \u092A\u093E\u0901\u091A \u092D\u0941\u0917\u0924\ + \u093E\u0928 \u0915\u0930\u0928\u0947 \u0938\u0947 \u092A\u093F\u0924\u093E\ + \ \u0915\u0940 \u0913\u0930 \u0938\u0947 \u0938\u0902\u0935\u093F\u0926\u093E\ + \u0924\u094D\u092E\u0915 \u0930\u0942\u092A \u0938\u0947 \u092C\u093E\u0927\ + \u094D\u092F \u0939\u094B\u0928\u0947 \u0915\u093E \u0917\u0902\u092D\u0940\ + \u0930 \u0907\u0930\u093E\u0926\u093E \u092A\u094D\u0930\u0915\u091F \u0939\ + \u094B\u0924\u093E \u0939\u0948, \u0914\u0930 \u0907\u0938 \u0924\u0930\u0939\ + \ \u0915\u0940 \u0905\u092D\u093F\u0935\u094D\u092F\u0915\u094D\u0924\u093F\ + \ \u0915\u094B \u0906\u092E \u0924\u094C\u0930 \u092A\u0930 \u0935\u093F\u091A\ + \u093E\u0930 \u0915\u0947 \u0932\u093F\u090F \u090F\u0915 \u092A\u094D\u0930\ + \u092D\u093E\u0935\u0940 \u0935\u093F\u0915\u0932\u094D\u092A \u0915\u0947\ + \ \u0930\u0942\u092A \u092E\u0947\u0902 \u092E\u093E\u0928\u094D\u092F\u0924\ + \u093E \u0926\u0940 \u091C\u093E\u0924\u0940 \u0939\u0948\u0964" + D: "\u092C\u0947\u091F\u0947 \u0926\u094D\u0935\u093E\u0930\u093E \u0932\u0947\ + \u0928\u0926\u093E\u0930 \u0915\u0947 \u092A\u094D\u0930\u0924\u093F \u0926\ + \u0947\u092F \u092A\u0942\u0930\u094D\u0935\u0935\u0930\u094D\u0924\u0940\ + \ \u090B\u0923 \u0926\u093E\u092F\u093F\u0924\u094D\u0935 \u0915\u094B \u092E\ + \u093E\u0928\u0915\u0930, \u092A\u093F\u0924\u093E \u090F\u0915 \u091C\u093C\ + \u092E\u093E\u0928\u0924 \u092C\u0928 \u0917\u092F\u093E \u091C\u093F\u0938\ + \u0915\u093E \u0932\u0947\u0928\u0926\u093E\u0930 \u0938\u0947 \u0915\u093F\ + \u092F\u093E \u0917\u092F\u093E \u0935\u093E\u0926\u093E \u0932\u093E\u0917\ + \u0942 \u0915\u0930\u0928\u0947 \u092F\u094B\u0917\u094D\u092F \u0925\u093E\ + , \u0915\u094D\u092F\u094B\u0902\u0915\u093F \u092F\u0939 \u0932\u093F\u0916\ + \u093F\u0924 \u0930\u0942\u092A \u092E\u0947\u0902 \u0925\u093E \u0914\u0930\ + \ \u092A\u0930\u094D\u092F\u093E\u092A\u094D\u0924 \u0935\u093F\u091A\u093E\ + \u0930-\u0935\u093F\u092E\u0930\u094D\u0936 \u0926\u094D\u0935\u093E\u0930\ + \u093E \u0938\u092E\u0930\u094D\u0925\u093F\u0924 \u0925\u093E\u0964" + input_correct_responses: + - A + input_question: "\u090F\u0915 \u092C\u0947\u091F\u0947 \u092A\u0930 \u090F\u0915\ + \ \u0932\u0947\u0928\u0926\u093E\u0930 \u0915\u093E 5,000 \u0921\u0949\u0932\ + \u0930 \u092C\u0915\u093E\u092F\u093E \u0925\u093E\u0964 \u092C\u0947\u091F\u0947\ + \ \u0915\u0947 \u092A\u093F\u0924\u093E \u0928\u0947 \u0932\u0947\u0928\u0926\ + \u093E\u0930 \u0938\u0947 \u0938\u0902\u092A\u0930\u094D\u0915 \u0915\u093F\u092F\ + \u093E \u0914\u0930 \u0909\u0938\u0938\u0947 \u0915\u0939\u093E \u0915\u093F\ + \ \u0935\u0939 \u092C\u0947\u091F\u0947 \u0915\u093E \u0915\u0930\u094D\u091C\ + \ \u091A\u0941\u0915\u093E\u0928\u093E \u091A\u093E\u0939\u0924\u093E \u0939\ + \u0948\u0964 \u092A\u093F\u0924\u093E \u0928\u0947 \u090F\u0915 \u0926\u0938\ + \u094D\u0924\u093E\u0935\u0947\u091C\u093C \u092A\u0930 \u0939\u0938\u094D\u0924\ + \u093E\u0915\u094D\u0937\u0930 \u0915\u093F\u090F \u091C\u093F\u0938\u092E\u0947\ + \u0902 \u0915\u0939\u093E \u0917\u092F\u093E \u0925\u093E \u0915\u093F \u092A\ + \u093F\u0924\u093E 10 \u092E\u0939\u0940\u0928\u0947 \u0924\u0915 500 \u0921\ + \u0949\u0932\u0930 \u092A\u094D\u0930\u0924\u093F \u092E\u093E\u0939 \u0915\u0940\ + \ \u0926\u0930 \u0938\u0947 \u092C\u0947\u091F\u0947 \u0915\u093E \u0915\u0930\ + \u094D\u091C\u093C \u091A\u0941\u0915\u093E\u090F\u0917\u093E\u0964 \u0932\u0947\ + \u0928\u0926\u093E\u0930 \u0928\u0947 5,000 \u0921\u0949\u0932\u0930 \u0915\u093E\ + \ \u0915\u0930\u094D\u091C\u093C \u0935\u0938\u0942\u0932\u0928\u0947 \u0915\ + \u0947 \u0932\u093F\u090F \u092C\u0947\u091F\u0947 \u092A\u0930 \u092E\u0941\ + \u0915\u0926\u092E\u093E \u0915\u0930\u0928\u0947 \u0938\u0947 \u0930\u094B\u0915\ + \u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u0915\u094B\u0908 \u0932\u093F\ + \u0916\u093F\u0924 \u092F\u093E \u092E\u094C\u0916\u093F\u0915 \u092A\u094D\u0930\ + \u0924\u093F\u092C\u0926\u094D\u0927\u0924\u093E \u0928\u0939\u0940\u0902 \u091C\ + \u0924\u093E\u0908, \u0914\u0930 \u092A\u093F\u0924\u093E \u0928\u0947 \u092D\ + \u0940 \u0910\u0938\u0940 \u0915\u093F\u0938\u0940 \u092D\u0940 \u0930\u094B\ + \u0915 \u0915\u0947 \u0932\u093F\u090F \u0915\u094B\u0908 \u092E\u094C\u0916\ + \u093F\u0915 \u092F\u093E \u0932\u093F\u0916\u093F\u0924 \u0905\u0928\u0941\u0930\ + \u094B\u0927 \u0928\u0939\u0940\u0902 \u0915\u093F\u092F\u093E\u0964 \u0905\u0917\ + \u0932\u0947 \u092A\u093E\u0901\u091A \u092E\u0939\u0940\u0928\u094B\u0902 \u0915\ + \u0947 \u0932\u093F\u090F, \u092A\u093F\u0924\u093E \u0928\u0947 \u092D\u0941\ + \u0917\u0924\u093E\u0928 \u0915\u093F\u092F\u093E \u0914\u0930 \u0932\u0947\u0928\ + \u0926\u093E\u0930 \u0928\u0947 \u0938\u0939\u092E\u0924\u093F \u0915\u0947\ + \ \u0905\u0928\u0941\u0938\u093E\u0930 $500 \u0915\u093E \u092E\u093E\u0938\u093F\ + \u0915 \u092D\u0941\u0917\u0924\u093E\u0928 \u0938\u094D\u0935\u0940\u0915\u093E\ + \u0930 \u0915\u0930 \u0932\u093F\u092F\u093E\u0964 \u0909\u0938 \u0905\u0935\ + \u0927\u093F \u0915\u0947 \u0926\u094C\u0930\u093E\u0928, \u090B\u0923\u0926\ + \u093E\u0924\u093E \u0928\u0947, \u0935\u093E\u0938\u094D\u0924\u0935 \u092E\ + \u0947\u0902, \u092C\u0947\u091F\u0947 \u0915\u0947 \u0916\u093F\u0932\u093E\ + \u092B \u0915\u094B\u0908 \u092D\u0940 \u0915\u093E\u0928\u0942\u0928\u0940\ + \ \u0915\u093E\u0930\u094D\u0930\u0935\u093E\u0908 \u0915\u0930\u0928\u0947\ + \ \u0938\u0947 \u092E\u0928\u093E \u0915\u0930 \u0926\u093F\u092F\u093E\u0964\ + \ \u0939\u093E\u0932\u093E\u0901\u0915\u093F, \u0924\u092C \u092A\u093F\u0924\ + \u093E \u0928\u0947 \u0932\u0947\u0928\u0926\u093E\u0930 \u0915\u094B \u0938\ + \u0942\u091A\u093F\u0924 \u0915\u093F\u092F\u093E \u0915\u093F \u0935\u0939\ + \ \u090B\u0923 \u092A\u0930 \u0915\u094B\u0908 \u0914\u0930 \u092D\u0941\u0917\ + \u0924\u093E\u0928 \u0928\u0939\u0940\u0902 \u0915\u0930\u0947\u0917\u093E\u0964\ + \ \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u092E\u0947\u0902\ + \ \u0938\u0947 \u0915\u094C\u0928 \u0938\u093E \u0938\u092C\u0938\u0947 \u092A\ + \u094D\u0930\u0947\u0930\u0915 \u0924\u0930\u094D\u0915 \u0939\u0948 \u0915\u093F\ + \ \u092A\u093F\u0924\u093E \u0905\u092A\u0928\u0947 \u0938\u092E\u091D\u094C\ + \u0924\u0947 \u0915\u0940 \u0936\u0930\u094D\u0924\u094B\u0902 \u0915\u0947\ + \ \u0924\u0939\u0924 \u090B\u0923\u0926\u093E\u0924\u093E \u0915\u0947 \u092A\ + \u094D\u0930\u0924\u093F \u0909\u0924\u094D\u0924\u0930\u0926\u093E\u092F\u0940\ + \ \u0939\u0948?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_law +tag: mmlu_hi_llama_humanities_tasks +task: mmlu_hi_llama_professional_law +task_alias: professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b977a3f240006ae252d3eef9dd27d4389ee2a9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_medicine.yaml @@ -0,0 +1,261 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0932\u0947\u092C\u0947\u091F\u093E\u0932\u094B\u0932" + B: "\u092A\u094B\u091F\u0947\u0936\u093F\u092F\u092E \u0915\u094D\u0932\u094B\ + \u0930\u093E\u0907\u0921 \u0915\u0940 \u090F\u0915 \u0932\u094B\u0921\u093F\ + \u0902\u0917 \u0916\u0941\u0930\u093E\u0915" + C: nifedipine + D: "\u092B\u0947\u0928\u094B\u0915\u094D\u0938\u0940\u092C\u0947\u0902\u091C\ + \u093C\u093E\u092E\u093E\u0907\u0928" + input_correct_responses: + - D + input_question: "\u090F\u0915 42 \u0935\u0930\u094D\u0937\u0940\u092F \u0935\u094D\ + \u092F\u0915\u094D\u0924\u093F 2 \u0938\u092A\u094D\u0924\u093E\u0939 \u092E\ + \u0947\u0902 \u0928\u093F\u0930\u094D\u0927\u093E\u0930\u093F\u0924 \u090F\u0921\ + \u094D\u0930\u0947\u0928\u093E\u0932\u0947\u0915\u094D\u091F\u0949\u092E\u0940\ + \ \u0938\u0947 \u0917\u0941\u091C\u0930\u0928\u0947 \u0938\u0947 \u092A\u0939\ + \u0932\u0947 \u092A\u094D\u0930\u0940\u0911\u092A\u0930\u0947\u091F\u093F\u0935\ + \ \u092E\u0942\u0932\u094D\u092F\u093E\u0902\u0915\u0928 \u0915\u0947 \u0932\ + \u093F\u090F \u0915\u093E\u0930\u094D\u092F\u093E\u0932\u092F \u092E\u0947\u0902\ + \ \u0906\u0924\u093E \u0939\u0948\u0964 \u090F\u0915 \u092E\u0939\u0940\u0928\ + \u0947 \u092A\u0939\u0932\u0947, \u090F\u0915 \u092E\u094B\u091F\u0930 \u0935\ + \u093E\u0939\u0928 \u0915\u0940 \u091F\u0915\u094D\u0915\u0930 \u0915\u0947\ + \ \u092C\u093E\u0926 \u0909\u0928\u0915\u0947 \u0926\u093E\u0939\u093F\u0928\ + \u0947 \u092A\u093E\u0930\u094D\u0936\u094D\u0935 \u092E\u0947\u0902 \u0926\u0930\ + \u094D\u0926 \u0915\u0947 \u0915\u093E\u0930\u0923 \u0909\u0928\u094D\u0939\u0947\ + \u0902 \u0906\u092A\u093E\u0924\u0915\u093E\u0932\u0940\u0928 \u0935\u093F\u092D\ + \u093E\u0917 \u092E\u0947\u0902 \u0926\u0947\u0916\u092D\u093E\u0932 \u092E\u093F\ + \u0932\u0940 \u0925\u0940\u0964 \u0909\u0938 \u0938\u092E\u092F, \u0930\u0915\ + \u094D\u0924\u091A\u093E\u092A 160/100 \u092E\u093F\u092E\u0940 \u090F\u091A\ + \u091C\u0940 \u0925\u093E \u0914\u0930 \u092A\u0947\u091F \u0915\u0947 \u0938\ + \u0940\u091F\u0940 \u0938\u094D\u0915\u0948\u0928 \u092E\u0947\u0902 \u0906\u0915\ + \u0938\u094D\u092E\u093F\u0915 10-\u0938\u0947\u092E\u0940 \u092C\u093E\u090F\ + \u0902 \u0905\u0927\u093F\u0935\u0943\u0915\u094D\u0915 \u0926\u094D\u0930\u0935\ + \u094D\u092F\u092E\u093E\u0928 \u0926\u093F\u0916\u093E\u0964 \u0938\u0902\u092A\ + \u0942\u0930\u094D\u0923 \u0930\u0915\u094D\u0924 \u0917\u0923\u0928\u093E,\ + \ \u0938\u0940\u0930\u092E \u0907\u0932\u0947\u0915\u094D\u091F\u094D\u0930\u094B\ + \u0932\u093E\u0907\u091F \u0938\u093E\u0902\u0926\u094D\u0930\u0924\u093E \u0914\ + \u0930 \u092F\u0915\u0943\u0924 \u0915\u093E\u0930\u094D\u092F \u092A\u0930\u0940\ + \u0915\u094D\u0937\u0923 \u0938\u0939\u093F\u0924 \u092A\u094D\u0930\u092F\u094B\ + \u0917\u0936\u093E\u0932\u093E \u0905\u0927\u094D\u092F\u092F\u0928\u094B\u0902\ + \ \u0915\u0947 \u092A\u0930\u093F\u0923\u093E\u092E \u0938\u0902\u0926\u0930\ + \u094D\u092D \u0938\u0940\u092E\u093E \u0915\u0947 \u092D\u0940\u0924\u0930\ + \ \u0925\u0947\u0964 \u0905\u0928\u094D\u092F\u0925\u093E \u0930\u094B\u0917\ + \u0940 \u0938\u094D\u0935\u0938\u094D\u0925 \u0925\u093E \u0914\u0930 \u0909\ + \u0938\u0947 \u0915\u092D\u0940 \u0928\u0939\u0940\u0902 \u092C\u0924\u093E\u092F\ + \u093E \u0917\u092F\u093E \u0915\u093F \u0909\u0938\u0915\u093E \u0930\u0915\ + \u094D\u0924\u091A\u093E\u092A \u092C\u0922\u093C\u093E \u0939\u0941\u0906 \u0939\ + \u0948\u0964 \u0935\u0939 \u0915\u094B\u0908 \u0926\u0935\u093E \u0928\u0939\ + \u0940\u0902 \u0932\u0947\u0924\u093E. 2 \u0938\u092A\u094D\u0924\u093E\u0939\ + \ \u092A\u0939\u0932\u0947 \u0915\u093E\u0930\u094D\u092F\u093E\u0932\u092F\ + \ \u092E\u0947\u0902 \u090F\u0915 \u0905\u0928\u0941\u0935\u0930\u094D\u0924\ + \u0940 \u092F\u093E\u0924\u094D\u0930\u093E \u092E\u0947\u0902 \u092E\u0942\u0924\ + \u094D\u0930 \u092E\u0947\u0902 \u0928\u0949\u0930\u092E\u0947\u091F\u0947\u0928\ + \u092B\u094D\u0930\u093F\u0928 \u0914\u0930 \u092E\u0947\u091F\u093E\u0928\u0947\ + \u092B\u094D\u0930\u093F\u0928 \u0914\u0930 \u092A\u094D\u0932\u093E\u091C\u094D\ + \u092E\u093E \u090F\u0932\u094D\u0921\u094B\u0938\u094D\u091F\u0947\u0930\u094B\ + \u0928 \u0938\u093E\u0902\u0926\u094D\u0930\u0924\u093E \u092E\u0947\u0902 \u0935\ + \u0943\u0926\u094D\u0927\u093F \u0915\u093E \u092A\u0924\u093E \u091A\u0932\u093E\ + \u0964 \u092E\u0930\u0940\u091C \u0915\u094B \u090F\u0915 \u0938\u0930\u094D\ + \u091C\u0928 \u0915\u0947 \u092A\u093E\u0938 \u092D\u0947\u091C\u093E \u0917\ + \u092F\u093E, \u091C\u093F\u0938\u0928\u0947 \u090F\u0921\u094D\u0930\u0947\u0928\ + \u093E\u0932\u0947\u0915\u094D\u091F\u0949\u092E\u0940 \u0915\u0940 \u0938\u093F\ + \u092B\u093E\u0930\u093F\u0936 \u0915\u0940\u0964 \u0906\u091C, \u092E\u0939\ + \u0924\u094D\u0935\u092A\u0942\u0930\u094D\u0923 \u0938\u0902\u0915\u0947\u0924\ + \ \u0939\u0948\u0902 \u0924\u093E\u092A\u092E\u093E\u0928 36.6 \u0921\u093F\u0917\ + \u094D\u0930\u0940 \u0938\u0947\u0932\u094D\u0938\u093F\u092F\u0938 (97.9 \u0921\ + \u093F\u0917\u094D\u0930\u0940 \u092B\u093C\u093E\u0930\u0947\u0928\u0939\u093E\ + \u0907\u091F), \u0928\u093E\u0921\u093C\u0940 100/\u092E\u093F\u0928\u091F,\ + \ \u0936\u094D\u0935\u0938\u0928 14/\u092E\u093F\u0928\u091F, \u0914\u0930 \u0930\ + \u0915\u094D\u0924\u091A\u093E\u092A 170/95 \u092E\u093F\u092E\u0940 \u090F\u091A\ + \u091C\u0940\u0964 \u0936\u093E\u0930\u0940\u0930\u093F\u0915 \u092A\u0930\u0940\ + \u0915\u094D\u0937\u0923 \u0938\u0947 \u0915\u094B\u0908 \u092E\u0939\u0924\u094D\ + \u0935\u092A\u0942\u0930\u094D\u0923 \u0928\u093F\u0937\u094D\u0915\u0930\u094D\ + \u0937 \u0928\u0939\u0940\u0902 \u0928\u093F\u0915\u0932\u0924\u093E\u0964 \u092A\ + \u094D\u0930\u093E\u0930\u0902\u092D\u093F\u0915 \u092A\u094D\u0930\u0940\u0911\ + \u092A\u0930\u0947\u091F\u093F\u0935 \u0924\u0948\u092F\u093E\u0930\u0940 \u092E\ + \u0947\u0902 \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u092E\ + \u0947\u0902 \u0938\u0947 \u0915\u093F\u0938\u0915\u0947 \u0938\u093E\u0925\ + \ \u0909\u092A\u091A\u093E\u0930 \u0936\u093E\u092E\u093F\u0932 \u0939\u094B\ + \u0928\u093E \u091A\u093E\u0939\u093F\u090F?" + - input_choice_list: + A: "\u092C\u093E\u092F\u0940\u0902 \u0913\u0930 \u092C\u093E\u092F\u0940\u0902\ + \ \u0913\u0930 \u0924\u094D\u0930\u093F\u0915 \u092E\u0930\u094B\u0921\u093C" + B: "\u092C\u093E\u090F\u0902-\u092A\u0930-\u0926\u093E\u090F\u0902 \u0924\u094D\ + \u0930\u093F\u0915 \u092E\u0930\u094B\u0921\u093C" + C: "\u0926\u093E\u092F\u093E\u0902 \u090F\u0915\u0924\u0930\u092B\u093E \u0924\ + \u094D\u0930\u093F\u0915 \u092E\u094B\u0921\u093C" + D: "\u0926\u093E\u090F\u0902-\u092A\u0930-\u0926\u093E\u090F\u0902 \u0924\u094D\ + \u0930\u093F\u0915 \u092E\u0930\u094B\u0921\u093C" + input_correct_responses: + - D + input_question: "\u090F\u0915 36 \u0935\u0930\u094D\u0937\u0940\u092F \u092A\u0941\ + \u0930\u0941\u0937 \u092A\u0940\u0920 \u0915\u0947 \u0928\u093F\u091A\u0932\u0947\ + \ \u0939\u093F\u0938\u094D\u0938\u0947 \u092E\u0947\u0902 \u0926\u0930\u094D\ + \u0926 \u0915\u0947 3 \u0938\u092A\u094D\u0924\u093E\u0939 \u0915\u0947 \u0907\ + \u0924\u093F\u0939\u093E\u0938 \u0915\u0947 \u0938\u093E\u0925 \u0915\u093E\u0930\ + \u094D\u092F\u093E\u0932\u092F \u092E\u0947\u0902 \u0909\u092A\u0938\u094D\u0925\ + \u093F\u0924 \u0939\u094B\u0924\u093E \u0939\u0948\u0964 \u0935\u0939 \u0915\ + \u093F\u0938\u0940 \u092D\u0940 \u0939\u093E\u0932\u093F\u092F\u093E \u0906\u0918\ + \u093E\u0924 \u0938\u0947 \u0907\u0928\u0915\u093E\u0930 \u0915\u0930\u0924\u093E\ + \ \u0939\u0948 \u0932\u0947\u0915\u093F\u0928 \u0915\u0939\u0924\u093E \u0939\ + \u0948 \u0915\u093F \u0935\u0939 \u0905\u092A\u0928\u0947 \u0915\u093E\u092E\ + \ \u0915\u0947 \u0932\u093F\u090F \u0926\u093F\u0928 \u092E\u0947\u0902 \u0915\ + \u0908 \u092C\u093E\u0930 \u0905\u092A\u0928\u0947 \u091F\u094D\u0930\u0915\ + \ \u092E\u0947\u0902 \u091A\u0922\u093C\u0924\u093E \u0914\u0930 \u0909\u0924\ + \u0930\u0924\u093E \u0939\u0948\u0964 \u092A\u094D\u0930\u0935\u0923 \u0938\u094D\ + \u0925\u093F\u0924\u093F \u092E\u0947\u0902 \u0930\u094B\u0917\u0940 \u0915\u0940\ + \ \u091C\u093E\u0902\u091A \u0938\u0947 \u092C\u093E\u0908\u0902 \u0913\u0930\ + \ \u090F\u0915 \u0917\u0939\u0930\u0940 \u0924\u094D\u0930\u093F\u0915 \u0938\ + \u0932\u094D\u0915\u0938, \u0926\u093E\u0908\u0902 \u0913\u0930 \u090F\u0915\ + \ \u092A\u0936\u094D\u091A \u0905\u0935\u0930 \u092A\u093E\u0930\u094D\u0936\ + \u094D\u0935 \u0915\u094B\u0923 \u0914\u0930 \u090F\u0915 \u0932\u0941\u0902\ + \u092C\u094B\u0938\u0948\u0915\u094D\u0930\u0932 \u091C\u0902\u0915\u094D\u0936\ + \u0928 \u0915\u093E \u092A\u0924\u093E \u091A\u0932\u0924\u093E \u0939\u0948\ + \ \u091C\u094B \u0938\u0902\u092A\u0940\u0921\u093C\u0928 \u092A\u0930 \u0938\ + \u094D\u0935\u0924\u0902\u0924\u094D\u0930 \u0930\u0942\u092A \u0938\u0947 \u0909\ + \u092D\u0930\u0924\u093E \u0939\u0948\u0964 \u0938\u092C\u0938\u0947 \u0938\u0902\ + \u092D\u093E\u0935\u093F\u0924 \u0928\u093F\u0926\u093E\u0928 \u0939\u0948" + - input_choice_list: + A: "\u0921\u094B\u092A\u093E\u092E\u093E\u0907\u0928" + B: "\u0917\u094D\u0932\u0942\u091F\u093E\u092E\u0947\u091F" + C: "\u0928\u0949\u0930\u092A\u0947\u0928\u0947\u092B\u094D\u0930\u093F\u0928" + D: "\u0938\u0947\u0930\u094B\u091F\u094B\u0928\u093F\u0928" + input_correct_responses: + - D + input_question: "\u090F\u0915 \u092A\u0939\u0932\u0947 \u0938\u0947 \u0938\u094D\ + \u0935\u0938\u094D\u0925 32 \u0935\u0930\u094D\u0937\u0940\u092F \u092E\u0939\ + \u093F\u0932\u093E \u0905\u092A\u0928\u0947 \u092A\u0924\u093F \u0915\u0940\ + \ \u0915\u093E\u0930 \u0926\u0941\u0930\u094D\u0918\u091F\u0928\u093E \u092E\ + \u0947\u0902 \u092E\u0943\u0924\u094D\u092F\u0941 \u0915\u0947 8 \u092E\u0939\ + \u0940\u0928\u0947 \u092C\u093E\u0926 \u091A\u093F\u0915\u093F\u0924\u094D\u0938\ + \u0915 \u0915\u0947 \u092A\u093E\u0938 \u0906\u0924\u0940 \u0939\u0948\u0964\ + \ \u0909\u0938 \u0938\u092E\u092F \u0938\u0947, \u0909\u0938\u0915\u0940 \u092D\ + \u0942\u0916 \u0915\u092E \u0939\u094B \u0917\u0908 \u0939\u0948 \u0914\u0930\ + \ \u0938\u094B\u0928\u0947 \u092E\u0947\u0902 \u0915\u0920\u093F\u0928\u093E\ + \u0908 \u0939\u094B \u0930\u0939\u0940 \u0939\u0948\u0964 \u0935\u0939 \u092C\ + \u0924\u093E\u0924\u0940 \u0939\u0948\u0902 \u0915\u093F \u0935\u0939 \u0905\ + \u0915\u094D\u0938\u0930 \u0909\u0926\u093E\u0938 \u0930\u0939\u0924\u0940 \u0939\ + \u0948\u0902 \u0914\u0930 \u092C\u093E\u0930-\u092C\u093E\u0930 \u0930\u094B\ + \u0924\u0940 \u0939\u0948\u0902\u0964 \u0935\u0939 \u0905\u092A\u0928\u093E\ + \ \u0918\u0930 \u091B\u094B\u0921\u093C\u0928\u0947 \u0938\u0947 \u092A\u0939\ + \u0932\u0947 \u092A\u093E\u0902\u091A \u092C\u093E\u0930 \u0926\u0930\u0935\u093E\ + \u091C\u0947 \u0915\u0947 \u0924\u093E\u0932\u0947 \u0915\u0940 \u091C\u093E\ + \u0902\u091A \u0915\u0930 \u0930\u0939\u0940 \u0939\u0948 \u0914\u0930 \u0909\ + \u0938\u0947 \u0907\u0938\u094D\u0924\u0947\u092E\u093E\u0932 \u0915\u0930\u0928\ + \u0947 \u0938\u0947 \u092A\u0939\u0932\u0947 \u091F\u0949\u092F\u0932\u0947\u091F\ + \ \u092A\u0947\u092A\u0930 \u0915\u0947 \u0920\u0940\u0915 \u092A\u093E\u0902\ + \u091A \u091F\u0941\u0915\u0921\u093C\u0947 \u0917\u093F\u0928\u0928\u0947 \u092A\ + \u0921\u093C\u0924\u0947 \u0939\u0948\u0902\u0964 \u0935\u0939 \u0915\u0939\u0924\ + \u0940 \u0939\u0948\u0902 \u0915\u093F \u0935\u0939 \u0939\u092E\u0947\u0936\ + \u093E \u0938\u0947 \u092A\u0930\u092B\u0947\u0915\u094D\u0936\u0928\u093F\u0938\ + \u094D\u091F \u0930\u0939\u0940 \u0939\u0948\u0902 \u0932\u0947\u0915\u093F\u0928\ + \ \u092F\u0947 \u0906\u0917\u094D\u0930\u0939 \u0914\u0930 \u0905\u0928\u0941\ + \u0937\u094D\u0920\u093E\u0928 \u0928\u090F \u0939\u0948\u0902\u0964 \u092B\u093E\ + \u0930\u094D\u092E\u093E\u0915\u094B\u0925\u0947\u0930\u0947\u092A\u0940 \u0915\ + \u094B \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u092E\u0947\ + \u0902 \u0938\u0947 \u0915\u093F\u0938 \u0928\u094D\u092F\u0942\u0930\u094B\u091F\ + \u094D\u0930\u093E\u0902\u0938\u092E\u0940\u091F\u0930 \u092A\u0930 \u0932\u0915\ + \u094D\u0937\u093F\u0924 \u0915\u093F\u092F\u093E \u091C\u093E\u0928\u093E \u091A\ + \u093E\u0939\u093F\u090F?" + - input_choice_list: + A: "\u090F\u0932\u0930\u094D\u091C\u0940 \u0930\u093F\u0928\u093F\u0925\u093F\ + \u0938" + B: "\u090F\u092A\u0938\u094D\u091F\u0940\u0928 \u092C\u093E\u0930 \u0935\u093E\ + \u092F\u0930\u0938" + C: "\u092E\u093E\u0907\u0915\u094B\u092A\u094D\u0932\u093E\u091C\u094D\u092E\ + \u093E \u0928\u093F\u092E\u094B\u0928\u093F\u092F\u093E" + D: rhinovirus + input_correct_responses: + - D + input_question: "\u090F\u0915 44 \u0935\u0930\u094D\u0937\u0940\u092F \u0935\u094D\ + \u092F\u0915\u094D\u0924\u093F \u0917\u0932\u0947 \u092E\u0947\u0902 \u0916\u0930\ + \u093E\u0936, \u0905\u0928\u0941\u0924\u094D\u092A\u093E\u0926\u0915 \u0916\u093E\ + \u0902\u0938\u0940, \u0928\u093E\u0915 \u092C\u0939\u0928\u0947 \u0914\u0930\ + \ \u0932\u0932\u093E\u091F \u0938\u093F\u0930\u0926\u0930\u094D\u0926 \u0915\ + \u0947 3 \u0926\u093F\u0928 \u0915\u0947 \u0907\u0924\u093F\u0939\u093E\u0938\ + \ \u0915\u0947 \u0915\u093E\u0930\u0923 \u0915\u093E\u0930\u094D\u092F\u093E\ + \u0932\u092F \u0906\u0924\u093E \u0939\u0948\u0964 \u0909\u0928\u0915\u093E\ + \ \u0915\u0939\u0928\u093E \u0939\u0948 \u0915\u093F \u0938\u093F\u0930\u0926\ + \u0930\u094D\u0926 \u0938\u0941\u092C\u0939 \u0915\u0947 \u0938\u092E\u092F\ + \ \u0905\u0927\u093F\u0915 \u0939\u094B\u0924\u093E \u0939\u0948 \u0914\u0930\ + \ \u0907\u092C\u0941\u092A\u094D\u0930\u094B\u092B\u0947\u0928 \u0938\u0947\ + \ \u0915\u0941\u091B \u0930\u093E\u0939\u0924 \u092E\u093F\u0932\u0924\u0940\ + \ \u0939\u0948\u0964 \u0909\u0928\u094D\u0939\u0947\u0902 \u0938\u093E\u0902\ + \u0938 \u0915\u0940 \u0924\u0915\u0932\u0940\u092B \u0928\u0939\u0940\u0902\ + \ \u0939\u0941\u0908 \u0939\u0948. \u091A\u093F\u0915\u093F\u0924\u094D\u0938\ + \u093E \u0907\u0924\u093F\u0939\u093E\u0938 \u0909\u0932\u094D\u0932\u0947\u0916\ + \u0928\u0940\u092F \u0928\u0939\u0940\u0902 \u0939\u0948. \u0935\u0939 \u0926\ + \u0930\u094D\u0926 \u0915\u0947 \u0932\u093F\u090F \u0907\u092C\u0941\u092A\u094D\ + \u0930\u094B\u092B\u0947\u0928 \u0915\u0947 \u0905\u0932\u093E\u0935\u093E \u0915\ + \u094B\u0908 \u0926\u0935\u093E \u0928\u0939\u0940\u0902 \u0932\u0947\u0924\u093E\ + \ \u0939\u0948\u0964 \u092E\u0939\u0924\u094D\u0935\u092A\u0942\u0930\u094D\u0923\ + \ \u0938\u0902\u0915\u0947\u0924 \u0924\u093E\u092A\u092E\u093E\u0928 37.4 \u0921\ + \u093F\u0917\u094D\u0930\u0940 \u0938\u0947\u0932\u094D\u0938\u093F\u092F\u0938\ + \ (99.4 \u0921\u093F\u0917\u094D\u0930\u0940 \u092B\u093C\u093E\u0930\u0947\u0928\ + \u0939\u093E\u0907\u091F), \u0928\u093E\u0921\u093C\u0940 88/\u092E\u093F\u0928\ + \u091F, \u0936\u094D\u0935\u0938\u0928 18/\u092E\u093F\u0928\u091F \u0914\u0930\ + \ \u0930\u0915\u094D\u0924\u091A\u093E\u092A 120/84 \u092E\u093F\u092E\u0940\ + \ \u090F\u091A\u091C\u0940 \u0939\u0948\u0902\u0964 \u0928\u093E\u0921\u093C\ + \u093F\u092F\u094B\u0902 \u0915\u0940 \u091C\u093E\u0902\u091A \u0938\u0947\ + \ \u090F\u0930\u093F\u0925\u0947\u092E\u0947\u091F\u0938 \u0936\u094D\u0932\u0947\ + \u0937\u094D\u092E\u093E \u091D\u093F\u0932\u094D\u0932\u0940 \u0915\u093E \u092A\ + \u0924\u093E \u091A\u0932\u0924\u093E \u0939\u0948\u0964 \u0917\u0932\u0947\ + \ \u0915\u0940 \u091C\u093E\u0902\u091A \u0938\u0947 \u092A\u0940\u091B\u0947\ + \ \u0915\u0947 \u0911\u0930\u094B\u092B\u0930\u0940\u0928\u0915\u094D\u0938\ + \ \u092A\u0930 \u090F\u0930\u093F\u0925\u0947\u092E\u093E \u0914\u0930 \u092B\ + \u0949\u0932\u093F\u0915\u094D\u092F\u0941\u0932\u0930 \u0932\u093F\u092E\u094D\ + \u092B\u094B\u0907\u0921 \u0939\u093E\u0907\u092A\u0930\u092A\u094D\u0932\u093E\ + \u0938\u093F\u092F\u093E \u0915\u093E \u092A\u0924\u093E \u091A\u0932\u0924\u093E\ + \ \u0939\u0948\u0964 \u0915\u094B\u0908 \u0938\u094D\u092A\u0937\u094D\u091F\ + \ \u0917\u094D\u0930\u0940\u0935\u093E \u090F\u0921\u0947\u0928\u094B\u092A\u0948\ + \u0925\u0940 \u0928\u0939\u0940\u0902 \u0939\u0948\u0964 \u092B\u0947\u092B\u0921\ + \u093C\u0947 \u0917\u0941\u0926\u093E\u092D\u094D\u0930\u0902\u0936 \u0915\u0947\ + \ \u0932\u093F\u090F \u0938\u094D\u092A\u0937\u094D\u091F \u0939\u0948\u0902\ + \u0964 \u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924 \u092E\u0947\ + \u0902 \u0938\u0947 \u0915\u094C\u0928 \u0938\u093E \u0907\u0938 \u0930\u094B\ + \u0917\u0940 \u0915\u0947 \u0932\u0915\u094D\u0937\u0923\u094B\u0902 \u0915\u093E\ + \ \u0938\u092C\u0938\u0947 \u0938\u0902\u092D\u093E\u0935\u093F\u0924 \u0915\ + \u093E\u0930\u0923 \u0939\u0948?" + - input_choice_list: + A: "\u092A\u0942\u0930\u094D\u0935\u0915\u093E\u0932 \u0938\u094D\u0915\u0947\ + \u0932\u0940\u0928" + B: "\u0932\u093E\u091F\u093F\u0938\u094D\u0938\u093F\u092E\u0941\u0938 \u0921\ + \u094B\u0930\u0938\u0940" + C: "\u092A\u0947\u0915\u094D\u091F\u094B\u0930\u0932\u093F\u0938 \u092E\u093E\ + \u0907\u0928\u0930" + D: "\u0915\u094D\u0935\u093E\u0921\u094D\u0930\u0947\u091F\u094D\u0938 \u0932\ + \u0948\u0902\u092C\u094B\u0930\u092E" + input_correct_responses: + - C + input_question: "\u090F\u0915 22 \u0935\u0930\u094D\u0937\u0940\u092F \u092A\u0941\ + \u0930\u0941\u0937 \u092E\u0948\u0930\u093E\u0925\u0928 \u0927\u093E\u0935\u0915\ + \ \u0932\u0902\u092C\u0940 \u0926\u0942\u0930\u0940 \u0924\u0915 \u0926\u094C\ + \u0921\u093C\u0928\u0947 \u092A\u0930 \u0926\u093E\u0939\u093F\u0928\u0940 \u0913\ + \u0930 \u0915\u0940 \u092A\u0938\u0932\u0940 \u092E\u0947\u0902 \u0926\u0930\ + \u094D\u0926 \u0915\u0940 \u0936\u093F\u0915\u093E\u092F\u0924 \u0915\u0947\ + \ \u0938\u093E\u0925 \u0915\u093E\u0930\u094D\u092F\u093E\u0932\u092F \u092E\ + \u0947\u0902 \u0906\u0924\u093E \u0939\u0948\u0964 \u0936\u093E\u0930\u0940\u0930\ + \u093F\u0915 \u092A\u0930\u0940\u0915\u094D\u0937\u0923 \u0938\u0947 \u0939\u0943\ + \u0926\u092F \u0914\u0930 \u092B\u0947\u092B\u0921\u093C\u094B\u0902 \u0915\u0940\ + \ \u0938\u093E\u092E\u093E\u0928\u094D\u092F \u0938\u094D\u0925\u093F\u0924\u093F\ + \ \u0914\u0930 \u0926\u093E\u0939\u093F\u0928\u0940 \u0913\u0930 4-5 \u092A\u0938\ + \u0932\u093F\u092F\u094B\u0902 \u092A\u0930 \u0938\u093E\u0901\u0938 \u091B\u094B\ + \u0921\u093C\u0928\u0947 \u0915\u0940 \u0938\u092E\u0938\u094D\u092F\u093E \u0915\ + \u093E \u092A\u0924\u093E \u091A\u0932\u0924\u093E \u0939\u0948\u0964 \u092A\ + \u094D\u0930\u0924\u094D\u092F\u0915\u094D\u0937 \u0935\u093F\u0927\u093F \u0915\ + \u093E \u0909\u092A\u092F\u094B\u0917 \u0915\u0930\u0915\u0947 \u0907\u0938\ + \ \u0936\u093F\u0925\u093F\u0932\u0924\u093E \u0915\u094B \u0920\u0940\u0915\ + \ \u0915\u0930\u0928\u0947 \u092E\u0947\u0902 \u0928\u093F\u092E\u094D\u0928\ + \u0932\u093F\u0916\u093F\u0924 \u092E\u0947\u0902 \u0938\u0947 \u0915\u094C\u0928\ + \ \u0938\u0940 \u092E\u093E\u0902\u0938\u092A\u0947\u0936\u093F\u092F\u093E\u0901\ + \ \u092F\u093E \u092E\u093E\u0902\u0938\u092A\u0947\u0936\u0940 \u0938\u092E\ + \u0942\u0939 \u0938\u092C\u0938\u0947 \u0905\u0927\u093F\u0915 \u0909\u092A\u092F\ + \u094B\u0917\u0940 \u0939\u094B\u0902\u0917\u0947?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_medicine +tag: mmlu_hi_llama_other_tasks +task: mmlu_hi_llama_professional_medicine +task_alias: professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94dc2e73f880e4d89208957cfa20484312c61e85 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_professional_psychology.yaml @@ -0,0 +1,170 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u091C\u093F\u0938 \u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E\ + \ \u092E\u0947\u0902 \u0906\u092A \u0928\u093E\u092E\u093E\u0902\u0915\u093F\ + \u0924 \u0939\u0948\u0902 \u0909\u0938\u0947 \u091B\u094B\u0921\u093C\u0915\ + \u0930 \u0905\u092A\u0928\u0947 \u0917\u094D\u0930\u093E\u0939\u0915 \u0915\ + \u094B \u092A\u094D\u0930\u0924\u094D\u092F\u0947\u0915 \u0915\u093E\u0930\ + \u094D\u092F\u0915\u094D\u0930\u092E \u0915\u0947 \u092B\u093E\u092F\u0926\ + \u0947 \u0914\u0930 \u0928\u0941\u0915\u0938\u093E\u0928 \u092C\u0924\u093E\ + \u090F\u0902 \u091C\u093F\u0928\u0915\u0947 \u092C\u093E\u0930\u0947 \u092E\ + \u0947\u0902 \u0906\u092A \u091C\u093E\u0928\u0924\u0947 \u0939\u0948\u0902" + B: "\u0905\u092A\u0928\u0947 \u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u094B\ + \ \u0909\u0938 \u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\ + \u0940 \u0905\u0928\u0941\u0936\u0902\u0938\u093E \u0915\u0930\u0947\u0902\ + \ \u091C\u093F\u0938\u092E\u0947\u0902 \u0906\u092A \u0928\u093E\u092E\u093E\ + \u0902\u0915\u093F\u0924 \u0939\u0948\u0902 \u0914\u0930 \u0906\u092A\u0915\ + \u094B \u092A\u094D\u0930\u093E\u092A\u094D\u0924 \u0939\u094B\u0928\u0947\ + \ \u0935\u093E\u0932\u0947 $50 \u092C\u094B\u0928\u0938 \u0915\u0947 \u092C\ + \u093E\u0930\u0947 \u092E\u0947\u0902 \u092C\u0924\u093E\u090F\u0902" + C: "\u0905\u092A\u0928\u0947 \u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u094B\ + \ \u0909\u0938 \u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\ + \u0940 \u0905\u0928\u0941\u0936\u0902\u0938\u093E \u0915\u0930\u0947\u0902\ + \ \u091C\u093F\u0938\u092E\u0947\u0902 \u0906\u092A \u0928\u093E\u092E\u093E\ + \u0902\u0915\u093F\u0924 \u0939\u0948\u0902 \u0914\u0930 \u0915\u093E\u0930\ + \u094D\u092F\u0915\u094D\u0930\u092E \u092E\u0947\u0902 \u0906\u092A\u0915\ + \u0947 \u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u0947 \u0916\u093E\u0924\ + \u0947 \u092E\u0947\u0902 $50 \u0915\u093E \u092C\u094B\u0928\u0938 \u091C\ + \u092E\u093E \u0915\u0930\u0928\u0947 \u0915\u0940 \u092A\u0947\u0936\u0915\ + \u0936 \u0915\u0930\u0947\u0902" + D: "\u0905\u092A\u0928\u0947 \u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u094B\ + \ \u0909\u0938 \u092A\u094D\u0930\u0924\u094D\u092F\u0947\u0915 \u0915\u093E\ + \u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\u0947 \u092B\u093E\u092F\ + \u0926\u0947 \u0914\u0930 \u0928\u0941\u0915\u0938\u093E\u0928 \u092C\u0924\ + \u093E\u090F\u0902 \u091C\u093F\u0938\u0915\u0947 \u092C\u093E\u0930\u0947\ + \ \u092E\u0947\u0902 \u0906\u092A \u091C\u093E\u0928\u0924\u0947 \u0939\u0948\ + \u0902, \u0932\u0947\u0915\u093F\u0928 \u092F\u0926\u093F \u0906\u092A\u0915\ + \u093E \u0917\u094D\u0930\u093E\u0939\u0915 \u0906\u092A\u0915\u0947 \u0915\ + \u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u092E\u0947\u0902 \u0928\ + \u093E\u092E\u093E\u0902\u0915\u0928 \u0915\u0930\u0924\u093E \u0939\u0948\ + \ \u0924\u094B $50 \u092C\u094B\u0928\u0938 \u0915\u093E \u0926\u093E\u0935\ + \u093E \u0928 \u0915\u0930\u0947\u0902\u0964" + input_correct_responses: + - D + input_question: "\u0906\u092A\u0915\u093E \u090F\u0915 \u0925\u0947\u0930\u0947\ + \u092A\u0940 \u0917\u094D\u0930\u093E\u0939\u0915 \u090F\u0915 \u0905\u091A\u094D\ + \u091B\u0947 \u0935\u091C\u0928 \u0918\u091F\u093E\u0928\u0947 \u0915\u0947\ + \ \u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\u0947 \u092C\ + \u093E\u0930\u0947 \u092E\u0947\u0902 \u0906\u092A\u0915\u0940 \u0938\u0932\u093E\ + \u0939 \u092A\u0942\u091B\u0924\u093E \u0939\u0948\u0964 \u0906\u092A\u0928\u0947\ + \ \u0938\u092E\u0941\u0926\u093E\u092F \u092E\u0947\u0902 \u0915\u093E\u0930\ + \u094D\u092F\u0915\u094D\u0930\u092E\u094B\u0902 \u0915\u0940 \u091C\u093E\u0902\ + \u091A \u0915\u0940 \u0939\u0948 \u0914\u0930 \u091C\u093F\u0938\u0947 \u0906\ + \u092A \u0938\u092C\u0938\u0947 \u0905\u091A\u094D\u091B\u093E \u092E\u093E\u0928\ + \u0924\u0947 \u0939\u0948\u0902 \u0909\u0938\u092E\u0947\u0902 \u0928\u093E\u092E\ + \u093E\u0902\u0915\u093F\u0924 \u0939\u0948\u0902\u0964 \u092F\u0939 \u0915\u093E\ + \u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0905\u092A\u0928\u0947 \u0938\u0902\ + \u0930\u0915\u094D\u0937\u0915\u094B\u0902 \u0915\u094B \u0915\u093E\u0930\u094D\ + \u092F\u0915\u094D\u0930\u092E \u092E\u0947\u0902 \u0932\u093E\u090F \u091C\u093E\ + \u0928\u0947 \u0935\u093E\u0932\u0947 \u092A\u094D\u0930\u0924\u094D\u092F\u0947\ + \u0915 \u0928\u090F \u0935\u094D\u092F\u0915\u094D\u0924\u093F \u0915\u0947\ + \ \u0932\u093F\u090F $50 \u0915\u093E \u092C\u094B\u0928\u0938 \u092A\u094D\u0930\ + \u0926\u093E\u0928 \u0915\u0930\u0924\u093E \u0939\u0948\u0964 \u0907\u0928\ + \ \u092A\u0930\u093F\u0938\u094D\u0925\u093F\u0924\u093F\u092F\u094B\u0902 \u092E\ + \u0947\u0902, \u0906\u092A\u0915\u0940 \u0938\u092C\u0938\u0947 \u0909\u092A\ + \u092F\u0941\u0915\u094D\u0924 \u092A\u094D\u0930\u0924\u093F\u0915\u094D\u0930\ + \u093F\u092F\u093E \u0939\u094B\u0917\u0940" + - input_choice_list: + A: "\u092E\u093E\u0927\u094D\u092F \u0915\u0940 \u0924\u0941\u0932\u0928\u093E\ + \ \u092E\u0947\u0902 \u091A\u0930\u092E \u0938\u094D\u0915\u094B\u0930 \u0915\ + \u0947 \u092A\u094D\u0930\u0924\u093F \u0915\u092E \u0938\u0902\u0935\u0947\ + \u0926\u0928\u0936\u0940\u0932" + B: "\u0935\u093F\u0937\u092E \u0935\u093F\u0924\u0930\u0923\u094B\u0902 \u0915\ + \u0947 \u0932\u093F\u090F \u0905\u0927\u093F\u0915 \u0909\u092A\u092F\u094B\ + \u0917\u0940" + C: "\u0905\u0924\u094D\u092F\u0927\u093F\u0915 \u092E\u0942\u0932\u094D\u092F\ + \u094B\u0902 \u0914\u0930 \u0905\u0924\u094D\u092F\u0927\u093F\u0915 \u0935\ + \u093F\u0937\u092E \u0935\u093F\u0924\u0930\u0923\u094B\u0902 \u0915\u0947\ + \ \u092A\u094D\u0930\u0924\u093F \u0938\u0902\u0935\u0947\u0926\u0928\u0936\ + \u0940\u0932" + D: "\u0938\u092C\u0938\u0947 \u0905\u0927\u093F\u0915 \u092C\u093E\u0930 \u0906\ + \u0928\u0947 \u0935\u093E\u0932\u0940 \u0938\u0902\u0916\u094D\u092F\u093E" + input_correct_responses: + - D + input_question: "\u0915\u0947\u0902\u0926\u094D\u0930\u0940\u092F \u092A\u094D\ + \u0930\u0935\u0943\u0924\u094D\u0924\u093F \u0915\u094B \u092E\u093E\u092A\u0928\ + \u0947 \u0915\u0947 \u0924\u0940\u0928 \u0924\u0930\u0940\u0915\u0947 \u0939\ + \u0948\u0902: \u092E\u093E\u0927\u094D\u092F, \u092E\u093E\u0927\u094D\u092F\ + \u093F\u0915\u093E \u0914\u0930 \u092C\u0939\u0941\u0932\u0915\u0964 \u0909\u0928\ + \u0915\u0947 \u092C\u093E\u0930\u0947 \u092E\u0947\u0902 \u0906\u092A\u0915\u0940\ + \ \u091C\u093E\u0928\u0915\u093E\u0930\u0940 \u0938\u0947, \u092E\u094B\u0921\ + \ \u0915\u094D\u092F\u093E \u0939\u0948?" + - input_choice_list: + A: "\u0935\u094D\u092F\u0915\u094D\u0924\u093F\u0935\u093E\u0926." + B: "\u0935\u094D\u092F\u0915\u094D\u0924\u093F\u0935\u093E\u0926 \u0914\u0930\ + \ \u0938\u0924\u094D\u0924\u093E \u0915\u0940 \u0926\u0942\u0930\u0940." + C: "\u0936\u0915\u094D\u0924\u093F \u0926\u0942\u0930\u0940 \u0914\u0930 \u092A\ + \u0941\u0930\u0941\u0937\u0924\u094D\u0935\u0964" + D: "\u0905\u0928\u093F\u0936\u094D\u091A\u093F\u0924\u0924\u093E \u092A\u0930\ + \u093F\u0939\u093E\u0930\u0964" + input_correct_responses: + - A + input_question: "\u0939\u0949\u092B\u0938\u094D\u091F\u0947\u0921 (1980) \u0915\ + \u0947 \u092A\u093E\u0902\u091A \u0938\u093E\u0902\u0938\u094D\u0915\u0943\u0924\ + \u093F\u0915 \u0906\u092F\u093E\u092E\u094B\u0902 \u0915\u0947 \u0938\u0902\u0926\ + \u0930\u094D\u092D \u092E\u0947\u0902, \u0938\u0902\u092F\u0941\u0915\u094D\u0924\ + \ \u0930\u093E\u091C\u094D\u092F \u0905\u092E\u0947\u0930\u093F\u0915\u093E\ + \ \u092A\u0948\u092E\u093E\u0928\u0947 \u0915\u0947 \u0936\u0940\u0930\u094D\ + \u0937 \u092A\u0930 \u0939\u0948:" + - input_choice_list: + A: "\u090F\u0915 \u0915\u0932\u094D\u092A\u0928\u093E \u0939\u0948 \u091C\u094B\ + \ \u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u094B \u0935\u093E\u0938\u094D\ + \u0924\u0935\u093F\u0915\u0924\u093E \u0938\u0947 \u0935\u093F\u091A\u0932\ + \u093F\u0924 \u0915\u0930\u0924\u0940 \u0939\u0948\u0964" + B: "\u091A\u093F\u0915\u093F\u0924\u094D\u0938\u0915 \u0915\u0947 \u092A\u094D\ + \u0930\u0924\u093F "\u092E\u093F\u0936\u094D\u0930\u093F\u0924 \u092D\ + \u093E\u0935\u0928\u093E\u0913\u0902" \u0915\u093E \u092A\u094D\u0930\ + \u0924\u093F\u0928\u093F\u0927\u093F\u0924\u094D\u0935 \u0915\u0930\u0924\u093E\ + \ \u0939\u0948\u0964" + C: ""\u0905\u092D\u093F\u0928\u092F" \u0915\u093E \u090F\u0915 \u0930\ + \u0942\u092A \u0939\u0948\u0964" + D: "\u0917\u094D\u0930\u093E\u0939\u0915 \u0915\u0947 \u0935\u094D\u092F\u0915\ + \u094D\u0924\u093F\u0917\u0924 \u0914\u0930 \u0938\u093E\u092E\u0942\u0939\ + \u093F\u0915 \u0905\u091A\u0947\u0924\u0928 \u0915\u094B \u0926\u0930\u094D\ + \u0936\u093E\u0924\u093E \u0939\u0948\u0964" + input_correct_responses: + - D + input_question: "\u0915\u093E\u0930\u094D\u0932 \u091C\u0902\u0917 \u0915\u093E\ + \ \u092E\u093E\u0928\u0928\u093E \u0925\u093E \u0915\u093F \u0917\u094D\u0930\ + \u093E\u0939\u0915 \u0915\u093E \u0938\u094D\u0925\u093E\u0928\u093E\u0902\u0924\ + \u0930\u0923:" + - input_choice_list: + A: "\u090F\u0915-\u0926\u0942\u0938\u0930\u0947 \u0938\u0947 \u0905\u0938\u0902\ + \u092C\u0902\u0927\u093F\u0924 \u0939\u0948\u0902 \u0932\u0947\u0915\u093F\ + \u0928 \u092E\u093E\u0928\u0926\u0902\u0921 \u0915\u0947 \u0938\u093E\u0925\ + \ \u092E\u0927\u094D\u092F\u092E \u0930\u0942\u092A \u0938\u0947 \u0938\u0939\ + \u0938\u0902\u092C\u0926\u094D\u0927 \u0939\u0948\u0902" + B: "\u090F\u0915 \u0926\u0942\u0938\u0930\u0947 \u0915\u0947 \u0938\u093E\u0925\ + \ \u0915\u092E \u0938\u0939\u0938\u0902\u092C\u0902\u0927 \u0914\u0930 \u092E\ + \u093E\u0928\u0926\u0902\u0921 \u0915\u0947 \u0938\u093E\u0925 \u0915\u092E\ + \ \u0938\u0939\u0938\u0902\u092C\u0902\u0927 \u0939\u0948\u0902" + C: "\u090F\u0915-\u0926\u0942\u0938\u0930\u0947 \u0915\u0947 \u0938\u093E\u0925\ + \ \u0905\u0924\u094D\u092F\u0927\u093F\u0915 \u0905\u0902\u0924\u0930\u094D\ + \u0938\u0902\u092C\u0902\u0927\u093F\u0924 \u0939\u0948\u0902 \u0914\u0930\ + \ \u092E\u093E\u0928\u0926\u0902\u0921 \u0915\u0947 \u0938\u093E\u0925 \u092E\ + \u0927\u094D\u092F\u092E \u0930\u0942\u092A \u0938\u0947 \u0938\u0939\u0938\ + \u0902\u092C\u0926\u094D\u0927 \u0939\u0948\u0902" + D: "\u092E\u093E\u0928\u0926\u0902\u0921 \u0915\u0947 \u0938\u093E\u0925 \u0915\ + \u092E \u0938\u0939\u0938\u0902\u092C\u0902\u0927 \u0939\u0948\u0902, \u092C\ + \u094D\u092F\u0942\u0930\u094B \u090F\u0915 \u0926\u0942\u0938\u0930\u0947\ + \ \u0915\u0947 \u0938\u093E\u0925 \u092E\u0927\u094D\u092F\u092E \u0930\u0942\ + \u092A \u0938\u0947 \u0938\u0939\u0938\u0902\u092C\u0926\u094D\u0927 \u0939\ + \u0948\u0902" + input_correct_responses: + - A + input_question: "\u092D\u0935\u093F\u0937\u094D\u092F\u0935\u093E\u0923\u0940\ + \ \u0915\u0947 \u092A\u094D\u0930\u092F\u094B\u091C\u0928\u094B\u0902 \u0915\ + \u0947 \u0932\u093F\u090F \u090F\u0915\u093E\u0927\u093F\u0915 \u092A\u094D\u0930\ + \u0924\u093F\u0917\u092E\u0928 \u0938\u092E\u0940\u0915\u0930\u0923 \u0915\u0947\ + \ \u0928\u093F\u0930\u094D\u092E\u093E\u0923 \u092E\u0947\u0902, \u0909\u092A\ + \u093E\u092F\u094B\u0902 \u0915\u093E \u0907\u0937\u094D\u091F\u0924\u092E \u0938\ + \u0902\u092F\u094B\u091C\u0928 \u0935\u0939 \u0939\u094B\u0924\u093E \u0939\u0948\ + \ \u091C\u093F\u0938\u092E\u0947\u0902 \u092D\u0935\u093F\u0937\u094D\u092F\u0935\ + \u0915\u094D\u0924\u093E \u0939\u094B\u0924\u0947 \u0939\u0948\u0902" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_psychology +tag: mmlu_hi_llama_social_sciences_tasks +task: mmlu_hi_llama_professional_psychology +task_alias: professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ed2aa685ce9eb89b72884965f1801b830d7873d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_public_relations.yaml @@ -0,0 +1,122 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0930\u093F\u092A\u094B\u0930\u094D\u091F\u0930 \u0915\u094B \u0905\u0928\ + \u094D\u092F \u091C\u093E\u0928\u0915\u093E\u0930\u0940 \u0926\u0947\u0902\ + \ \u091C\u093F\u0938\u0938\u0947 \u0909\u0938\u0947 \u092F\u0915\u0940\u0928\ + \ \u0939\u094B \u0915\u093F \u0935\u0939 \u0938\u0939\u0940 \u0939\u0948\u0964" + B: "\u0915\u0939\u0947\u0902 \u0915\u093F \u091C\u093E\u0928\u0915\u093E\u0930\ + \u0940 '\u0911\u092B\u093C \u0926 \u0930\u093F\u0915\u0949\u0930\u094D\ + \u0921' \u0939\u0948 \u0914\u0930 \u092C\u093E\u0926 \u092E\u0947\u0902\ + \ \u092A\u094D\u0930\u0938\u093E\u0930\u093F\u0924 \u0915\u0940 \u091C\u093E\ + \u090F\u0917\u0940\u0964" + C: "'\u092E\u0941\u091D\u0947 \u0928\u0939\u0940\u0902 \u092A\u0924\u093E\ + ' \u0915\u0939\u0947\u0902 \u0914\u0930 \u092C\u093E\u0926 \u092E\u0947\ + \u0902 \u091C\u093E\u0928\u0915\u093E\u0930\u0940 \u0909\u092A\u0932\u092C\ + \u094D\u0927 \u0915\u0930\u093E\u0928\u0947 \u0915\u093E \u0935\u093E\u0926\ + \u093E \u0915\u0930\u0947\u0902\u0964" + D: "\u0905\u0928\u092D\u093F\u091C\u094D\u091E \u0926\u093F\u0916\u0928\u0947\ + \ \u0915\u0947 \u092C\u091C\u093E\u092F '\u0915\u094B\u0908 \u091F\u093F\ + \u092A\u094D\u092A\u0923\u0940 \u0928\u0939\u0940\u0902' \u0915\u0939\u0947\ + \u0902\u0964" + input_correct_responses: + - C + input_question: "\u092F\u0926\u093F \u090F\u0915 \u091C\u0928\u0938\u0902\u092A\ + \u0930\u094D\u0915 \u092E\u0940\u0921\u093F\u092F\u093E \u0935\u094D\u092F\u0935\ + \u0938\u093E\u092F\u0940 \u0915\u094B \u0915\u093F\u0938\u0940 \u0930\u093F\u092A\ + \u094B\u0930\u094D\u091F\u0930 \u0915\u0947 \u092A\u094D\u0930\u0936\u094D\u0928\ + \ \u0915\u093E \u0909\u0924\u094D\u0924\u0930 \u0928\u0939\u0940\u0902 \u092A\ + \u0924\u093E \u0939\u0948 \u0924\u094B \u0909\u0938\u0947 \u0915\u094D\u092F\ + \u093E \u0915\u0930\u0928\u093E \u091A\u093E\u0939\u093F\u090F?" + - input_choice_list: + A: "\u0910\u0938\u0947 \u0921\u094B\u092E\u0947\u0928 \u0928\u093E\u092E \u0916\ + \u0930\u0940\u0926\u0947\u0902 \u091C\u093F\u0928\u0915\u093E \u0909\u092A\ + \u092F\u094B\u0917 \u0935\u093F\u092A\u0915\u094D\u0937\u0940 \u0938\u092E\ + \u0942\u0939\u094B\u0902 \u0926\u094D\u0935\u093E\u0930\u093E \u0915\u093F\ + \u092F\u093E \u091C\u093E \u0938\u0915\u0947\u0964" + B: "\u0907\u0938 \u091C\u093E\u0928\u0915\u093E\u0930\u0940 \u0938\u0947 \u0928\ + \u093F\u092A\u091F\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u092C\u094D\ + \u0932\u0949\u0917 \u092A\u0930 \u0917\u0941\u092E\u0928\u093E\u092E \u091F\ + \u093F\u092A\u094D\u092A\u0923\u093F\u092F\u093E\u0901 \u092A\u094B\u0938\u094D\ + \u091F \u0915\u0930\u0947\u0902\u0964" + C: "\u090F\u0915 \u0938\u092E\u093E\u091A\u093E\u0930 \u0935\u093F\u091C\u094D\ + \u091E\u092A\u094D\u0924\u093F \u0924\u0948\u092F\u093E\u0930 \u0915\u0930\ + \u0947\u0902 \u091C\u094B \u0917\u0932\u0924 \u091C\u093E\u0928\u0915\u093E\ + \u0930\u0940 \u0915\u094B \u0916\u093E\u0930\u093F\u091C \u0915\u0930 \u0926\ + \u0947\u0964" + D: "\u0907\u0928 \u0938\u093E\u0907\u091F\u094B\u0902 \u092A\u0930 \u0909\u091C\ + \u093E\u0917\u0930 \u0915\u0940 \u0917\u0908 \u0936\u093F\u0915\u093E\u092F\ + \u0924\u094B\u0902 \u0915\u0947 \u0938\u092E\u093E\u0927\u093E\u0928 \u0915\ + \u0947 \u0932\u093F\u090F \u0928\u0940\u0924\u093F \u092E\u0947\u0902 \u092C\ + \u0926\u0932\u093E\u0935 \u0915\u0930\u0947\u0902\u0964" + input_correct_responses: + - D + input_question: "\u092E\u0941\u0926\u094D\u0926\u094B\u0902 \u0915\u0947 \u092A\ + \u094D\u0930\u092C\u0902\u0927\u0928 \u092E\u0947\u0902, \u0906\u092A\u0915\u0947\ + \ \u0938\u0902\u0917\u0920\u0928 \u0915\u0947 \u092C\u093E\u0930\u0947 \u092E\ + \u0947\u0902 \u0911\u0928\u0932\u093E\u0907\u0928 \u092A\u094B\u0938\u094D\u091F\ + \ \u0915\u0940 \u0917\u0908 \u0928\u0915\u093E\u0930\u093E\u0924\u094D\u092E\ + \u0915 \u092F\u093E \u092D\u094D\u0930\u093E\u092E\u0915 \u091C\u093E\u0928\u0915\ + \u093E\u0930\u0940 \u0915\u094B \u0938\u0902\u092C\u094B\u0927\u093F\u0924 \u0915\ + \u0930\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u0938\u092C\u0938\u0947\ + \ \u0938\u0915\u094D\u0930\u093F\u092F \u0926\u0943\u0937\u094D\u091F\u093F\u0915\ + \u094B\u0923 \u0915\u094D\u092F\u093E \u0939\u0948?" + - input_choice_list: + A: "\u090F\u0915 \u0938\u092E\u0928\u094D\u0935\u093F\u0924 \u092E\u0940\u0921\ + \u093F\u092F\u093E \u092A\u094D\u0930\u0924\u093F\u0915\u094D\u0930\u093F\u092F\ + \u093E \u0925\u0940\u0964" + B: "\u0932\u0917\u093E\u0924\u093E\u0930 \u0938\u0902\u0926\u0947\u0936 \u0938\ + \u0902\u092A\u094D\u0930\u0947\u0937\u093F\u0924 \u0915\u093F\u090F \u0917\ + \u090F\u0964" + C: "\u0906\u0932\u094B\u091A\u0928\u093E\u0913\u0902 \u0915\u094B \u0915\u0948\ + \u0925\u094B\u0932\u093F\u0915 \u091A\u0930\u094D\u091A \u092A\u0930 \u0939\ + \u092E\u0932\u0947 \u0915\u0947 \u0930\u0942\u092A \u092E\u0947\u0902 \u0932\ + \u093F\u092F\u093E \u0917\u092F\u093E\u0964" + D: "\u0935\u0947\u091F\u093F\u0915\u0928 \u0915\u0940 \u0935\u093F\u0936\u094D\ + \u0935\u0938\u0928\u0940\u092F\u0924\u093E \u092C\u0930\u0915\u0930\u093E\u0930\ + \ \u0930\u0916\u0940 \u0917\u0908\u0964" + input_correct_responses: + - C + input_question: "2010 \u092E\u0947\u0902 \u092C\u093E\u0932 \u0936\u094B\u0937\ + \u0923 \u0915\u094B \u091B\u0941\u092A\u093E\u0928\u0947 \u0915\u0947 \u0906\ + \u0930\u094B\u092A\u094B\u0902 \u0915\u0947 \u0938\u092E\u092F \u0935\u0947\u091F\ + \u093F\u0915\u0928 \u0915\u0947 \u092C\u093E\u0930\u0947 \u092E\u0947\u0902\ + \ \u0907\u0928\u092E\u0947\u0902 \u0938\u0947 \u0915\u094C\u0928 \u0938\u093E\ + \ \u0915\u0925\u0928 \u0938\u0924\u094D\u092F \u0939\u0948?" + - input_choice_list: + A: "\u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\u094B \u092A\ + \u0930\u093F\u092D\u093E\u0937\u093F\u0924 \u0915\u0930\u0928\u093E" + B: "\u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\u0940 \u092F\ + \u094B\u091C\u0928\u093E \u092C\u0928\u093E\u0928\u093E" + C: "\u0915\u093E\u0930\u094D\u0930\u0935\u093E\u0908 \u0915\u0930\u0928\u093E\ + \ \u0914\u0930 \u0935\u093F\u091A\u093E\u0930\u094B\u0902 \u0915\u094B \u0915\ + \u093E\u0930\u094D\u092F\u093E\u0928\u094D\u0935\u093F\u0924 \u0915\u0930\u0928\ + \u093E" + D: "\u0915\u093E\u0930\u094D\u092F\u0915\u094D\u0930\u092E \u0915\u093E \u092E\ + \u0942\u0932\u094D\u092F\u093E\u0902\u0915\u0928" + input_correct_responses: + - A + input_question: "\u0928\u093F\u092F\u094B\u091C\u0928 \u092A\u094D\u0930\u0915\ + \u094D\u0930\u093F\u092F\u093E \u0915\u0947 \u0915\u093F\u0938 \u091A\u0930\u0923\ + \ \u092E\u0947\u0902 \u0938\u094D\u0925\u093F\u0924\u093F \u0935\u093F\u0936\ + \u094D\u0932\u0947\u0937\u0923 \u0915\u093F\u092F\u093E \u091C\u093E\u090F\u0917\ + \u093E?" + - input_choice_list: + A: "\u0939\u0930\u093F\u0924 \u0936\u093E\u0902\u0924\u093F" + B: "\u0938\u0902\u092F\u0941\u0915\u094D\u0924 \u0930\u093E\u0937\u094D\u091F\ + \u094D\u0930" + C: "\u0911\u0915\u094D\u0938\u092B\u0947\u092E" + D: "\u0935\u093F\u0936\u094D\u0935 \u0935\u0928\u094D\u092F\u091C\u0940\u0935\ + \u0928 \u0915\u094B\u0937" + input_correct_responses: + - D + input_question: "\u0905\u0930\u094D\u0925 \u0906\u0935\u0930 \u0915\u093F\u0938\ + \ \u0938\u0902\u0917\u0920\u0928 \u0926\u094D\u0935\u093E\u0930\u093E \u0936\ + \u0941\u0930\u0942 \u0915\u093F\u092F\u093E \u0917\u092F\u093E \u090F\u0915\ + \ \u0905\u092D\u093F\u092F\u093E\u0928 \u0925\u093E?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_public_relations +tag: mmlu_hi_llama_social_sciences_tasks +task: mmlu_hi_llama_public_relations +task_alias: public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cff6774008ab1ee2d27c3a0bac6bb452a1f360aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_sociology.yaml @@ -0,0 +1,134 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0938\u092D\u0940 \u0915\u0947 \u0932\u093F\u090F \u092E\u0941\u092B\u094D\ + \u0924 \u0938\u094D\u0935\u093E\u0938\u094D\u0925\u094D\u092F \u0926\u0947\ + \u0916\u092D\u093E\u0932 \u0914\u0930 \u0936\u093F\u0915\u094D\u0937\u093E" + B: "\u090F\u0915 \u0928\u094D\u092F\u0942\u0928\u0924\u092E \u0935\u0947\u0924\ + \u0928" + C: "\u092A\u0942\u0930\u094D\u0923 \u0930\u094B\u091C\u093C\u0917\u093E\u0930" + D: "\u0938\u093E\u0930\u094D\u0935\u092D\u094C\u092E\u093F\u0915 \u0915\u0932\ + \u094D\u092F\u093E\u0923" + input_correct_responses: + - B + input_question: "1948 \u0915\u0947 \u092F\u0941\u0926\u094D\u0927\u094B\u092A\u0930\ + \u093E\u0902\u0924 \u0915\u0932\u094D\u092F\u093E\u0923\u0915\u093E\u0930\u0940\ + \ \u0930\u093E\u091C\u094D\u092F \u0915\u093E \u0928\u093F\u092E\u094D\u0928\ + \u0932\u093F\u0916\u093F\u0924 \u092E\u0947\u0902 \u0938\u0947 \u0915\u094D\u092F\ + \u093E \u092A\u094D\u0930\u0926\u093E\u0928 \u0915\u0930\u0928\u0947 \u0915\u093E\ + \ \u0932\u0915\u094D\u0937\u094D\u092F \u0928\u0939\u0940\u0902 \u0925\u093E\ + :" + - input_choice_list: + A: "\u090F\u0915 \u092E\u0947\u0932\u0947 \u0915\u0947 \u092E\u0948\u0926\u093E\ + \u0928 \u0915\u0940 \u0938\u0935\u093E\u0930\u0940" + B: "\u090F\u0915 \u0938\u0930\u094D\u0915\u0938" + C: "\u090F\u0915 \u0915\u0920\u092A\u0941\u0924\u0932\u0940 \u0925\u093F\u092F\ + \u0947\u091F\u0930" + D: "\u090F\u0915 \u092C\u0948\u0932\u0947" + input_correct_responses: + - C + input_question: "\u092C\u0930\u094D\u091C\u0930 (1963) \u0938\u093E\u092E\u093E\ + \u091C\u093F\u0915 \u0935\u093E\u0938\u094D\u0924\u0935\u093F\u0915\u0924\u093E\ + \ \u0915\u0947 \u0930\u0942\u092A\u0915 \u0915\u0947 \u0930\u0942\u092A \u092E\ + \u0947\u0902 \u0915\u094D\u092F\u093E \u0935\u0930\u094D\u0923\u0928 \u0915\u0930\ + \u0924\u0947 \u0939\u0948\u0902?" + - input_choice_list: + A: "\u0930\u093E\u091C\u094D\u092F \u0915\u0940 \u092C\u0922\u093C\u0924\u0940\ + \ \u0928\u094C\u0915\u0930\u0936\u093E\u0939\u0940 \u0928\u0947 \u0927\u0930\ + \u094D\u092E \u0915\u094B \u0939\u092E\u093E\u0930\u0947 \u091C\u0940\u0935\ + \u0928 \u0915\u093E \u0915\u0947\u0935\u0932 \u0939\u093E\u0936\u093F\u092F\ + \u0947 \u0915\u093E \u0939\u093F\u0938\u094D\u0938\u093E \u092C\u0928\u093E\ + \ \u0926\u093F\u092F\u093E \u0939\u0948" + B: "\u092A\u093E\u0930\u0902\u092A\u0930\u093F\u0915 \u092A\u094D\u0930\u093E\ + \u0927\u093F\u0915\u093E\u0930 \u0915\u0947 \u0915\u092E\u091C\u094B\u0930\ + \ \u0939\u094B\u0928\u0947 \u0915\u0947 \u092C\u093E\u0935\u091C\u0942\u0926\ + , \u0939\u092E\u093E\u0930\u093E \u0930\u094B\u091C\u092E\u0930\u094D\u0930\ + \u093E \u0915\u093E \u091C\u0940\u0935\u0928 \u0914\u0930 '\u0938\u093E\ + \u092E\u093E\u0928\u094D\u092F \u091C\u094D\u091E\u093E\u0928' \u0927\u093E\ + \u0930\u094D\u092E\u093F\u0915 \u0935\u093F\u0936\u094D\u0935\u093E\u0938\u094B\ + \u0902 \u0914\u0930 \u092E\u0942\u0932\u094D\u092F\u094B\u0902 \u0938\u0947\ + \ \u0906\u0915\u093E\u0930 \u0932\u0947\u0924\u093E \u0939\u0948" + C: "\u0938\u093E\u092E\u0942\u0939\u093F\u0915 \u092A\u0942\u091C\u093E \u092E\ + \u0947\u0902 \u0927\u093E\u0930\u094D\u092E\u093F\u0915 \u092D\u093E\u0917\ + \u0940\u0926\u093E\u0930\u0940 \u092E\u0947\u0902 \u092D\u0932\u0947 \u0939\ + \u0940 \u0917\u093F\u0930\u093E\u0935\u091F \u0906\u0908 \u0939\u094B, \u0932\ + \u0947\u0915\u093F\u0928 \u0932\u094B\u0917 \u0905\u092D\u0940 \u092D\u0940\ + \ \u0928\u093F\u091C\u0940 \u0924\u094C\u0930 \u092A\u0930 \u0905\u092A\u0928\ + \u0947 \u0927\u0930\u094D\u092E \u0915\u093E \u092A\u093E\u0932\u0928 \u0915\ + \u0930\u0924\u0947 \u0939\u0948\u0902" + D: "\u0932\u094B\u0917 \u0938\u093E\u0930\u094D\u0935\u091C\u0928\u093F\u0915\ + , \u0905\u0928\u094C\u092A\u091A\u093E\u0930\u093F\u0915 \u0938\u0947\u091F\ + \u093F\u0902\u0917 \u092E\u0947\u0902 \u0905\u092A\u0928\u0947 \u0927\u093E\ + \u0930\u094D\u092E\u093F\u0915 \u0935\u093F\u0936\u094D\u0935\u093E\u0938\u094B\ + \u0902 \u092A\u0930 \u091A\u0930\u094D\u091A\u093E \u0915\u0930\u0928\u0947\ + \ \u0915\u0940 \u0905\u0927\u093F\u0915 \u0938\u0902\u092D\u093E\u0935\u0928\ + \u093E \u0930\u0916\u0924\u0947 \u0939\u0948\u0902" + input_correct_responses: + - B + input_question: "'\u0928\u093E\u0917\u0930\u093F\u0915 \u0927\u0930\u094D\u092E\ + ' \u0938\u0947 '\u0938\u093E\u092E\u093E\u0928\u094D\u092F \u0927\u0930\ + \u094D\u092E' \u092E\u0947\u0902 \u092C\u0926\u0932\u093E\u0935 \u0915\u093E\ + \ \u0905\u0930\u094D\u0925 \u0939\u0948:" + - input_choice_list: + A: "\u0936\u094D\u0930\u092E\u093F\u0915 \u0935\u0930\u094D\u0917 \u0915\u0940\ + \ \u0905\u092A\u0928\u0947 \u0939\u093F\u0924\u094B\u0902 \u0915\u094B \u0928\ + \ \u0938\u092E\u091D\u0928\u0947 \u0915\u0940 \u092A\u094D\u0930\u0935\u0943\ + \u0924\u094D\u0924\u093F" + B: "\u090F\u0915 \u092A\u094D\u0930\u092E\u0941\u0916 \u0935\u093F\u091A\u093E\ + \u0930\u0927\u093E\u0930\u093E \u091C\u094B \u0906\u0930\u094D\u0925\u093F\ + \u0915, \u0930\u093E\u091C\u0928\u0940\u0924\u093F\u0915 \u0914\u0930 \u0938\ + \u093E\u0902\u0938\u094D\u0915\u0943\u0924\u093F\u0915 \u0936\u0915\u094D\u0924\ + \u093F \u0915\u094B \u0935\u0948\u0927 \u092C\u0928\u093E\u0924\u0940 \u0939\ + \u0948" + C: "\u0935\u093F\u091A\u093E\u0930\u0927\u093E\u0930\u093E \u0914\u0930 \u0930\ + \u094B\u091C\u092E\u0930\u094D\u0930\u093E \u0915\u0947 \u0905\u0928\u0941\ + \u092D\u0935\u094B\u0902 \u092A\u0930 \u0906\u0927\u093E\u0930\u093F\u0924\ + \ \u0926\u094B\u0939\u0930\u0940 \u091A\u0947\u0924\u0928\u093E \u0915\u093E\ + \ \u090F\u0915 \u0930\u0942\u092A" + D: "\u092C\u0915\u093E\u092F\u093E \u091F\u0949\u092A\u0930\u0940 \u0915\u0947\ + \ \u0932\u093F\u090F \u092D\u0941\u0917\u0924\u093E\u0928 \u0915\u093E \u090F\ + \u0915 \u0924\u0930\u0940\u0915\u093E \u0926\u093F\u092F\u093E \u0917\u092F\ + \u093E" + input_correct_responses: + - B + input_question: "'\u0906\u0927\u093F\u092A\u0924\u094D\u092F' \u0936\u092C\ + \u094D\u0926 \u0915\u093E \u0924\u093E\u0924\u094D\u092A\u0930\u094D\u092F \u0939\ + \u0948:" + - input_choice_list: + A: "\u0905\u0927\u093F\u0915\u093E\u0902\u0936 \u0939\u0921\u093C\u0924\u093E\ + \u0932\u094B\u0902 \u092A\u0930 \u0928\u093F\u092F\u094B\u0915\u094D\u0924\ + \u093E\u0913\u0902 \u0914\u0930 \u091C\u0928\u0938\u0902\u091A\u093E\u0930\ + \ \u092E\u093E\u0927\u094D\u092F\u092E\u094B\u0902 \u0915\u093E \u0927\u094D\ + \u092F\u093E\u0928 \u0928\u0939\u0940\u0902 \u091C\u093E\u0924\u093E" + B: "\u0938\u092D\u0940 \u0914\u0926\u094D\u092F\u094B\u0917\u093F\u0915 \u0935\ + \u093F\u0935\u093E\u0926\u094B\u0902 \u0915\u0940 \u0930\u093F\u092A\u094B\ + \u0930\u094D\u091F \u0928\u093F\u092F\u094B\u0915\u094D\u0924\u093E \u0926\ + \u094D\u0935\u093E\u0930\u093E \u0928\u0939\u0940\u0902 \u0915\u0940 \u091C\ + \u093E\u090F\u0917\u0940" + C: "\u0939\u0921\u093C\u0924\u093E\u0932 \u0915\u0940 \u092A\u0930\u093F\u092D\ + \u093E\u0937\u093E \u092E\u0947\u0902 \u0935\u0947 \u0939\u0921\u093C\u0924\ + \u093E\u0932\u0947\u0902 \u0936\u093E\u092E\u093F\u0932 \u0928\u0939\u0940\ + \u0902 \u0939\u0948\u0902 \u091C\u093F\u0928\u092E\u0947\u0902 \u0926\u0938\ + \ \u0938\u0947 \u0915\u092E \u0915\u0930\u094D\u092E\u091A\u093E\u0930\u0940\ + \ \u0936\u093E\u092E\u093F\u0932 \u0939\u094B\u0902 \u092F\u093E \u090F\u0915\ + \ \u0926\u093F\u0928 \u0938\u0947 \u0915\u092E \u0938\u092E\u092F \u0924\u0915\ + \ \u091A\u0932\u0947" + D: "\u0909\u0928 \u0939\u092E\u0932\u094B\u0902 \u0915\u0940 \u0924\u0941\u0932\ + \u0928\u093E \u0915\u0930\u0928\u093E \u0915\u0920\u093F\u0928 \u0939\u0948\ + \ \u091C\u093F\u0928\u094D\u0939\u0947\u0902 \u0905\u0932\u0917-\u0905\u0932\ + \u0917 \u0924\u0930\u0940\u0915\u094B\u0902 \u0938\u0947 \u092E\u093E\u092A\ + \u093E \u0917\u092F\u093E \u0925\u093E" + input_correct_responses: + - A + input_question: "\u0928\u093F\u092E\u094D\u0928\u0932\u093F\u0916\u093F\u0924\ + \ \u092E\u0947\u0902 \u0938\u0947 \u0915\u094C\u0928 \u0938\u0940 \u0939\u0921\ + \u093C\u0924\u093E\u0932 \u0915\u093E\u0930\u094D\u0930\u0935\u093E\u0908 \u092A\ + \u0930 \u0906\u0927\u093F\u0915\u093E\u0930\u093F\u0915 \u0906\u0902\u0915\u0921\ + \u093C\u094B\u0902 \u0938\u0947 \u091C\u0941\u0921\u093C\u0940 \u0938\u092E\u0938\ + \u094D\u092F\u093E \u0928\u0939\u0940\u0902 \u0939\u0948?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_sociology +tag: mmlu_hi_llama_social_sciences_tasks +task: mmlu_hi_llama_sociology +task_alias: sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbf70d7299ed21bbbf67187dfc965155cbd9d8c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_us_foreign_policy.yaml @@ -0,0 +1,138 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0907\u0938\u0928\u0947 \u0930\u093E\u091C\u0928\u0940\u0924\u093F\u0915\ + \ \u0905\u0930\u094D\u0925\u0935\u094D\u092F\u0935\u0938\u094D\u0925\u093E\ + \ \u0914\u0930 \u092A\u0942\u0902\u091C\u0940\u0935\u093E\u0926 \u0915\u0947\ + \ \u0905\u092E\u0947\u0930\u093F\u0915\u0940 \u092E\u0949\u0921\u0932 \u0915\ + \u0947 \u0938\u092E\u0930\u094D\u0925\u0928 \u0915\u094B \u0928\u0941\u0915\ + \u0938\u093E\u0928 \u092A\u0939\u0941\u0902\u091A\u093E\u092F\u093E" + B: "\u0907\u0938\u0928\u0947 \u0938\u0902\u0915\u091F \u0915\u094B \u092C\u0922\ + \u093C\u093E-\u091A\u0922\u093C\u093E\u0915\u0930 \u092A\u0947\u0936 \u0915\ + \u0930\u0928\u0947 \u0915\u0947 \u0932\u093F\u090F \u0938\u0902\u092F\u0941\ + \u0915\u094D\u0924 \u0930\u093E\u091C\u094D\u092F \u0905\u092E\u0947\u0930\ + \u093F\u0915\u093E \u092A\u0930 \u0917\u0941\u0938\u094D\u0938\u093E \u092A\ + \u0948\u0926\u093E \u0915\u093F\u092F\u093E" + C: "\u0907\u0938\u0938\u0947 \u0930\u093E\u0937\u094D\u091F\u094D\u0930\u092A\ + \u0924\u093F \u0913\u092C\u093E\u092E\u093E \u0915\u0947 \u0928\u0947\u0924\ + \u0943\u0924\u094D\u0935 \u092E\u0947\u0902 \u0905\u092E\u0947\u0930\u093F\ + \u0915\u0940 \u0935\u0948\u0936\u094D\u0935\u093F\u0915 \u0928\u0947\u0924\ + \u0943\u0924\u094D\u0935 \u0915\u0947 \u0932\u093F\u090F \u0938\u092E\u0930\ + \u094D\u0925\u0928 \u092C\u0922\u093C\u093E" + D: "\u0907\u0938\u0938\u0947 \u0905\u092E\u0947\u0930\u093F\u0915\u0940 \u0921\ + \u0949\u0932\u0930 \u0915\u093E \u0935\u0948\u0936\u094D\u0935\u093F\u0915\ + \ \u0909\u092A\u092F\u094B\u0917 \u0915\u092E \u0939\u094B \u0917\u092F\u093E" + input_correct_responses: + - A + input_question: "2008 \u0915\u0947 \u0935\u093F\u0924\u094D\u0924\u0940\u092F\ + \ \u0938\u0902\u0915\u091F \u0928\u0947 \u0905\u092E\u0947\u0930\u093F\u0915\ + \u093E \u0915\u0940 \u0905\u0902\u0924\u0930\u094D\u0930\u093E\u0937\u094D\u091F\ + \u094D\u0930\u0940\u092F \u092A\u094D\u0930\u0924\u093F\u0937\u094D\u0920\u093E\ + \ \u0915\u094B \u0915\u0948\u0938\u0947 \u092A\u094D\u0930\u092D\u093E\u0935\ + \u093F\u0924 \u0915\u093F\u092F\u093E?" + - input_choice_list: + A: "\u0907\u0938\u0928\u0947 \u0930\u094B\u0915\u0925\u093E\u092E \u0915\u093E\ + \ \u0935\u0948\u0936\u094D\u0935\u0940\u0915\u0930\u0923 \u0915\u093F\u092F\ + \u093E\u0964" + B: "\u0907\u0938\u0928\u0947 \u0928\u093F\u092F\u0902\u0924\u094D\u0930\u0923\ + \ \u0915\u093E \u0938\u0948\u0928\u094D\u092F\u0940\u0915\u0930\u0923 \u0915\ + \u093F\u092F\u093E\u0964" + C: "\u0907\u0938\u092E\u0947\u0902 \u0939\u093E\u0907\u0921\u094D\u0930\u094B\ + \u091C\u0928 \u092C\u092E \u0915\u0947 \u0935\u093F\u0915\u093E\u0938 \u0915\ + \u093E \u0906\u0939\u094D\u0935\u093E\u0928 \u0915\u093F\u092F\u093E \u0917\ + \u092F\u093E\u0964" + D: "\u090A\u092A\u0930 \u0915\u0947 \u0938\u092D\u0940" + input_correct_responses: + - D + input_question: "\u090F\u0928\u090F\u0938\u0938\u0940-68 \u0928\u0947 \u0905\u092E\ + \u0947\u0930\u093F\u0915\u0940 \u0930\u0923\u0928\u0940\u0924\u093F \u0915\u094B\ + \ \u0915\u0948\u0938\u0947 \u092C\u0926\u0932\u093E?" + - input_choice_list: + A: "\u0906\u0924\u0902\u0915\u0935\u093E\u0926 \u0928\u0940\u0924\u093F." + B: "\u0906\u0930\u094D\u0925\u093F\u0915 \u0928\u0940\u0924\u093F\u0964" + C: "\u0935\u093F\u0926\u0947\u0936 \u0928\u0940\u0924\u093F\u0964" + D: "\u0905\u0902\u0924\u0930\u094D\u0930\u093E\u0937\u094D\u091F\u094D\u0930\ + \u0940\u092F \u0928\u0940\u0924\u093F." + input_correct_responses: + - C + input_question: "\u092E\u0941\u0916\u094D\u092F \u0930\u0942\u092A \u0938\u0947\ + \ \u0938\u0902\u092F\u0941\u0915\u094D\u0924 \u0930\u093E\u091C\u094D\u092F\ + \ \u0905\u092E\u0947\u0930\u093F\u0915\u093E \u0914\u0930 \u0936\u0947\u0937\ + \ \u0935\u093F\u0936\u094D\u0935 \u0915\u0947 \u092C\u0940\u091A \u0938\u0902\ + \u092C\u0902\u0927\u094B\u0902 \u0938\u0947 \u0938\u0902\u092C\u0902\u0927\u093F\ + \u0924 \u0928\u0940\u0924\u093F\u0917\u0924 \u0928\u093F\u0930\u094D\u0923\u092F\ + \u094B\u0902 \u0915\u0947 \u0915\u094D\u0937\u0947\u0924\u094D\u0930 \u0915\u094B\ + \ \u0915\u0939\u093E \u091C\u093E\u0924\u093E \u0939\u0948" + - input_choice_list: + A: "\u0930\u0915\u094D\u0937\u093E\u0924\u094D\u092E\u0915 \u092F\u0925\u093E\ + \u0930\u094D\u0925\u0935\u093E\u0926\u0940 \u0905\u0902\u0924\u0930\u094D\u0930\ + \u093E\u0937\u094D\u091F\u094D\u0930\u0940\u092F \u0938\u0902\u0938\u094D\u0925\ + \u093E\u0928\u094B\u0902 \u0915\u0940 \u092D\u0942\u092E\u093F\u0915\u093E\ + \ \u092A\u0930 \u0905\u0927\u093F\u0915 \u091C\u094B\u0930 \u0926\u0947\u0924\ + \u0947 \u0939\u0948\u0902" + B: "\u0930\u0915\u094D\u0937\u093E\u0924\u094D\u092E\u0915 \u092F\u0925\u093E\ + \u0930\u094D\u0925\u0935\u093E\u0926\u0940 \u092D\u094C\u0917\u094B\u0932\u093F\ + \u0915 \u0915\u093E\u0930\u0915\u094B\u0902 \u092A\u0930 \u0915\u092E \u091C\ + \u094B\u0930 \u0926\u0947\u0924\u0947 \u0939\u0948\u0902" + C: "\u0906\u0915\u094D\u0930\u093E\u092E\u0915 \u092F\u0925\u093E\u0930\u094D\ + \u0925\u0935\u093E\u0926\u0940 \u0930\u0915\u094D\u0937\u093E\u0924\u094D\u092E\ + \u0915 \u092F\u0925\u093E\u0930\u094D\u0925\u0935\u093E\u0926\u093F\u092F\u094B\ + \u0902 \u0915\u0940 \u0924\u0941\u0932\u0928\u093E \u092E\u0947\u0902 \u0930\ + \u093E\u0937\u094D\u091F\u094D\u0930\u0940\u092F \u0939\u093F\u0924 \u0915\ + \u094B \u0905\u0927\u093F\u0915 \u092A\u094D\u0930\u093E\u0925\u092E\u093F\ + \u0915\u0924\u093E \u0926\u0947\u0924\u0947 \u0939\u0948\u0902\u0964" + D: "\u0930\u0915\u094D\u0937\u093E\u0924\u094D\u092E\u0915 \u092F\u0925\u093E\ + \u0930\u094D\u0925\u0935\u093E\u0926\u0940 \u092E\u093E\u0928\u0924\u0947\ + \ \u0939\u0948\u0902 \u0915\u093F \u0930\u093E\u091C\u094D\u092F \u0938\u0941\ + \u0930\u0915\u094D\u0937\u093E \u0915\u094B \u0905\u0927\u093F\u0915\u0924\ + \u092E \u0915\u0930\u0928\u0947 \u0935\u093E\u0932\u0947 \u0939\u0948\u0902\ + , \u091C\u092C\u0915\u093F \u0906\u0915\u094D\u0930\u093E\u092E\u0915 \u092F\ + \u0925\u093E\u0930\u094D\u0925\u0935\u093E\u0926\u0940 \u092E\u093E\u0928\u0924\ + \u0947 \u0939\u0948\u0902 \u0915\u093F \u0930\u093E\u091C\u094D\u092F \u0936\ + \u0915\u094D\u0924\u093F \u0915\u094B \u0905\u0927\u093F\u0915\u0924\u092E\ + \ \u0915\u0930\u0928\u0947 \u0935\u093E\u0932\u0947 \u0939\u0948\u0902" + input_correct_responses: + - D + input_question: "\u0930\u093E\u091C\u094D\u092F \u0915\u0947 \u0935\u094D\u092F\ + \u0935\u0939\u093E\u0930 \u0915\u0940 \u0935\u094D\u092F\u093E\u0916\u094D\u092F\ + \u093E \u092E\u0947\u0902 \u0930\u0915\u094D\u0937\u093E\u0924\u094D\u092E\u0915\ + \ \u092F\u0925\u093E\u0930\u094D\u0925\u0935\u093E\u0926 \u0914\u0930 \u0906\ + \u0915\u094D\u0930\u093E\u092E\u0915 \u092F\u0925\u093E\u0930\u094D\u0925\u0935\ + \u093E\u0926 \u0915\u093F\u0938 \u092A\u094D\u0930\u0915\u093E\u0930 \u092D\u093F\ + \u0928\u094D\u0928 \u0939\u0948\u0902?" + - input_choice_list: + A: "\u0935\u0948\u0936\u094D\u0935\u0940\u0915\u0930\u0923 \u0928\u0947 \u0909\ + \u0928\u0915\u0947 \u091C\u0948\u0938\u0947 \u0932\u094B\u0917\u094B\u0902\ + \ \u0915\u094B \u092C\u0939\u0941\u0924 \u0905\u092E\u0940\u0930 \u092C\u0928\ + \u093E \u0926\u093F\u092F\u093E \u0925\u093E" + B: "\u0935\u0948\u0936\u094D\u0935\u0940\u0915\u0930\u0923 \u0938\u0947 \u0915\ + \u0947\u0935\u0932 \u0915\u0941\u091B \u0905\u092E\u0947\u0930\u093F\u0915\ + \u0940 \u0930\u093E\u091C\u094D\u092F\u094B\u0902, \u091C\u0948\u0938\u0947\ + \ \u0928\u094D\u092F\u0942\u092F\u0949\u0930\u094D\u0915, \u0915\u094B \u0932\ + \u093E\u092D \u0939\u0941\u0906" + C: "\u0909\u0926\u093E\u0930\u0935\u093E\u0926\u0940 \u0905\u092D\u093F\u091C\ + \u093E\u0924 \u0935\u0930\u094D\u0917 \u0928\u0947 \u0935\u0948\u0936\u094D\ + \u0935\u0940\u0915\u0930\u0923 \u0915\u094B \u092A\u094D\u0930\u094B\u0924\ + \u094D\u0938\u093E\u0939\u093F\u0924 \u0915\u093F\u092F\u093E \u0925\u093E\ + , \u091C\u092C\u0915\u093F '\u0938\u093E\u092E\u093E\u0928\u094D\u092F\ + \ \u0905\u092E\u0947\u0930\u093F\u0915\u093F\u092F\u094B\u0902' \u0928\ + \u0947 \u0907\u0938\u0915\u0947 \u0915\u093E\u0930\u0923 \u0905\u092A\u0928\ + \u0940 \u0928\u094C\u0915\u0930\u093F\u092F\u093E\u0901 \u0916\u094B \u0926\ + \u0940\u0902" + D: "\u0935\u0948\u0936\u094D\u0935\u0940\u0915\u0930\u0923 \u0928\u0947 \u0939\ + \u093E\u0928\u093F\u0915\u093E\u0930\u0915 \u0935\u094D\u092F\u093E\u092A\u093E\ + \u0930 \u092F\u0941\u0926\u094D\u0927\u094B\u0902 \u0915\u094B \u092A\u094D\ + \u0930\u094B\u0924\u094D\u0938\u093E\u0939\u093F\u0924 \u0915\u093F\u092F\u093E" + input_correct_responses: + - C + input_question: "2016 \u0915\u0947 \u0905\u092D\u093F\u092F\u093E\u0928 \u092E\ + \u0947\u0902 \u0921\u094B\u0928\u093E\u0932\u094D\u0921 \u091F\u094D\u0930\u092E\ + \u094D\u092A \u0928\u0947 \u0935\u0948\u0936\u094D\u0935\u0940\u0915\u0930\u0923\ + \ \u092A\u0930 \u0915\u0948\u0938\u0947 \u0939\u092E\u0932\u093E \u0915\u093F\ + \u092F\u093E?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_us_foreign_policy +tag: mmlu_hi_llama_social_sciences_tasks +task: mmlu_hi_llama_us_foreign_policy +task_alias: us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d95223adf291f70592a87529ff25ec2db954c6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/mmlu_hi_world_religions.yaml @@ -0,0 +1,68 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0907\u0902\u0926\u094D\u0930\u0927\u0928\u0941\u0937" + B: "\u092A\u0930\u093F\u0936\u0941\u0926\u094D\u0927 \u0915\u0930\u0923" + C: "\u090F\u0915 \u092C\u0947\u091F\u093E" + D: "\u092C\u093E\u0930 \u092E\u093F\u0924\u094D\u091C\u093C\u0935\u093E\u0939" + input_correct_responses: + - B + input_question: "\u092F\u0939\u0942\u0926\u0940 \u092A\u0941\u0930\u0941\u0937\ + \u094B\u0902 \u0915\u0947 \u0932\u093F\u090F \u0935\u093E\u091A\u093E \u0915\ + \u093E \u091A\u093F\u0928\u094D\u0939 \u0915\u094D\u092F\u093E \u0939\u0948?" + - input_choice_list: + A: "\u0927\u0930\u094D\u092E" + B: "\u0938\u0902\u0918" + C: "\u092C\u0941\u0926\u094D\u0927" + D: "\u092C\u094B\u0927\u093F\u0938\u0924\u094D\u0935" + input_correct_responses: + - A + input_question: "\u092C\u094C\u0926\u094D\u0927 \u0927\u0930\u094D\u092E \u092E\ + \u0947\u0902 \u0926\u0942\u0938\u0930\u093E \u0930\u0924\u094D\u0928 \u0915\u094C\ + \u0928 \u0938\u093E \u0939\u0948?" + - input_choice_list: + A: "\u0936\u093E\u0902\u0917" + B: "\u091D\u094B\u0909" + C: "\u0939\u093E\u0928" + D: "\u091C\u093C\u093F\u092F\u093E" + input_correct_responses: + - B + input_question: "\u0915\u093F\u0938 \u0930\u093E\u091C\u0935\u0902\u0936 \u092E\ + \u0947\u0902 \u0928\u090F \u0936\u093E\u0938\u0915\u094B\u0902 \u0915\u094B\ + \ \u0935\u0948\u0927 \u092C\u0928\u093E\u0928\u0947 \u0915\u0947 \u0932\u093F\ + \u090F "\u0938\u094D\u0935\u0930\u094D\u0917 \u0915\u093E \u0906\u0926\u0947\ + \u0936" \u0935\u093F\u0915\u0938\u093F\u0924 \u0915\u093F\u092F\u093E \u0917\ + \u092F\u093E \u0925\u093E?" + - input_choice_list: + A: "\u0939\u094B\u0928\u0947\u0928" + B: "\u0924\u0928\u093E\u0915\u093E" + C: "\u0924\u094B\u0915\u0941\u0917\u093E\u0935\u093E" + D: "\u092E\u0940\u091C\u0940" + input_correct_responses: + - D + input_question: "\u0915\u093F\u0938 \u091C\u093E\u092A\u093E\u0928\u0940 \u0938\ + \u0930\u0915\u093E\u0930 \u0928\u0947 \u0938\u092E\u094D\u0930\u093E\u091F \u0914\ + \u0930 \u0915\u093E\u092E\u0940 \u0915\u0947 \u0938\u093E\u0925 \u0909\u0938\ + \u0915\u0947 \u0938\u0902\u092C\u0902\u0927\u094B\u0902 \u092A\u0930 \u0906\u0927\ + \u093E\u0930\u093F\u0924 \u090F\u0915 \u092A\u094D\u0930\u0915\u093E\u0930 \u0915\ + \u0947 \u0930\u093E\u0937\u094D\u091F\u094D\u0930\u0940\u092F \u092A\u0902\u0925\ + \ \u0915\u094B \u092C\u0922\u093C\u093E\u0935\u093E \u0926\u093F\u092F\u093E\ + ?" + - input_choice_list: + A: "\u0905\u0928\u0941\u0937\u094D\u0920\u093E\u0928 \u0917\u094D\u0930\u0902\ + \u0925" + B: "\u0926\u093E\u0930\u094D\u0936\u0928\u093F\u0915 \u0917\u094D\u0930\u0902\ + \u0925" + C: "\u092D\u091C\u0928" + D: "\u092E\u0942\u0932 \u0915\u0939\u093E\u0928\u093F\u092F\u093E\u0901" + input_correct_responses: + - B + input_question: "\u0909\u092A\u0928\u093F\u0937\u0926\u094B\u0902 \u0915\u093E\ + \ \u0935\u0930\u094D\u0923\u0928 \u0915\u0948\u0938\u0947 \u0915\u093F\u092F\ + \u093E \u091C\u093E \u0938\u0915\u0924\u093E \u0939\u0948?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_world_religions +tag: mmlu_hi_llama_humanities_tasks +task: mmlu_hi_llama_world_religions +task_alias: world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/utils.py b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ce0608d4ba262d2c7ce16eb37847928601e848df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_hi/utils.py @@ -0,0 +1,104 @@ +from functools import partial + +import datasets + + +def process_docs(dataset: datasets.Dataset, subtask) -> datasets.Dataset: + return dataset.filter( + lambda example: example["subtask_name"] == f"mmlu_hi_chat.{subtask}" + ) + + +process_docs_econometrics = partial(process_docs, subtask="econometrics") +process_docs_public_relations = partial(process_docs, subtask="public_relations") +process_docs_astronomy = partial(process_docs, subtask="astronomy") +process_docs_conceptual_physics = partial(process_docs, subtask="conceptual_physics") +process_docs_college_medicine = partial(process_docs, subtask="college_medicine") +process_docs_global_facts = partial(process_docs, subtask="global_facts") +process_docs_marketing = partial(process_docs, subtask="marketing") +process_docs_high_school_physics = partial(process_docs, subtask="high_school_physics") +process_docs_college_computer_science = partial( + process_docs, subtask="college_computer_science" +) +process_docs_business_ethics = partial(process_docs, subtask="business_ethics") +process_docs_miscellaneous = partial(process_docs, subtask="miscellaneous") +process_docs_moral_disputes = partial(process_docs, subtask="moral_disputes") +process_docs_sociology = partial(process_docs, subtask="sociology") +process_docs_high_school_geography = partial( + process_docs, subtask="high_school_geography" +) +process_docs_high_school_microeconomics = partial( + process_docs, subtask="high_school_microeconomics" +) +process_docs_world_religions = partial(process_docs, subtask="world_religions") +process_docs_nutrition = partial(process_docs, subtask="nutrition") +process_docs_high_school_computer_science = partial( + process_docs, subtask="high_school_computer_science" +) +process_docs_college_biology = partial(process_docs, subtask="college_biology") +process_docs_logical_fallacies = partial(process_docs, subtask="logical_fallacies") +process_docs_elementary_mathematics = partial( + process_docs, subtask="elementary_mathematics" +) +process_docs_virology = partial(process_docs, subtask="virology") +process_docs_prehistory = partial(process_docs, subtask="prehistory") +process_docs_college_physics = partial(process_docs, subtask="college_physics") +process_docs_computer_security = partial(process_docs, subtask="computer_security") +process_docs_machine_learning = partial(process_docs, subtask="machine_learning") +process_docs_electrical_engineering = partial( + process_docs, subtask="electrical_engineering" +) +process_docs_professional_psychology = partial( + process_docs, subtask="professional_psychology" +) +process_docs_high_school_biology = partial(process_docs, subtask="high_school_biology") +process_docs_high_school_statistics = partial( + process_docs, subtask="high_school_statistics" +) +process_docs_moral_scenarios = partial(process_docs, subtask="moral_scenarios") +process_docs_high_school_world_history = partial( + process_docs, subtask="high_school_world_history" +) +process_docs_college_mathematics = partial(process_docs, subtask="college_mathematics") +process_docs_high_school_government_and_politics = partial( + process_docs, subtask="high_school_government_and_politics" +) +process_docs_professional_accounting = partial( + process_docs, subtask="professional_accounting" +) +process_docs_jurisprudence = partial(process_docs, subtask="jurisprudence") +process_docs_high_school_european_history = partial( + process_docs, subtask="high_school_european_history" +) +process_docs_professional_medicine = partial( + process_docs, subtask="professional_medicine" +) +process_docs_high_school_mathematics = partial( + process_docs, subtask="high_school_mathematics" +) +process_docs_anatomy = partial(process_docs, subtask="anatomy") +process_docs_abstract_algebra = partial(process_docs, subtask="abstract_algebra") +process_docs_philosophy = partial(process_docs, subtask="philosophy") +process_docs_medical_genetics = partial(process_docs, subtask="medical_genetics") +process_docs_us_foreign_policy = partial(process_docs, subtask="us_foreign_policy") +process_docs_security_studies = partial(process_docs, subtask="security_studies") +process_docs_high_school_macroeconomics = partial( + process_docs, subtask="high_school_macroeconomics" +) +process_docs_human_sexuality = partial(process_docs, subtask="human_sexuality") +process_docs_high_school_us_history = partial( + process_docs, subtask="high_school_us_history" +) +process_docs_human_aging = partial(process_docs, subtask="human_aging") +process_docs_formal_logic = partial(process_docs, subtask="formal_logic") +process_docs_professional_law = partial(process_docs, subtask="professional_law") +process_docs_international_law = partial(process_docs, subtask="international_law") +process_docs_high_school_psychology = partial( + process_docs, subtask="high_school_psychology" +) +process_docs_management = partial(process_docs, subtask="management") +process_docs_high_school_chemistry = partial( + process_docs, subtask="high_school_chemistry" +) +process_docs_college_chemistry = partial(process_docs, subtask="college_chemistry") +process_docs_clinical_knowledge = partial(process_docs, subtask="clinical_knowledge") diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/_mmlu_it_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/_mmlu_it_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca3376a78dd63b7596241bfdd458fa0b0327bb78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/_mmlu_it_humanities.yaml @@ -0,0 +1,11 @@ +group: mmlu_it_llama_humanities +group_alias: humanities +task: + - mmlu_it_llama_humanities_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/_mmlu_it_other.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/_mmlu_it_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eedcba37bb689722fc2b5d5498a5ac220e07c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/_mmlu_it_other.yaml @@ -0,0 +1,11 @@ +group: mmlu_it_llama_other +group_alias: other +task: + - mmlu_it_llama_other_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dd12b848cd0751a858162df999fc9a6a26f06f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_astronomy.yaml @@ -0,0 +1,65 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Sarebbe pi\xF9 difficile dato che il camion \xE8 pi\xF9 pesante su Marte." + B: "Sarebbe pi\xF9 facile dato che il camion \xE8 pi\xF9 leggero su Marte." + C: "Sarebbe pi\xF9 difficile poich\xE9 il camion \xE8 pi\xF9 leggero su Marte." + D: Sarebbe lo stesso, non importa dove ti trovi. + input_correct_responses: + - D + input_question: "Stai spingendo un camion lungo una strada. Sarebbe pi\xF9 facile\ + \ accelerare questo camion su Marte? Perch\xE9? (Supponiamo che non vi sia attrito)" + - input_choice_list: + A: La cintura di Kuiper; le comete di breve periodo tendono a trovarsi nel piano + del sistema solare proprio come la fascia di Kuiper. + B: La cintura di Kuiper; le comete di breve periodo tendono a provenire da direzioni + casuali che indicano una distribuzione sferica delle comete chiamata fascia + di Kuiper. + C: La cintura degli asteroidi; le comete di breve periodo hanno periodi orbitali + simili a quelli degli asteroidi come Vesta e si trovano nel piano del sistema + solare proprio come la cintura degli asteroidi. + D: La nube di Oort; le comete di breve periodo tendono a trovarsi nel piano + del sistema solare proprio come la nube di Oort. + input_correct_responses: + - A + input_question: Da dove provengono la maggior parte delle comete di breve periodo + e come lo sappiamo? + - input_choice_list: + A: "10000 volte di pi\xF9" + B: "100 volte di pi\xF9" + C: "1000 volte di pi\xF9" + D: "10 volte di pi\xF9" + input_correct_responses: + - A + input_question: "Supponiamo che la pupilla del tuo occhio abbia un diametro di\ + \ 5 mm e che tu abbia un telescopio con un'apertura di 50 cm. Quanta pi\xF9\ + \ luce pu\xF2 raccogliere il telescopio rispetto al tuo occhio?" + - input_choice_list: + A: "Qui una volta si form\xF2 un pianeta, ma fu distrutto da una collisione\ + \ catastrofica." + B: In questa parte della nebulosa solare non c'era abbastanza materiale + per formare un pianeta. + C: C'era troppo materiale roccioso per formare un pianeta terrestre ma non + abbastanza materiale gassoso per formare un pianeta gioviano. + D: La risonanza con Giove ha impedito al materiale di riunirsi per formare un + pianeta. + input_correct_responses: + - D + input_question: "Perch\xE9 non esiste un pianeta dove si trova la fascia degli\ + \ asteroidi?" + - input_choice_list: + A: "Perch\xE9 la superficie \xE8 ricoperta da minerali fortemente ossidati ("arrugginiti")." + B: "Perch\xE9 l\u2019atmosfera disperde pi\xF9 luce alle lunghezze d\u2019onda\ + \ pi\xF9 blu trasmettendo principalmente luce rossa." + C: "Perch\xE9 Marte \xE8 ricoperto da antiche colate laviche di colore rosso." + D: "Perch\xE9 l'acqua che scorre sulla superficie di Marte ha alterato i\ + \ minerali superficiali diversi miliardi di anni fa." + input_correct_responses: + - A + input_question: "Perch\xE9 Marte \xE8 rosso?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_astronomy +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_astronomy +task_alias: astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec296d8136482f98ce885b41f7b639e77fd422d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_clinical_knowledge.yaml @@ -0,0 +1,54 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ATP. + B: ADP. + C: fosfocreatina. + D: fosforilazione ossidativa. + input_correct_responses: + - A + input_question: "L\u2019energia per tutte le forme di contrazione muscolare \xE8\ + \ fornita da:" + - input_choice_list: + A: I cateteri maschili e femminili sono di colori diversi. + B: "I cateteri maschili sono pi\xF9 lunghi dei cateteri femminili." + C: "I cateteri maschili sono pi\xF9 grandi dei cateteri femminili." + D: "I cateteri femminili sono pi\xF9 lunghi dei cateteri maschili." + input_correct_responses: + - B + input_question: "Qual \xE8 la differenza tra un catetere maschile e uno femminile?" + - input_choice_list: + A: "L'abduzione del pollice \xE8 fornita dalla radice spinale T2" + B: "L'opposizione del pollice da parte dell'opponens policis \xE8 fornita\ + \ dalla radice spinale T1" + C: "L'adduzione delle dita \xE8 fornita dal nervo mediano" + D: "Il rapimento delle dita \xE8 mediato dagli interossei palmari" + input_correct_responses: + - B + input_question: "Nella valutazione della funzione della mano quale delle seguenti\ + \ affermazioni \xE8 vera?" + - input_choice_list: + A: '4' + B: '3' + C: '2' + D: '1' + input_correct_responses: + - C + input_question: "Quanti tentativi si dovrebbero fare per incannulare un paziente\ + \ prima di passare il lavoro a un collega pi\xF9 anziano, secondo le conoscenze\ + \ mediche del 2020?" + - input_choice_list: + A: glicogeno a glucosio-1-fosfato. + B: glicogeno o glucosio in fruttosio. + C: glicogeno o glucosio a piruvato o lattato. + D: glicogeno o glucosio a piruvato o acetil CoA. + input_correct_responses: + - C + input_question: "La glicolisi \xE8 il nome dato al percorso che comporta la conversione\ + \ di:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_clinical_knowledge +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_clinical_knowledge +task_alias: clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d43f275dd50f9d6b947feff26f4f3412dd67c8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_biology.yaml @@ -0,0 +1,58 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Possiedono un esoscheletro composto principalmente da peptidoglicano. + B: Possiedono un sistema circolatorio aperto con un cuore dorsale. + C: Sono membri di un phylum biologicamente fallito, incapace di sfruttare diversi + habitat e fonti di nutrimento. + D: Sono privi di appendici accoppiate e articolate. + input_correct_responses: + - B + input_question: Quale delle seguenti rappresenta un'affermazione accurata + riguardo agli artropodi? + - input_choice_list: + A: 1/400 + B: 19/400 + C: 20/400 + D: 38/400 + input_correct_responses: + - D + input_question: "In una data popolazione, 1 persona su 400 ha un cancro causato\ + \ da un allele completamente recessivo, b. Supponendo che la popolazione sia\ + \ in equilibrio di Hardy-Weinberg, quale delle seguenti \xE8 la percentuale\ + \ attesa di individui portatori dell'allele b ma che non si prevede sviluppino\ + \ il cancro?" + - input_choice_list: + A: l'uomo e l'uccello sono specie polifiletiche + B: "l'evoluzione di un essere umano e di un uccello \xE8 convergente" + C: l'umano e l'uccello appartengono a un clade + D: l'uomo e l'uccello si sono sviluppati per analogia + input_correct_responses: + - C + input_question: La presenza di strutture omologhe in due organismi diversi, come + l'omero nell'arto anteriore di un essere umano e di un uccello, lo indica + - input_choice_list: + A: una pompa pressione-flusso dipendente dall'ATP + B: "un gradiente potenziale della pressione dell\u2019acqua" + C: traspirazione + D: diffusione apoplastica + input_correct_responses: + - B + input_question: "Secondo il modello pressione-flusso del movimento del contenuto\ + \ del floema, il movimento del fotosintetato dalla sorgente al pozzo \xE8 guidato\ + \ da" + - input_choice_list: + A: Telomeri + B: Centromeri + C: Nucleosomi + D: Spliceosomi + input_correct_responses: + - B + input_question: Quale dei seguenti contiene sequenze di DNA necessarie per la + segregazione dei cromosomi nella mitosi e nella meiosi? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_biology +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_college_biology +task_alias: college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f631901b732bfa2aa17aefdf9de9a38276f49ac7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_chemistry.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Lo stato di ossidazione pi\xF9 comune per gli elementi lantanidi \xE8 +3." + B: I complessi di lantanidi hanno spesso numeri di coordinazione elevati (> + 6). + C: Tutti gli elementi lantanidi reagiscono con l'acido acquoso per liberare + idrogeno. + D: I raggi atomici degli elementi lantanidi aumentano nel periodo da La a Lu. + input_correct_responses: + - D + input_question: "Quale delle seguenti affermazioni sugli elementi lantanidi NON\ + \ \xE8 vera?" + - input_choice_list: + A: 1,0 ml + B: 10 ml + C: 20 ml + D: 50 ml + input_correct_responses: + - C + input_question: Un campione di 0,217 g di HgO (massa molare = 217 g) reagisce + con gli ioni ioduro in eccesso secondo la reazione mostrata sopra. La titolazione + della soluzione risultante richiede quanti ml di HCl 0,10 M per raggiungere + il punto equivalente? + - input_choice_list: + A: '4' + B: '3' + C: '6' + D: '24' + input_correct_responses: + - A + input_question: "Prevedere il numero di linee nello spettro EPR di una soluzione\ + \ di radicale metilico marcato con 13C (13CH3\u2022), presupponendo che le linee\ + \ non si sovrappongano." + - input_choice_list: + A: un acido + B: una base + C: un catalizzatore + D: un agente riducente + input_correct_responses: + - D + input_question: "3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq)\ + \ + 4 Cr3+(aq) + 10 H2O(l). Nella reazione mostrata sopra, Cl\u2212(aq) si comporta\ + \ come" + - input_choice_list: + A: PbH4 < SnH4 < GeH4 < SiH4 < CH4 + B: PbH4 < SnH4 < CH4 < GeH4 < SiH4 + C: CH4 < SiH4 < GeH4 < SnH4 < PbH4 + D: CH4 < PbH4 < GeH4 < SnH4 < SiH4 + input_correct_responses: + - A + input_question: "Quale delle seguenti elenca gli idruri degli elementi del gruppo\ + \ 14 in ordine di stabilit\xE0 termica, dal pi\xF9 basso al pi\xF9 alto?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_chemistry +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_college_chemistry +task_alias: college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47f9c7e5150261c2b97a8dfa1bdcb0cf5f3ba486 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_computer_science.yaml @@ -0,0 +1,81 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: a*(c + d)+ b(c + d) + B: a*(c+d)* + b(c+d)* + C: a*(c + d)+ b*(c + d) + D: (a + b)*c +(a + b)*d + input_correct_responses: + - D + input_question: "Quale delle seguenti espressioni regolari \xE8 equivalente a\ + \ (descrive lo stesso insieme di stringhe di) (a* + b)*(c + d)?" + - input_choice_list: + A: '5' + B: '6' + C: '7' + D: '8' + input_correct_responses: + - B + input_question: "Una determinata macchina RISC in pipeline ha 8 registri di uso\ + \ generale R0, R1, . . . , R7 e supporta le seguenti operazioni. AGGIUNGI Rs1,\ + \ Rs2, Rd Somma Rs1 a Rs2 e metti la somma in Rd MUL Rs1, Rs2, Rd Moltiplica\ + \ Rs1 per Rs2 e metti il prodotto in Rd Un'operazione normalmente richiede\ + \ un ciclo; tuttavia, un'operazione richiede due cicli se produce un risultato\ + \ richiesto dall'operazione immediatamente successiva in una sequenza di\ + \ operazioni. Considera l'espressione AB + ABC + BC, dove le variabili A,\ + \ B, C si trovano nei registri R0, R1, R2. Se il contenuto di questi tre registri\ + \ non deve essere modificato, qual \xE8 il numero minimo di cicli di clock richiesti\ + \ per una sequenza di operazioni che calcola il valore di AB + ABC + BC?" + - input_choice_list: + A: I solo + B: Solo II + C: Solo III + D: I, II e III + input_correct_responses: + - D + input_question: "Il modello di progettazione Singleton viene utilizzato per garantire\ + \ che sia possibile istanziare solo una singola istanza di una classe. Quale\ + \ delle seguenti affermazioni \xE8 (sono) vera per questo modello di progettazione?\ + \ I. La classe Singleton ha un metodo factory statico per fornire la sua istanza.\ + \ II. La classe Singleton pu\xF2 essere una sottoclasse di un'altra classe.\ + \ III. La classe Singleton ha un costruttore privato." + - input_choice_list: + A: '5' + B: '6' + C: '7' + D: '9' + input_correct_responses: + - D + input_question: "Un compilatore genera codice per la seguente istruzione di assegnazione.\ + \ G := (A + B) * C - (D + E) * F La macchina target ha un singolo accumulatore\ + \ e un set di istruzioni a indirizzo singolo costituito da istruzioni caricare,\ + \ memorizzare, aggiungere, sottrarre e moltiplicare. Per le operazioni aritmetiche,\ + \ l'operando sinistro viene prelevato dall'accumulatore e il risultato\ + \ appare nell'accumulatore. Il minor numero possibile di istruzioni nel\ + \ codice risultante \xE8" + - input_choice_list: + A: 1/50 + B: 1/27 + C: 1/25 + D: 27/2 + input_correct_responses: + - B + input_question: "Consideriamo un progetto di computer in cui pi\xF9 processori,\ + \ ciascuno con una memoria cache privata, condividono la memoria globale utilizzando\ + \ un singolo bus. Questo bus \xE8 la risorsa critica del sistema. Ogni processore\ + \ pu\xF2 eseguire un'istruzione ogni 500 nanosecondi purch\xE9 i riferimenti\ + \ di memoria siano soddisfatti dalla cache locale. Quando si verifica un errore\ + \ nella cache, il processore viene ritardato di altri 2.000 nanosecondi. Durante\ + \ la met\xE0 di questo ritardo aggiuntivo, l'autobus \xE8 dedicato a servire\ + \ la cache miss. Durante l'altra met\xE0, il processore non pu\xF2 continuare,\ + \ ma il bus \xE8 libero di soddisfare le richieste di altri processori. In media,\ + \ ogni istruzione richiede 2 riferimenti di memoria. In media, gli errori di\ + \ cache si verificano nell'1% dei riferimenti. Quale percentuale della capacit\xE0\ + \ del bus consumerebbe un singolo processore, ignorando i ritardi dovuti alla\ + \ concorrenza di altri processori?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_computer_science +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_college_computer_science +task_alias: college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bb740ddd2733d5f040979ebdd24e0663634b8c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_mathematics.yaml @@ -0,0 +1,64 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ST = 0 + B: ST = T + C: ST = ST + D: "ST - TS \xE8 la mappa identitaria di V su se stessa." + input_correct_responses: + - D + input_question: "Sia V l'insieme di tutti i polinomi reali p(x). Siano definite\ + \ le trasformazioni T, S su V da T:p(x) -> xp(x) e S:p(x) -> p'(x)\ + \ = d/dx p(x), e si interpreti (ST) (p(x)) come S(T(p(x))). Quale delle seguenti\ + \ affermazioni \xE8 vera?" + - input_choice_list: + A: '2' + B: 2 - e^-2 + C: 2+e^-2 + D: 2+e^-4 + input_correct_responses: + - D + input_question: "Un serbatoio contiene inizialmente una soluzione salina composta\ + \ da 3 grammi di sale disciolti in 100 litri di acqua. Nel serbatoio viene spruzzata\ + \ una soluzione salina contenente 0,02 grammi di sale per litro d'acqua\ + \ ad una velocit\xE0 di 4 litri al minuto. La soluzione spruzzata viene continuamente\ + \ miscelata con la soluzione salina contenuta nel serbatoio e la miscela fuoriesce\ + \ dal serbatoio ad una velocit\xE0 di 4 litri al minuto. Se la miscelazione\ + \ \xE8 istantanea, quanti grammi di sale ci sono nel serbatoio dopo che sono\ + \ trascorsi 100 minuti?" + - input_choice_list: + A: I solo + B: Solo II + C: Solo III + D: Solo II e III + input_correct_responses: + - B + input_question: "Sia A una matrice 2x2 reale. Quale delle seguenti affermazioni\ + \ deve essere vera? I. Tutte le voci di A^2 sono non negative. II. Il determinante\ + \ di A^2 non \xE8 negativo. III. Se A ha due autovalori distinti, allora A^2\ + \ ha due autovalori distinti." + - input_choice_list: + A: '-11' + B: '0' + C: '11' + D: 33/2 + input_correct_responses: + - C + input_question: "Supponiamo che f(1 + x) = f(x) per ogni x reale. Se f \xE8 un\ + \ polinomio e f(5) = 11, allora f(15/2)" + - input_choice_list: + A: '-5' + B: '-4' + C: '-3' + D: '-2' + input_correct_responses: + - B + input_question: "Sia A l'insieme di tutte le coppie ordinate di interi (m,\ + \ n) tali che 7m + 12n = 22. Qual \xE8 il numero negativo pi\xF9 grande nell'insieme\ + \ B = {m + n : (m, n) \\in A}?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_mathematics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_college_mathematics +task_alias: college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..121855ad4e536ef59ed8e2c95e2adc4cd6ca7582 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_college_physics.yaml @@ -0,0 +1,62 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '4' + B: '5' + C: '6' + D: '20' + input_correct_responses: + - A + input_question: "Un telescopio rifrattore \xE8 formato da due lenti convergenti\ + \ separate da 100 cm. La lente dell'oculare ha una lunghezza focale di 20\ + \ cm. L'ingrandimento angolare del telescopio \xE8" + - input_choice_list: + A: Temperatura costante + B: Volume costante + C: Pressione costante + D: Adiabatico + input_correct_responses: + - B + input_question: "Per quale dei seguenti processi termodinamici l'aumento dell'energia\ + \ interna di un gas ideale \xE8 uguale al calore ceduto al gas?" + - input_choice_list: + A: 2,4 V + B: 3,3 V + C: 4,5 V + D: 5,7 V + input_correct_responses: + - A + input_question: "Un'estremit\xE0 di un filo di nichelcromo di lunghezza 2L\ + \ e area di sezione trasversale A \xE8 collegata a un'estremit\xE0 di un\ + \ altro filo di nichelcromo di lunghezza L e area di sezione trasversale 2A.\ + \ Se l'estremit\xE0 libera del filo pi\xF9 lungo ha un potenziale elettrico\ + \ di 8,0 volt e l'estremit\xE0 libera del filo pi\xF9 corto ha un potenziale\ + \ elettrico di 1,0 volt, il potenziale alla giunzione dei due fili \xE8 quasi\ + \ uguale a" + - input_choice_list: + A: '4' + B: '5' + C: '6' + D: '20' + input_correct_responses: + - A + input_question: "Un telescopio rifrattore \xE8 formato da due lenti convergenti\ + \ separate da 100 cm. La lente dell'oculare ha una lunghezza focale di 20\ + \ cm. L'ingrandimento angolare del telescopio \xE8" + - input_choice_list: + A: carica + B: massa + C: energia e slancio + D: numero leptonico + input_correct_responses: + - D + input_question: "Il muone decade con una vita caratteristica di circa 10^-6 secondi\ + \ in un elettrone, un neutrino muonico e un antineutrino elettronico. Al muone\ + \ \xE8 vietato decadere in un elettrone e in un solo neutrino dalla legge di\ + \ conservazione" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_physics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_college_physics +task_alias: college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19e532200e61280062f0a9485412cb686accb6f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_conceptual_physics.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: meno + B: "Di pi\xF9" + C: lo stesso + D: zero + input_correct_responses: + - A + input_question: Rispetto alla massa di un atomo di uranio sottoposto a fissione, + le masse combinate dei prodotti dopo la fissione sono + - input_choice_list: + A: spazio e tempo. + B: un gemello viaggiante e un gemello casalingo. + C: "gravit\xE0 e accelerazione." + D: massa ed energia. + input_correct_responses: + - C + input_question: Le cose che sono equivalenti secondo il principio di equivalenza + lo sono + - input_choice_list: + A: convertito in una frequenza diversa + B: deflessione + C: interferenza + D: polarizzazione + input_correct_responses: + - C + input_question: I colori in una bolla di sapone risultano dalla luce + - input_choice_list: + A: lo stesso + B: maggiore + C: meno + D: "maggiore o minore a seconda della velocit\xE0 del vento" + input_correct_responses: + - B + input_question: "Un modellino di aeroplano vola pi\xF9 lentamente quando vola\ + \ controvento e pi\xF9 velocemente quando il vento \xE8 alle sue spalle. Quando\ + \ lanciato ad angolo retto rispetto al vento, con vento trasversale la sua velocit\xE0\ + \ al suolo \xE8 rispetto al volo in aria ferma" + - input_choice_list: + A: Idrogeno + B: Ferro + C: Uranio + D: Lo stesso in ciascuno + input_correct_responses: + - A + input_question: Quale di questi tre elementi ha la massa maggiore per nucleone? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_conceptual_physics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_conceptual_physics +task_alias: conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f92edf9e0c492efbb92a621f9a23577c75c2361a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_econometrics.yaml @@ -0,0 +1,64 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Alla fine morirai + B: Perseverare indefinitamente + C: Crescere in modo esponenziale + D: Non verificarsi mai + input_correct_responses: + - A + input_question: Per un processo autoregressivo stazionario, gli shock lo faranno + - input_choice_list: + A: 0,2 + B: '0.4' + C: 0,5 + D: 0,33 + input_correct_responses: + - D + input_question: "Consideriamo il seguente modello AR(1) con i disturbi aventi\ + \ media nulla e varianza unitaria yt = 0,2 + 0,4 yt-1 + ut La media (incondizionata)\ + \ di y sar\xE0 data da" + - input_choice_list: + A: Solo (ii) e (iv). + B: Solo (i) e (iii). + C: solo (i), (ii) e (iii). + D: (i), (ii), (iii) e (iv) + input_correct_responses: + - C + input_question: "Supponiamo che ad una statistica test sia associato un valore\ + \ p pari a 0,08. Quale delle seguenti affermazioni \xE8 vera? (i) Se la dimensione\ + \ del test fosse esattamente l\u20198%, saremmo indifferenti tra rifiutare e\ + \ non rifiutare l\u2019ipotesi nulla (ii) Il valore nullo verrebbe rifiutato\ + \ se venisse utilizzata una dimensione del test del 10% (iii) Il valore nullo\ + \ non verrebbe rifiutato essere rifiutato se fosse utilizzata una dimensione\ + \ del test dell'1%. (iv) Il valore nullo verrebbe rifiutato se fosse utilizzata\ + \ una dimensione del test del 5%." + - input_choice_list: + A: "Sar\xE0 di parte" + B: "Sar\xE0 incoerente" + C: "Sar\xE0 inefficiente" + D: Tutti i punti (a), (b) e (c) saranno veri. + input_correct_responses: + - C + input_question: "Quali sarebbero allora le conseguenze per lo stimatore OLS se\ + \ l\u2019eteroschedasticit\xE0 fosse presente in un modello di regressione ma\ + \ venisse ignorata?" + - input_choice_list: + A: 1 ritardo + B: 2 ritardi + C: 3 ritardi + D: 4 ritardi + input_correct_responses: + - C + input_question: "Supponiamo ora che un ricercatore desideri utilizzare criteri\ + \ informativi per determinare la lunghezza di ritardo ottimale per un VAR. Sono\ + \ disponibili 500 osservazioni per il VAR bivariato e i valori del determinante\ + \ della matrice di varianza-covarianza dei residui sono 0,0336, 0,0169, 0,0084\ + \ e 0,0062 rispettivamente per 1, 2, 3 e 4 ritardi. Qual \xE8 l'ordine ottimale\ + \ dei modelli secondo il criterio informativo di Akaike?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_econometrics +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_econometrics +task_alias: econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddc59953e17be3fed6952f70612b55261b3b3c66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_elementary_mathematics.yaml @@ -0,0 +1,58 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: 5 migliaia + B: 5 centinaia + C: 5 decine + D: 5 quelli + input_correct_responses: + - A + input_question: "La popolazione della citt\xE0 dove \xE8 nata Michelle \xE8 di\ + \ 145.826 abitanti. Qual \xE8 il valore del 5 nel numero 145.826?" + - input_choice_list: + A: "Il decimo numero nello schema sar\xE0 un numero pari." + B: "Lo schema numerico non avr\xE0 mai due numeri pari uno accanto all'altro." + C: I prossimi due numeri nello schema saranno un numero pari e poi un numero + dispari. + D: Se lo schema numerico iniziasse con un numero dispari, allora lo schema conterrebbe + solo numeri dispari. + input_correct_responses: + - B + input_question: "Olivia ha utilizzato la regola "Aggiungi 11" per creare\ + \ lo schema numerico mostrato di seguito. 10, 21, 32, 43, 54 Quale affermazione\ + \ sullo schema numerico \xE8 vera?" + - input_choice_list: + A: Aggiungi 5 a 30 per trovare 35 squadre. + B: Dividi 30 per 5 per trovare 6 squadre. + C: Moltiplica 30 e 5 per trovare 150 squadre. + D: Sottrai 5 da 30 per trovare 25 squadre. + input_correct_responses: + - B + input_question: Un totale di 30 giocatori giocheranno a basket in un parco. Ci + saranno esattamente 5 giocatori in ogni squadra. Quale affermazione spiega correttamente + come trovare il numero di squadre necessarie? + - input_choice_list: + A: '749' + B: '2.675' + C: '2.945' + D: '4.250' + input_correct_responses: + - B + input_question: "Un negozio vende 107 colori diversi di vernice. Hanno 25 lattine\ + \ di ogni colore in magazzino. Il numero di barattoli di vernice che il negozio\ + \ ha in magazzino pu\xF2 essere trovato utilizzando l'espressione seguente.\ + \ 107\xD725. Quanti barattoli di vernice ha in magazzino il negozio?" + - input_choice_list: + A: "(5\xD74)\xD7(6\xD75)" + B: "(5\xD75) + (5\xD74)" + C: "(5\xD75) + (5\xD79)" + D: "(5\xD79)\xD7(6\xD79)" + input_correct_responses: + - B + input_question: Quale espressione equivale a 5 x 9? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_elementary_mathematics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_elementary_mathematics +task_alias: elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b68ac27f2fd13eaef6289182413206f7c1007df6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_formal_logic.yaml @@ -0,0 +1,63 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ~Pd + B: "(\u2200x)(Px \u2228 ~Dx)" + C: "(\u2200x)(Px \u2283 ~Dx)" + D: ~Dp + input_correct_responses: + - C + input_question: ' Seleziona la migliore traduzione nella logica dei predicati: + nessuna persona guida su Marte.' + - input_choice_list: + A: Blgh + B: Bhlg + C: Bglh + D: Bghl + input_correct_responses: + - C + input_question: 'Seleziona la migliore traduzione nella logica dei predicati. + George prende in prestito il tosaerba di Hector. (g: George; h: Hector; l: il + tosaerba di Hector; Bxyx: x prende in prestito y da z)' + - input_choice_list: + A: "Marina \xE8 una ballerina. Alcuni deboli non sono ballerini. O tutto \xE8\ + \ debole o la Georgia gioca a pallavolo. Quindi qualcosa gioca a pallavolo." + B: "Marina \xE8 una ballerina. Nessun debole \xE8 un ballerino. Tutto o \xE8\ + \ un debole o gioca a pallavolo. Quindi qualcosa gioca a pallavolo." + C: "Marina \xE8 una ballerina. Alcuni deboli non sono ballerini. Tutto o \xE8\ + \ un debole o gioca a pallavolo. Quindi qualcosa gioca a pallavolo." + D: "Marina \xE8 una ballerina. Nessun debole \xE8 un ballerino. O tutto \xE8\ + \ debole o la Georgia gioca a pallavolo. Quindi qualcosa gioca a pallavolo." + input_correct_responses: + - D + input_question: " Seleziona la migliore interpretazione inglese degli argomenti\ + \ forniti nella logica dei predicati. Dm (\u2200x)(Wx \u2283 ~Dx) (\u2200x)Wx\ + \ \u2228 Ag / (\u2203x)Ax" + - input_choice_list: + A: Logicamente equivalente + B: Contraddittorio + C: "N\xE9 logicamente equivalenti n\xE9 contraddittori, ma coerenti" + D: Incoerente + input_correct_responses: + - C + input_question: " Costruisci una tavola di verit\xE0 completa per le seguenti\ + \ coppie di proposizioni. Quindi, utilizzando le tavole di verit\xE0, determina\ + \ se le affermazioni sono logicamente equivalenti o contraddittorie. In caso\ + \ contrario, determinare se sono coerenti o incoerenti. Motiva le tue risposte.\ + \ E \u2283 (F \xB7 E) e ~E \xB7 F" + - input_choice_list: + A: "(L \u2022 H) \u2261 I" + B: "(L \u2022 H) \u2228 I" + C: "L \u2022 (H \u2228 I)" + D: "L \u2022 (H \u2283 R)" + input_correct_responses: + - B + input_question: " Quale delle formule fornite di PL \xE8 la migliore simbolizzazione\ + \ della seguente frase? Le tartarughe vivono a lungo e sono creature felici,\ + \ a meno che non vengano ferite." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_formal_logic +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_formal_logic +task_alias: formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ade44c5eee5d457a4a67f9ef9bf17945acb91f82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_global_facts.yaml @@ -0,0 +1,57 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Le persone tendono ad essere ottimiste riguardo al proprio futuro e al futuro + della propria nazione o del mondo. + B: Le persone tendono ad essere ottimiste riguardo al proprio futuro ma pessimiste + riguardo al futuro della propria nazione o del mondo. + C: Le persone tendono ad essere pessimiste riguardo al proprio futuro ma ottimiste + riguardo al futuro della propria nazione o del mondo. + D: Le persone tendono ad essere pessimiste riguardo al proprio futuro e al futuro + della propria nazione o del mondo. + input_correct_responses: + - B + input_question: Quali delle seguenti coppie di affermazioni sono entrambe vere + (al 2019)? + - input_choice_list: + A: 31% + B: 46% + C: 61% + D: 76% + input_correct_responses: + - B + input_question: "Nel 2019, quale percentuale di americani concorda sul fatto che\ + \ lo stato \xE8 gestito a beneficio di tutte le persone?" + - input_choice_list: + A: aumentato, aumentato + B: aumentato, diminuito + C: diminuito, aumentato + D: diminuito, diminuito + input_correct_responses: + - B + input_question: Nel 2015, dal 1990 le foreste hanno ____ in Europa e hanno ____ + in Africa e nelle Americhe. + - input_choice_list: + A: 38% + B: 53% + C: 68% + D: 83% + input_correct_responses: + - A + input_question: "Nel 2019, quale percentuale di russi afferma che \xE8 molto importante\ + \ avere media liberi nel nostro Paese senza censura governativa/statale?" + - input_choice_list: + A: 80% + B: 60% + C: 40% + D: 20% + input_correct_responses: + - A + input_question: Nel 2017, quanti bambini di 1 anno nel mondo sono stati oggi vaccinati + contro alcune malattie? * +include: _continuation_template_yaml +process_docs: !function utils.process_docs_global_facts +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_global_facts +task_alias: global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f488cb09a830bf6eec3bb3e6caabe68a3111d3ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_chemistry.yaml @@ -0,0 +1,58 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: HCl + B: H2SO3 + C: SO2 + D: Al(NO3)3 + input_correct_responses: + - C + input_question: "Quale delle seguenti \xE8 considerata un'anidride acida?" + - input_choice_list: + A: PCl4F + B: BF3 + C: CO2 + D: Si(CH3)4 + input_correct_responses: + - A + input_question: Quale delle seguenti dovrebbe essere una molecola polare? + - input_choice_list: + A: Tutti i cloruri, i bromuri e gli ioduri sono solubili + B: Tutti i solfati sono solubili + C: Tutti gli idrossidi sono solubili + D: Tutti i composti contenenti ammonio sono solubili + input_correct_responses: + - D + input_question: "Dalle regole di solubilit\xE0, quale delle seguenti affermazioni\ + \ \xE8 vera?" + - input_choice_list: + A: 3,89 + B: '7.78' + C: '5.78' + D: '2.33' + input_correct_responses: + - C + input_question: "Viene sintetizzato un nuovo composto e si scopre che \xE8 un\ + \ acido monoprotico con una massa molare di 248 g/mol. Quando 0,0050 moli di\ + \ questo acido vengono sciolte in 0,500 L di acqua, il pH viene misurato come\ + \ 3,89. Qual \xE8 il pKa di questo acido?" + - input_choice_list: + A: 0,500 moli + B: 1,00 talpa + C: 2,00 talpe + D: 3,00 talpe + input_correct_responses: + - C + input_question: "Una soluzione contiene 2,00 moli di acido acetico, CH3COOH, e\ + \ 1,00 moli di acetato di calcio, Ca(CH3COO)2. La soluzione \xE8 in grado di\ + \ resistere all'aggiunta di una piccola quantit\xE0 di acido forte o base\ + \ forte con solo lievi variazioni del pH della soluzione. Quantit\xE0 maggiori\ + \ di acido forte o base forte possono causare un cambiamento significativo nel\ + \ pH. Quante moli di acido nitrico, HNO3, possono essere aggiunte prima che\ + \ il pH inizi a cambiare in modo significativo?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_chemistry +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_high_school_chemistry +task_alias: high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c1cab1fcb24babe6d25cd4d4eb97867259b0659 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_computer_science.yaml @@ -0,0 +1,83 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Un'auto avvisa il conducente che sta per colpire un oggetto. + B: Un escursionista utilizza un orologio GPS per tenere traccia della sua posizione. + C: "Un frigorifero ordina il latte da un servizio di consegna online quando\ + \ il latte nel frigorifero \xE8 quasi finito." + D: Un corridore utilizza un orologio con sensori ottici per monitorare la sua + frequenza cardiaca. + input_correct_responses: + - C + input_question: "Quale dei seguenti \xE8 un esempio di utilizzo di un dispositivo\ + \ nell'Internet delle cose (IoT)?" + - input_choice_list: + A: "Le attivit\xE0 di un utente che naviga in una finestra anonima non saranno\ + \ visibili alle persone che monitorano la rete dell'utente, come l'amministratore\ + \ di sistema." + B: Gli articoli inseriti nel carrello di un negozio Web per futuri acquisti + durante la sessione di navigazione anonima non verranno salvati sul computer + dell'utente. + C: "Un utente non sar\xE0 in grado di accedere agli account di posta elettronica\ + \ o ai social media durante la sessione di navigazione anonima." + D: "Un utente che naviga in una finestra anonima sar\xE0 protetto dai virus\ + \ lanciati da qualsiasi sito web visitato o file scaricati." + input_correct_responses: + - B + input_question: "Molti browser Web consentono agli utenti di aprire finestre anonime.\ + \ Durante una sessione di navigazione in una finestra anonima, il browser non\ + \ registra n\xE9 la cronologia di navigazione n\xE9 l'elenco dei file scaricati.\ + \ Quando si esce dalla finestra anonima, i cookie creati durante la sessione\ + \ vengono cancellati. Quale delle seguenti affermazioni relative alle sessioni\ + \ di navigazione in una finestra anonima \xE8 vera?" + - input_choice_list: + A: Errore + B: abc + C: cba + D: C + input_correct_responses: + - C + input_question: "Qual \xE8 l'output di "abc"[::-1] in Python 3?" + - input_choice_list: + A: Foxtrot + B: Hotel + C: novembre + D: yankee + input_correct_responses: + - C + input_question: "Nel programma seguente, il valore iniziale di x \xE8 5 e il valore\ + \ iniziale di y \xE8 10. IF (X < O) { DISPLAY ("Foxtrot") } ELSE\ + \ { IF (X > y) { DISPLAY ("Hotel") } ELSE { IF (y > O) { DISPLAY\ + \ ("Novembre") } ELSE { DISPLAY ("Yankee") } } } Cosa viene\ + \ visualizzato come risultato dell'esecuzione del programma?" + - input_choice_list: + A: "Passaggio 3: aumentare il valore della posizione di 1. Passaggio 4: ripetere\ + \ i passaggi 2 e 3 finch\xE9 il valore del conteggio non \xE8 maggiore di\ + \ 100." + B: "Passaggio 3: aumentare il valore della posizione di 1. Passaggio 4: ripetere\ + \ i passaggi 2 e 3 fino a quando il valore della posizione \xE8 maggiore di\ + \ n." + C: "Passaggio 3: ripetere il passaggio 2 finch\xE9 il valore del conteggio non\ + \ \xE8 maggiore di 100. Passaggio 4: aumentare il valore della posizione di\ + \ 1." + D: "Passaggio 3: ripetere il passaggio 2 finch\xE9 il valore della posizione\ + \ non \xE8 maggiore di n. Passaggio 4: aumentare il valore del conteggio di\ + \ 1." + input_correct_responses: + - D + input_question: "Una lista di numeri ha n elementi, indicizzati da 1 a n. Il seguente\ + \ algoritmo ha lo scopo di visualizzare il numero di elementi nell'elenco\ + \ che hanno un valore maggiore di 100. L'algoritmo utilizza le variabili\ + \ count e position. Mancano i passaggi 3 e 4. Passaggio 1: impostare il conteggio\ + \ su 0 e la posizione su 1. Passaggio 2: se il valore dell'elemento nella\ + \ posizione dell'indice \xE8 maggiore di 100, aumentare il valore del conteggio\ + \ di 1. Passaggio 3: (passaggio mancante) Passaggio 4: (passaggio mancante )\ + \ Passo 5: Visualizzare il valore del conteggio. Quale dei seguenti potrebbe\ + \ essere utilizzato per sostituire i passaggi 3 e 4 in modo che l'algoritmo\ + \ funzioni come previsto?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_computer_science +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_high_school_computer_science +task_alias: high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9f905d4f1b9c98fd66b850aea78efcbe6bf9571 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_european_history.yaml @@ -0,0 +1,194 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Jean-Jacques Rousseau + B: Barone Montesquieu + C: Mary Wollstonecraft + D: Adam Smith + input_correct_responses: + - B + input_question: "Questa domanda si riferisce alle seguenti informazioni. Il seguente\ + \ estratto \xE8 tratto da un opuscolo. Mi renderai giustizia ricordando che\ + \ ho sempre strenuamente sostenuto il diritto di ogni uomo alla propria opinione,\ + \ per quanto diversa possa essere dalla mia. Chi nega a un altro questo diritto,\ + \ si rende schiavo della sua opinione presente, perch\xE9 si preclude il diritto\ + \ di cambiarla. L'arma pi\xF9 formidabile contro gli errori di ogni genere\ + \ \xE8 la Ragione. Non ne ho mai usato nessun altro e confido che non lo far\xF2\ + \ mai. La circostanza che si \xE8 verificata ora in Francia, dell\u2019abolizione\ + \ totale dell\u2019intero ordine nazionale del sacerdozio e di tutto ci\xF2\ + \ che appartiene ai sistemi compulsivi di religione e agli articoli di fede\ + \ compulsivi, non solo ha accelerato la mia intenzione, ma ha reso l\u2019opera\ + \ di questo un genere estremamente necessario, affinch\xE9 nel generale naufragio\ + \ della superstizione, dei falsi sistemi di governo e della falsa teologia,\ + \ perdiamo di vista la moralit\xE0, l'umanit\xE0 e la teologia che \xE8\ + \ vera. Credo in un solo Dio, e niente di pi\xF9; e spero nella felicit\xE0\ + \ oltre questa vita. Credo nell'uguaglianza dell'uomo; e credo che i\ + \ doveri religiosi consistano nel fare giustizia, amare la misericordia e sforzarsi\ + \ di rendere felici i nostri simili. Non credo nel credo professato dalla chiesa\ + \ ebraica, dalla chiesa romana, dalla chiesa greca, dalla chiesa turca, dalla\ + \ chiesa protestante, n\xE9 da alcuna chiesa che io conosca. La mia mente \xE8\ + \ la mia chiesa. Tutte le istituzioni nazionali delle chiese, siano esse ebraiche,\ + \ cristiane o turche, non mi sembrano altro che invenzioni umane, istituite\ + \ per terrorizzare e schiavizzare l\u2019umanit\xE0 e monopolizzare il potere\ + \ e il profitto. Con questa dichiarazione non intendo condannare coloro che\ + \ credono il contrario; hanno lo stesso diritto alla loro convinzione che io\ + \ ho alla mia. \u2014Thomas Paine, The Age of Reason, 1794\u20131795 Quale dei\ + \ seguenti filosofi illuministi progett\xF2 un sistema di controlli ed equilibri\ + \ affinch\xE9 il governo evitasse abusi di potere?" + - input_choice_list: + A: "Le idee di libert\xE0 personale e nazionalismo concepite durante l\u2019\ + Illuminismo portarono a rivoluzioni radicali che poterono diffondersi in tutta\ + \ Europa." + B: "La conquista dell'Europa da parte di Napoleone port\xF2 alla creazione\ + \ di nuove fazioni e spost\xF2 gli equilibri di potere europei." + C: Il potere dei monarchi era cresciuto al punto che doveva essere controllato + da altri poteri all'interno di ciascuna nazione o si sarebbe verificata + la dominazione dei civili. + D: Il ciclo economico in salita e in discesa della nuova economia capitalista + emergente potrebbe portare a disordini civili che devono essere repressi. + input_correct_responses: + - A + input_question: "Questa domanda si riferisce alle seguenti informazioni. Leggi\ + \ il seguente estratto. Il seme rivoluzionario era penetrato in tutti i paesi\ + \ e si era pi\xF9 o meno diffuso. Si svilupp\xF2 molto sotto il regime del dispotismo\ + \ militare di Bonaparte. Le sue conquiste hanno spostato una serie di leggi,\ + \ istituzioni e costumi; ruppe i legami sacri tra tutte le nazioni, abbastanza\ + \ forti da resistere al tempo stesso; il che \xE8 pi\xF9 di quanto si possa\ + \ dire di alcuni benefici conferiti da questi innovatori. I monarchi adempiranno\ + \ i doveri imposti loro da Colui che, affidando loro il potere, li ha incaricati\ + \ di vigilare sul mantenimento della giustizia e dei diritti di tutti, di evitare\ + \ le vie dell'errore e di percorrere con fermezza la via dell'errore.\ + \ verit\xE0. Posti al di l\xE0 delle passioni che agitano la societ\xE0, \xE8\ + \ soprattutto nei giorni della prova che sono chiamati a spogliare la realt\xE0\ + \ delle loro false apparenze, e a mostrarsi quali sono, padri investiti dell'autorit\xE0\ + \ che spetta di diritto ai capi famiglia, per dimostrare che, nei giorni di\ + \ lutto, sanno essere giusti, saggi e quindi forti, e che non abbandoneranno\ + \ il popolo che dovrebbero governare come gioco delle fazioni, all\u2019errore\ + \ e alle sue conseguenze, che devono implicano la perdita della societ\xE0.\ + \ L'unione tra i monarchi \xE8 la base della politica che ora deve essere\ + \ seguita per salvare la societ\xE0 dalla rovina totale. . . . Non confondano\ + \ le concessioni fatte ai partiti con il bene che dovrebbero fare al popolo,\ + \ modificando, secondo i bisogni riconosciuti, i rami dell'amministrazione\ + \ che lo richiedono. Siano giusti, ma forti; benefico, ma severo. Mantengano\ + \ i principi religiosi in tutta la loro purezza e non permettano che la fede\ + \ venga attaccata e la moralit\xE0 interpretata secondo il contratto sociale\ + \ o le visioni di sciocchi settari. Sopprimano le societ\xE0 segrete; quella\ + \ cancrena della societ\xE0. \u2014Klemens von Metternich, Confessione politica\ + \ di fede, 1820 Quale delle seguenti \xE8 stata la causa principale dei timori\ + \ espressi da Metternich nel documento di cui sopra?" + - input_choice_list: + A: Capitalista + B: Scientifico + C: comunista + D: Esistenzialista + input_correct_responses: + - C + input_question: "Questa domanda si riferisce alle seguenti informazioni. In Russia\ + \ non andava tutto bene e [Souvarine] era disperato per la notizia che aveva\ + \ ricevuto. I suoi vecchi compagni si rivolgevano tutti ai politici; i famosi\ + \ nichilisti che facevano tremare l\u2019Europa \u2013 figli di preti di villaggio,\ + \ di piccoli borghesi, di commercianti \u2013 non potevano elevarsi al di sopra\ + \ dell\u2019idea di liberazione nazionale, e sembravano credere che il mondo\ + \ sarebbe stato liberato \u2013 dopo aver ucciso il loro despota\u2026 "Sciocchezze!\ + \ Con le loro sciocchezze non ne usciranno mai." Poi, abbassando ancora\ + \ di pi\xF9 la voce, con poche parole amare descrisse il suo vecchio sogno di\ + \ fraternit\xE0. Aveva rinunciato al suo rango e alla sua fortuna; era andato\ + \ tra gli operai, solo nella speranza di vedere finalmente la fondazione di\ + \ una nuova societ\xE0 di lavoro in comune. Tutti i soldi che aveva in tasca\ + \ erano andati da tempo ai monelli del villaggio; era stato tenero come un fratello\ + \ con i minatori, sorridendo dei loro sospetti, conquistandoli con i suoi modi\ + \ tranquilli da operaio e la sua avversione per le chiacchiere. Ma decisamente\ + \ la fusione non era avvenuta. La sua voce cambi\xF2, i suoi occhi si illuminarono,\ + \ li fiss\xF2 su \xC9tienne, rivolgendosi direttamente a lui: "Adesso capisci?\ + \ Questi cappellai di Marsiglia che hanno vinto il grande premio di centomila\ + \ franchi della lotteria, sono subito partiti e hanno investito voi, voi tutti,\ + \ operai francesi, volete dissotterrare un tesoro per divorarlo poi da soli,\ + \ in qualche angolo pigro ed egoista. per quanto tu voglia contro i ricchi,\ + \ non hai abbastanza coraggio per restituire ai poveri il denaro che la fortuna\ + \ ti porta. Non sarai mai degno di felicit\xE0 finch\xE9 possiedi qualcosa,\ + \ e il tuo odio contro i borghesi procede unicamente per il desiderio rabbioso\ + \ di essere voi stessi borghesi al loro posto." \xE9mile Zola, scrittore\ + \ francese, Germinal, 1885 Il passaggio mostra la preoccupazione diretta per\ + \ il benessere delle classi lavoratrici che tipicamente faceva parte di quale\ + \ movimento?" + - input_choice_list: + A: Servirono da catalizzatore per la crescita della navigazione inglese e del + commercio estero, ma fecero ben poco per limitare le prospettive degli olandesi + nel XVII secolo. + B: "Causarono difficolt\xE0 quasi immediate per l\u2019economia olandese poich\xE9\ + \ il loro dominio sul commercio estero fin\xEC rapidamente." + C: "Furono annullati durante la restaurazione degli Stuart poich\xE9 cercavano\ + \ normali relazioni diplomatiche con gli olandesi in modo da non aver bisogno\ + \ del sostegno finanziario del Parlamento per la guerra." + D: "Portarono a quasi un secolo di guerre ricorrenti tra Inghilterra e Paesi\ + \ Bassi, che non sarebbero finite fino a dopo l\u2019indipendenza americana." + input_correct_responses: + - A + input_question: "Questa domanda si riferisce alle seguenti informazioni. Gli estratti\ + \ seguenti provengono dagli Atti di navigazione del 1651. [Dopo il primo giorno\ + \ di dicembre milleseicentocinquantuno, e da allora in poi, nessun bene o merce\ + \ di sorta derivante dalla crescita, produzione o manifattura dell'Asia,\ + \ dell'Africa o l'America, o qualsiasi parte di essa; o di qualsiasi\ + \ isola ad essi appartenente, o che sono descritte o indicate nelle consuete\ + \ mappe o carte di quei luoghi, cos\xEC come delle piantagioni inglesi come\ + \ altre, saranno importate o portate in questo Commonwealth d'Inghilterra,\ + \ o in Irlanda, o qualsiasi altra terra, isola, piantagione o territorio di\ + \ questo Commonwealth appartenente, o in loro possesso, in qualsiasi altra nave\ + \ o nave, vascello o vascello di sorta, ma solo in quelli che veramente e senza\ + \ frode appartengono solo al popolo di questo Commonwealth , o le relative piantagioni,\ + \ in qualit\xE0 di proprietari o titolari dei diritti; e di cui il capitano\ + \ e i marinai appartengono anche al popolo di questo Commonwealth, sotto pena\ + \ di confisca e perdita di tutti i beni che saranno importati contrariamente\ + \ a questo atto, , , , [N] o beni o merci della crescita, la produzione, o la\ + \ manifattura dell'Europa, o di qualsiasi parte di essa, dopo il primo giorno\ + \ di dicembre milleseicentocinquantuno, sar\xE0 importata o portata in questo\ + \ Commonwealth d'Inghilterra, o in qualsiasi altra terra o territorio appartenente\ + \ a questo Commonwealth, o in loro possesso, in qualsiasi nave o navi, vascello\ + \ o vascelli di qualsiasi genere, ma in quelli che appartengono veramente e\ + \ senza frode solo al popolo di questo Commonwealth, e in nessun altro, eccetto\ + \ solo le navi e i vascelli stranieri che appartengono veramente e appartengono\ + \ propriamente al popolo di quel paese o luogo di cui detti beni sono la crescita,\ + \ la produzione o la manifattura. Quale delle seguenti affermazioni descrive\ + \ meglio l'esito dei Navigation Acts del 1651?" + - input_choice_list: + A: "dare al re inglese una nuova posizione di autorit\xE0" + B: dare la carica di capo della Chiesa d'Inghilterra al solo Enrico VIII + ed escludere i suoi eredi + C: "stabilire il calvinismo come l\u2019unica vera teologia in Inghilterra" + D: porre fine alle varie forme di corruzione che affliggono la Chiesa in Inghilterra + input_correct_responses: + - D + input_question: "Questa domanda si riferisce alle seguenti informazioni. Sebbene\ + \ la Maest\xE0 del re sia giustamente e legittimamente il capo supremo della\ + \ Chiesa d'Inghilterra, e cos\xEC sia riconosciuta dal clero di questo regno\ + \ nelle loro convocazioni, tuttavia, per corroborazione e conferma di ci\xF2\ + , e per aumento della virt\xF9 in religione di Cristo in questo regno d'Inghilterra,\ + \ e per reprimere ed estirpare tutti gli errori, le eresie e le altre enormit\xE0\ + \ e abusi finora utilizzati nella stessa, sia decretato, per autorit\xE0 di\ + \ questo attuale Parlamento, che il re, il nostro signore sovrano, i suoi eredi\ + \ e i successori, i re di questo regno, saranno presi, accettati e ritenuti\ + \ l'unico capo supremo sulla terra della Chiesa d'Inghilterra, chiamata\ + \ Ecclesia anglicana; e avranno e godranno, annessi e uniti alla corona imperiale\ + \ di questo regno, nonch\xE9 il titolo e lo stile della stessa, come tutti gli\ + \ onori, dignit\xE0, preminenze, giurisdizioni, privilegi, autorit\xE0, immunit\xE0\ + , profitti e beni alla detta dignit\xE0 di il capo supremo della stessa Chiesa\ + \ appartenente e appartenente; e che il nostro suddetto signore sovrano, i suoi\ + \ eredi e successori, re di questo regno, avranno pieno potere e autorit\xE0\ + \ di volta in volta per visitare, reprimere, riparare, registrare, ordinare,\ + \ correggere, frenare ed emendare tutti questi errori, eresie, abusi, offese,\ + \ disprezzi ed enormit\xE0, qualunque essi siano, che per qualsiasi tipo di\ + \ autorit\xE0 o giurisdizione spirituale dovrebbero o potrebbero essere legittimamente\ + \ riformati, repressi, ordinati, riparati, corretti, frenati o modificati, per\ + \ il massimo piacere di Dio Onnipotente, l'aumento della virt\xF9 nella\ + \ religione di Cristo e per la conservazione della pace, dell'unit\xE0 e\ + \ della tranquillit\xE0 di questo regno; nonostante qualsiasi uso, terra straniera,\ + \ autorit\xE0 straniera, prescrizione o qualsiasi altra cosa contraria al presente\ + \ documento. Parlamento inglese, Atto di Supremazia, 1534 Dal passaggio si pu\xF2\ + \ dedurre che il Parlamento inglese volesse sostenere che l'Atto di Supremazia\ + \ avrebbe" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_european_history +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_high_school_european_history +task_alias: high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7a3d571cd0f6e4348a8dea12741f7108c7581bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_geography.yaml @@ -0,0 +1,58 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "tasso di mortalit\xE0 grezzo dalla data di nascita grezza." + B: "tasso di natalit\xE0 grezzo dal tasso di mortalit\xE0 grezzo." + C: "raddoppiando il tempo rispetto al tasso di natalit\xE0 grezzo." + D: "tasso di fertilit\xE0 dal tasso grezzo di mortalit\xE0." + input_correct_responses: + - A + input_question: Il tasso di crescita naturale di una popolazione si trova sottraendo + il + - input_choice_list: + A: "I tassi di natalit\xE0 aumentano e il tasso di crescita della popolazione\ + \ \xE8 meno rapido." + B: "I tassi di natalit\xE0 diminuiscono e il tasso di crescita della popolazione\ + \ \xE8 meno rapido." + C: "I tassi di natalit\xE0 aumentano e il tasso di crescita della popolazione\ + \ aumenta." + D: "I tassi di natalit\xE0 diminuiscono e il tasso di crescita della popolazione\ + \ aumenta." + input_correct_responses: + - B + input_question: "Durante la terza fase del modello di transizione demografica,\ + \ quale delle seguenti affermazioni \xE8 vera?" + - input_choice_list: + A: La duplicazione degli sforzi avviene spesso. + B: "I problemi sociali del centro citt\xE0 si riversano nei sobborghi residenziali\ + \ circostanti." + C: Spesso si verificano inefficienze nella fornitura dei servizi. + D: "Gli sforzi di un quartiere per ridurre l'inquinamento sono sempre sostenuti\ + \ dalle comunit\xE0 vicine." + input_correct_responses: + - D + input_question: "Quale delle seguenti affermazioni NON \xE8 accurata riguardo\ + \ ai servizi forniti dai governi locali negli Stati Uniti?" + - input_choice_list: + A: esternalizzazione. + B: delocalizzazione. + C: maquiladoras. + D: interdipendenza localizzata. + input_correct_responses: + - B + input_question: Viene chiamata la pratica di assumere un fornitore di servizi + terzo straniero per eseguire un'operazione + - input_choice_list: + A: Saponetta colomba + B: Barretta di cioccolato colomba + C: Simbolo della colomba + D: Una colomba (uccello) + input_correct_responses: + - C + input_question: "Quale dei seguenti elementi \xE8 un esempio di cultura non materiale?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_geography +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_high_school_geography +task_alias: high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cacce8dda966953a230550b44424d192d3c7c44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_government_and_politics.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "la definizione costituzionale di tali poteri \xE8 ampia e non specifica" + B: la maggior parte delle persone concorda sul fatto che la Costituzione pone + troppi limiti al potere presidenziale + C: la Corte Suprema rifiuta costantemente di pronunciarsi su casi riguardanti + i poteri presidenziali + D: gli emendamenti costituzionali hanno notevolmente aumentato i poteri presidenziali + input_correct_responses: + - A + input_question: "L\u2019incertezza sui limiti del potere presidenziale \xE8 causata\ + \ principalmente da questo" + - input_choice_list: + A: aumento annuale della spesa federale per le forze armate + B: ammontare degli interessi sul debito pubblico + C: differenza tra le proposte di bilancio iniziali fatte dal presidente e dal + Congresso + D: importo che il governo spende in eccesso rispetto alle sue entrate + input_correct_responses: + - D + input_question: Il termine "deficit di bilancio" si riferisce al + - input_choice_list: + A: Weeks contro Stati Uniti + B: Betts contro Brady + C: Mapp contro Ohio + D: Miranda contro Arizona + input_correct_responses: + - D + input_question: Quale dei seguenti casi ha stabilito il precedente secondo cui + un imputato deve essere informato del diritto al silenzio, del diritto a un + avvocato e della protezione dall'autoincriminazione? + - input_choice_list: + A: Sono stabiliti dal potere legislativo. + B: I loro membri spesso non hanno molta influenza sulle decisioni presidenziali. + C: Non possono essere tutti gestiti da leader che appartengono allo stesso partito + politico del presidente. + D: Non tutte le agenzie federali sono un dipartimento di gabinetto. + input_correct_responses: + - C + input_question: "Quale delle seguenti affermazioni sui dipartimenti di gabinetto\ + \ \xE8 FALSA?" + - input_choice_list: + A: I politici onesti possono impedire lo sviluppo di fazioni. + B: "\xC8 pi\xF9 probabile che le fazioni si formino nelle grandi repubbliche\ + \ che in quelle piccole." + C: "Gli effetti negativi della faziosit\xE0 possono essere ridotti da un governo\ + \ repubblicano." + D: "Le libere elezioni sono la migliore difesa del popolo contro le faziosit\xE0\ + ." + input_correct_responses: + - C + input_question: Quale delle seguenti affermazioni meglio afferma un'argomentazione + avanzata da James Madison in The Federalist numero 10? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_government_and_politics +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_high_school_government_and_politics +task_alias: high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc9e41009352d9da2c5c604f803e6ed1c1393ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_macroeconomics.yaml @@ -0,0 +1,52 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: L'esercito americano apre una nuova base in un paese straniero con 1.000 + dipendenti statunitensi. + B: I consumatori giapponesi acquistano migliaia di CD prodotti negli Stati Uniti. + C: Una cantante pop americana si esibisce in un concerto tutto esaurito a Parigi. + D: "Una produzione teatrale francese gira decine di citt\xE0 americane." + input_correct_responses: + - C + input_question: "Quale dei seguenti elementi non \xE8 incluso nel PIL degli Stati\ + \ Uniti?" + - input_choice_list: + A: relazione diretta tra disoccupazione e inflazione + B: "relazione diretta tra prezzo e quantit\xE0 domandata" + C: "relazione inversa tra prezzo e quantit\xE0 domandata" + D: relazione inversa tra disoccupazione e inflazione + input_correct_responses: + - D + input_question: La curva di Phillips di breve periodo indica a + - input_choice_list: + A: le esportazioni superano le importazioni. + B: le importazioni superano le esportazioni. + C: la riscossione delle tasse federali supera la spesa. + D: la spesa federale supera le entrate fiscali federali. + input_correct_responses: + - D + input_question: Un deficit federale si verifica quando + - input_choice_list: + A: Aumento del tasso di sconto + B: Aumentare il coefficiente di riserva + C: Acquistare titoli di Stato + D: Abbassamento delle tariffe + input_correct_responses: + - C + input_question: "A parit\xE0 di condizioni, quale delle seguenti politiche monetarie\ + \ verrebbe utilizzata per stimolare le esportazioni statunitensi?" + - input_choice_list: + A: "Un aumento dell\u2019offerta di moneta" + B: Aumento della spesa pubblica + C: "Tasse pi\xF9 basse su ricerca e sviluppo di nuove tecnologie" + D: "Tasse pi\xF9 alte sul reddito delle famiglie" + input_correct_responses: + - C + input_question: "Quale delle seguenti politiche descrive meglio la politica fiscale\ + \ dal lato dell\u2019offerta?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_macroeconomics +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_high_school_macroeconomics +task_alias: high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b547ec502d9d785abb01db6415d36417923ad1da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_mathematics.yaml @@ -0,0 +1,61 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '3' + B: '15' + C: '6' + D: '5' + input_correct_responses: + - B + input_question: Joe era incaricato delle luci per un ballo. La luce rossa lampeggia + ogni due secondi, la luce gialla ogni tre secondi e la luce blu ogni cinque + secondi. Se includiamo l'inizio e la fine della danza, quante volte durante + una danza di sette minuti si accenderanno tutte le luci contemporaneamente? + (Supponiamo che tutte e tre le luci lampeggino simultaneamente all'inizio + della danza.) + - input_choice_list: + A: '12' + B: '1' + C: '30' + D: '5' + input_correct_responses: + - C + input_question: Cinquemila dollari composti ogni anno a un tasso di interesse + $x\%$ impiegano sei anni per raddoppiare. Allo stesso tasso di interesse, quanti + anni occorreranno $\$300$ per crescere fino a $\$9600$? + - input_choice_list: + A: '-1' + B: '16' + C: -\frac{1}{256} + D: \frac{1}{16} + input_correct_responses: + - C + input_question: "La variabile $x$ varia direttamente come il quadrato di $y$,\ + \ e $y$ varia direttamente come il cubo di $z$. Se $x$ \xE8 uguale a $-16$ quando\ + \ $z$ \xE8 uguale a 2, qual \xE8 il valore di $x$ quando $z$ \xE8 uguale a $\\\ + frac{1}{2}$?" + - input_choice_list: + A: \frac{3\sqrt{3}}{3} + B: \frac{1}{3} + C: \qrt{3} + D: \frac{\sqrt{3}}{3} + input_correct_responses: + - D + input_question: 'Semplifica e scrivi il risultato con un denominatore razionale: + $$\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}$$' + - input_choice_list: + A: '55' + B: '60' + C: '62' + D: '65' + input_correct_responses: + - D + input_question: "Dieci studenti sostengono un test di biologia e ricevono i seguenti\ + \ punteggi: 45, 55, 50, 70, 65, 80, 40, 90, 70, 85. Qual \xE8 la media dei punteggi\ + \ dei test degli studenti?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_mathematics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_high_school_mathematics +task_alias: high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2be97ac39f6fe6043de330189796de2126a59e02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_microeconomics.yaml @@ -0,0 +1,54 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Un salario minimo effettivo imposto a questo mercato del lavoro. + B: Un aumento del prezzo dei litri di vernice. + C: Un aumento nella costruzione di nuove case. + D: "Un aumento del prezzo dei verniciatori meccanici finch\xE9 l\u2019effetto\ + \ produzione supera l\u2019effetto sostituzione." + input_correct_responses: + - C + input_question: In un mercato del lavoro competitivo per gli imbianchini, quale + dei seguenti fattori aumenterebbe la domanda di imbianchini? + - input_choice_list: + A: "la domanda del prodotto aumenter\xE0" + B: "la domanda del prodotto diminuir\xE0" + C: "il surplus del consumatore aumenter\xE0" + D: "il surplus del consumatore diminuir\xE0" + input_correct_responses: + - C + input_question: Se il governo sovvenziona i produttori in un mercato perfettamente + competitivo, allora + - input_choice_list: + A: '0' + B: '5' + C: '10' + D: '100' + input_correct_responses: + - D + input_question: "Il rapporto di concentrazione per un monopolio \xE8" + - input_choice_list: + A: Il prezzo minimo sposta la curva di domanda verso sinistra. + B: Un pavimento efficace crea una carenza di bene. + C: Il prezzo minimo sposta la curva di offerta del bene verso destra. + D: Per essere un livello efficace, deve essere fissato al di sopra del prezzo + di equilibrio. + input_correct_responses: + - D + input_question: "Quale delle seguenti affermazioni \xE8 vera riguardo al prezzo\ + \ minimo?" + - input_choice_list: + A: Libera entrata e uscita dal mercato + B: Pochi grandi produttori + C: Un produttore di un bene senza sostituti prossimi + D: Un prodotto omogeneo + input_correct_responses: + - B + input_question: "Quale delle seguenti \xE8 necessariamente una caratteristica\ + \ dell'oligopolio?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_microeconomics +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_high_school_microeconomics +task_alias: high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..faf2cc2d8a38d8e35a47e88f3d94b7bf70b5e67d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_physics.yaml @@ -0,0 +1,62 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Solo I e II + B: Solo I e III + C: Solo II e III + D: Solo III + input_correct_responses: + - D + input_question: "Quale delle seguenti condizioni garantisce la conservazione del\ + \ momento angolare? I. Conservazione della quantit\xE0 di moto lineare II. Forza\ + \ esterna netta nulla III. Coppia esterna netta nulla" + - input_choice_list: + A: "La pressione \xE8 in un nodo, ma lo spostamento delle particelle \xE8 in\ + \ un antinodo." + B: "La pressione \xE8 all'antinodo, ma lo spostamento delle particelle \xE8\ + \ al nodo." + C: La pressione e lo spostamento delle particelle sono entrambi ai nodi. + D: La pressione e lo spostamento delle particelle sono entrambi agli antinodi. + input_correct_responses: + - B + input_question: "Un tubo pieno d'aria \xE8 chiuso ad un'estremit\xE0.\ + \ Nel tubo viene prodotta un'onda stazionaria, che fa suonare una nota.\ + \ Quale delle seguenti \xE8 un'affermazione corretta sulle propriet\xE0\ + \ dell'onda all'estremit\xE0 chiusa del tubo?" + - input_choice_list: + A: 02:00 + B: 6:00 DI MATTINA + C: 00:00 + D: 24A + input_correct_responses: + - D + input_question: "Una fotocellula con funzione di lavoro \u03D5 = 2eV \xE8 collegata\ + \ ad un resistore in serie. La luce di frequenza f = 1 \xD7 10^15 Hz colpisce\ + \ una piastra metallica della fotocellula. Se la potenza della luce \xE8 P =\ + \ 100 W, qual \xE8 la corrente che attraversa il resistore?" + - input_choice_list: + A: 10 W + B: 30 W + C: 60 W + D: 240 W + input_correct_responses: + - D + input_question: "Un forno a microonde \xE8 collegato a una presa da 120 V e assorbe\ + \ una corrente di 2 A. A quale velocit\xE0 viene utilizzata l'energia del\ + \ forno a microonde?" + - input_choice_list: + A: 3,5 J + B: 6,0 J + C: 22,5 J + D: 40 J + input_correct_responses: + - B + input_question: "Una carica puntiforme, Q = +1 mC, \xE8 fissata all'origine.\ + \ Quanto lavoro \xE8 necessario per spostare una carica, Q = +8 \xB5C, dal punto\ + \ (0,4 metri) al punto (3 metri, 0)?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_physics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_high_school_physics +task_alias: high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8e0c40c71146a7bfe83f7f9197b20c9385e8b5d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_psychology.yaml @@ -0,0 +1,63 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: un forte Super-Io. + B: bassa autostima. + C: bassa autoefficacia. + D: un locus of control interno. + input_correct_responses: + - D + input_question: "Ani crede che i suoi atteggiamenti e comportamenti giochino un\ + \ ruolo centrale in ci\xF2 che le accade. \xC8 probabile che tale convinzione\ + \ sia associata a" + - input_choice_list: + A: identificare le cause e le soluzioni dei problemi presentati dal cliente + B: "identificare ed eliminare le cause delle difficolt\xE0 del consultato nella\ + \ gestione di un problema" + C: "stabilire una gerarchia di autorit\xE0 per consentire un processo decisionale\ + \ efficace" + D: "presentare una linea d'azione unica, ben definita e inequivocabile affinch\xE9\ + \ il consulente possa superare i deficit di competenze" + input_correct_responses: + - B + input_question: "Secondo il modello di Caplan della consultazione del caso centrata\ + \ sul consultato, il consulente \xE8 principalmente interessato a" + - input_choice_list: + A: I messaggi vengono inviati dal talamo direttamente all'amigdala. + B: I messaggi vengono inviati dal talamo ai percorsi "cosa" e "dove". + C: I messaggi vengono inviati dal sistema nervoso parasimpatico alla corteccia + cerebrale. + D: I messaggi vengono inviati dai lobi frontali alla ghiandola pituitaria. + input_correct_responses: + - A + input_question: "Mentre nuota nell'oceano, Ivan viene spaventato da un'ombra\ + \ scura nell'acqua ancor prima che abbia la possibilit\xE0 di identificare\ + \ di cosa si tratta. Le connessioni sinaptiche che hanno luogo durante questo\ + \ episodio di paura sono meglio descritte da quale dei seguenti?" + - input_choice_list: + A: Date al bambino un periodo di prova nel nuovo ambiente + B: Avvisare per iscritto i genitori + C: Ottieni l'approvazione del consiglio scolastico + D: Ottieni il consenso dei genitori + input_correct_responses: + - B + input_question: "Secondo l\u2019Individuals with Disabilities Education Improvement\ + \ Act, quale delle seguenti azioni deve fare un\u2019agenzia educativa prima\ + \ di modificare l\u2019inserimento scolastico di uno studente con disabilit\xE0\ + ?" + - input_choice_list: + A: Socioculturale + B: clinico + C: cognitivo + D: comportamentista + input_correct_responses: + - C + input_question: "Pascale \xE8 interessata alle strategie di elaborazione che i\ + \ bambini utilizzano per apprendere nuove informazioni. In che tipo di psicologo\ + \ sarebbe meglio classificare Pascale?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_psychology +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_high_school_psychology +task_alias: high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c5b9cd048df68ec78531ea9916d9633ef2789da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_statistics.yaml @@ -0,0 +1,72 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Se la pendenza della retta di regressione \xE8 esattamente 1, allora la\ + \ correlazione \xE8 esattamente 1." + B: "Se la correlazione \xE8 0, la pendenza della retta di regressione non \xE8\ + \ definita." + C: Invertendo quale variabile si chiama x e quale si chiama y si cambia il segno + della correlazione. + D: "La correlazione r \xE8 uguale alla pendenza della linea di regressione quando\ + \ i punteggi z per la variabile y vengono tracciati rispetto ai punteggi z\ + \ per la variabile x." + input_correct_responses: + - D + input_question: "Quale delle seguenti \xE8 un'affermazione corretta sulla\ + \ correlazione?" + - input_choice_list: + A: E(X + Y) = 99, var(X + Y) = 8,5 + B: E(X + Y) = 99, var(X + Y) = 13 + C: E(X + Y) = 99, var(X + Y) = 17 + D: Non ci sono informazioni sufficienti per rispondere a questa domanda. + input_correct_responses: + - D + input_question: "Supponiamo che X e Y siano variabili casuali con E(X) = 37, var(X)\ + \ = 5, E(Y) = 62 e var(Y) = 12. Quali sono il valore atteso e la varianza della\ + \ variabile casuale X + S\xEC?" + - input_choice_list: + A: "La percentuale di alberi che hanno subito pi\xF9 del 50% di danni a causa\ + \ del gelo." + B: Il numero di alberi colpiti dal gelo. + C: Il numero di alberi campionati dal boschetto. + D: "Per ogni albero campionato, se ha subito pi\xF9 del 50% di danni o al massimo\ + \ il 50% di danni." + input_correct_responses: + - D + input_question: "Dopo l'allarme gelo, il proprietario di un grande aranceto\ + \ ha chiesto ai suoi lavoratori di spruzzare acqua su tutti i suoi alberi. Si\ + \ supponeva che l'acqua si congelasse e formasse uno strato protettivo di\ + \ ghiaccio attorno ai fiori d'arancio. Tuttavia il proprietario sospettava\ + \ che alcuni alberi avessero subito notevoli danni a causa del gelo. Per stimare\ + \ la percentuale di alberi che hanno subito pi\xF9 del 50% di danni a causa\ + \ del gelo, ha prelevato un campione casuale di 100 alberi dal suo boschetto.\ + \ Qual \xE8 la variabile di risposta in questo esperimento?" + - input_choice_list: + A: Media 518 grammi; deviazione standard 7,0 grammi + B: Media 518 grammi; deviazione standard 3,5 grammi + C: Media 518 grammi; deviazione standard 6,1 grammi + D: Media 394 grammi; deviazione standard 6,1 grammi + input_correct_responses: + - C + input_question: 'Un nuovo smartwatch viene prodotto in una parte della fabbrica, + quindi assicurato per la spedizione in un'altra parte indipendente della + fabbrica. Il peso dello smartwatch ha una media di 62 grammi e una deviazione + standard di 1,0 grammi. Il peso dell'imballaggio (scatola, manuale d'uso, + pluriball, ecc.) ha una media di 456 grammi e una deviazione standard di 6 grammi. + Insieme, la distribuzione del peso dello smartwatch e della sua confezione avrebbe + la seguente media e deviazione standard:' + - input_choice_list: + A: Io, II + B: II, III + C: III, I + D: III, II + input_correct_responses: + - D + input_question: "Quale dei seguenti insiemi ha la deviazione standard pi\xF9 piccola?\ + \ Quale ha il pi\xF9 grande? I: {1,2,3} II: {-10,10} III: {100}\\n" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_statistics +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_high_school_statistics +task_alias: high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9218fe901b4b55ac9fd2c7fdae25ccddbecbf72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_us_history.yaml @@ -0,0 +1,163 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Risposta organizzata alla ribellione di Bacon + B: Risposta federale alla ribellione di Shays + C: Risposta federale alla ribellione del whisky + D: Risposta federale alla ribellione di Pontiac + input_correct_responses: + - C + input_question: "Questa domanda si riferisce alle seguenti informazioni. "La\ + \ societ\xE0 in ogni stato \xE8 una benedizione, ma il governo, anche nel suo\ + \ stato migliore, non \xE8 altro che un male necessario; nel suo stato peggiore,\ + \ un male intollerabile; perch\xE9 quando soffriamo, o siamo esposti alle stesse\ + \ miserie da parte di un governo, cosa che potremmo aspettarci In un paese senza\ + \ governo, la nostra calamit\xE0 \xE8 accresciuta se riflettiamo sul fatto che\ + \ siamo noi a fornire i mezzi con cui soffriamo. Il governo, come l\u2019abito,\ + \ \xE8 il distintivo dell\u2019innocenza perduta; i palazzi dei re sono costruiti\ + \ sulle rovine delle pergole del paradiso. se gli impulsi della coscienza fossero\ + \ chiari, uniformi e irresistibilmente obbediti, l\u2019uomo non avrebbe bisogno\ + \ di altro legislatore; ma non essendo cos\xEC, egli ritiene necessario cedere\ + \ una parte della sua propriet\xE0 per fornire i mezzi per la protezione del\ + \ resto; e a ci\xF2 \xE8 indotto dalla stessa prudenza che in ogni altro caso\ + \ gli consiglia di scegliere il minimo tra due mali. Pertanto, essendo la sicurezza\ + \ il vero disegno e fine del governo, ne consegue indiscutibilmente che qualunque\ + \ forma di essa appaia pi\xF9 idonea a garantire per noi, con la minima spesa\ + \ e il massimo beneficio, \xE8 preferibile a tutti gli altri." Thomas Paine,\ + \ Common Sense, 1776 Quale delle seguenti "miserie" a cui si \xE8\ + \ accennato sopra \xE8 stata maggiormente condannata dagli antifederalisti dell'era\ + \ post-rivoluzionaria?" + - input_choice_list: + A: Tensioni tra le politiche britanniche e le aspirazioni dei coloni nordamericani. + B: Tensioni tra gli indiani d'America alleati con i francesi e quelli alleati + con gli inglesi. + C: Tensioni tra afroamericani liberati e piantatori bianchi. + D: "Tensioni tra coloni dell'entroterra ed \xE9lite nell'America coloniale." + input_correct_responses: + - D + input_question: "Questa domanda si riferisce alle seguenti informazioni. "Poich\xE9\ + \ la nostra ultima condotta a Conestoga Manor e a Lancaster ha dato luogo a\ + \ molte speculazioni e a una grande diversit\xE0 di sentimenti in questo e nei\ + \ governi vicini; alcuni lo hanno rivendicato e altri lo hanno condannato; alcuni\ + \ hanno caritatevolmente alleviato il Crimine, e altri maliziosamente lo dipingono\ + \ nel modo pi\xF9 odioso e colori detestabili, riteniamo nostro dovere esporre\ + \ al pubblico l'intera questione come ci \xE8 apparsa, e ci appare ancora...\ + \ "Se queste cose non bastano a dimostrare un ingiustificabile attaccamento\ + \ dei quaccheri agli indiani selvaggi , una ferma risoluzione di fare amicizia\ + \ con loro e un'assoluta insensibilit\xE0 alle sofferenze umane, consideriamo\ + \ alcuni fatti pi\xF9 recenti. Quando l'estate scorsa abbiamo scoperto che\ + \ probabilmente non avremmo ricevuto assistenza dal governo, alcuni volontari\ + \ sono partiti a nostre spese, determinati a scacciare i nostri nemici dai nostri\ + \ confini; e quando ci avvicinammo alla grande Isola, capimmo che un certo numero\ + \ dei loro Guerrieri erano usciti contro le nostre Frontiere. Dopodich\xE9 siamo\ + \ tornati, li abbiamo raggiunti e abbiamo combattuto con loro a Munfey Hill,\ + \ dove abbiamo perso alcuni dei nostri uomini e ucciso alcuni dei loro guerrieri,\ + \ salvando cos\xEC le nostre frontiere da questa storia in un'altra spedizione.\ + \ Ma non appena avevamo distrutto le loro provviste sulla grande isola e rovinato\ + \ il loro commercio con la brava gente di Betlemme, questi stessi indiani, che\ + \ erano giustamente sospettati di aver ucciso i nostri amici nella contea di\ + \ Northampton, furono presi dall'influenza di alcuni quaccheri. sotto la\ + \ protezione del governo per proteggerli dai risentimenti degli amici e dei\ + \ parenti degli assassinati e per sostenerli durante l'inverno." \u2014\ + "Apology of the Paxton Boys" (opuscolo), 1764 (Nota: "apology"\ + \ in questo contesto dovrebbe essere letto come una spiegazione, non come un'ammissione\ + \ di colpa o di rammarico.) I sentimenti espressi nella spiegazione di cui sopra\ + \ riflettono quali delle tensioni in corso durante il periodo coloniale della\ + \ storia americana?" + - input_choice_list: + A: "l\u2019emendamento sulla parit\xE0 di diritti" + B: suffragio universale + C: diritti degli Stati + D: divieto + input_correct_responses: + - B + input_question: "Questa domanda si riferisce alle seguenti informazioni. "Nel\ + \ nuovo Codice di Leggi che suppongo sar\xE0 necessario che tu faccia, desidero\ + \ che ti ricordi delle Signore, e che tu sia pi\xF9 generoso e favorevole verso\ + \ loro rispetto ai tuoi antenati. Non mettere un potere cos\xEC illimitato nelle\ + \ mani dei Mariti . Ricorda che tutti gli uomini sarebbero tiranni se potessero.\ + \ Se non viene prestata particolare cura e attenzione alle donne, siamo determinati\ + \ a fomentare una ribellione e non ci riterremo vincolati da alcuna legge in\ + \ cui non abbiamo voce o rappresentanza. Abigail Adams, in una lettera a John\ + \ Adams, 1776 "La legislazione speciale per le donne ci ha posto in una\ + \ posizione molto anomala. Le donne investite dei diritti di cittadinanza in\ + \ una sezione - elettori, giurati, titolari di cariche - che attraversano una\ + \ linea immaginaria, sono soggetti nei successivi. In alcuni Stati, una donna\ + \ sposata pu\xF2 detenere propriet\xE0 e trattare affari a proprio nome; in\ + \ altri, i suoi guadagni appartengono a suo marito. In alcuni Stati, una donna\ + \ pu\xF2 testimoniare contro suo marito, fare causa ed essere citata in giudizio\ + \ nei tribunali; in altri, non ha alcun risarcimento in caso di danno alla persona,\ + \ alla propriet\xE0 o al carattere. In caso di divorzio per adulterio del marito,\ + \ la moglie innocente non \xE8 ritenuta titolare di alcun diritto sui figli\ + \ o sulla propriet\xE0, a meno che per decreto speciale del tribunale. Ma in\ + \ nessuno Stato dell'Unione la moglie ha diritto sulla propria persona,\ + \ o su una parte dei guadagni comuni della convivenza durante la vita del marito.\ + \ In alcuni Stati le donne possono entrare nel scuole di diritto e pratica nei\ + \ tribunali; in altri sono vietati. In alcune universit\xE0 le ragazze godono\ + \ di pari vantaggi educativi rispetto ai ragazzi, mentre molte delle istituzioni\ + \ pi\xF9 orgogliose del paese negano loro l\u2019ammissione, sebbene i figli\ + \ della Cina, del Giappone e dell\u2019Africa vi siano i benvenuti. Ma i privilegi\ + \ gi\xE0 concessi nei vari Stati non sono affatto sicuri." Susan B. Anthony,\ + \ "Dichiarazione dei diritti delle donne", 4 luglio 1876 I sentimenti\ + \ espressi nel secondo estratto da Susan B. Anthony sono molto probabilmente\ + \ a sostegno Di" + - input_choice_list: + A: Gli americani devono massimizzare il loro vantaggio tecnologico in Vietnam. + B: I bombardamenti americani in Vietnam stanno portando passo dopo passo al + progresso della guerra. + C: "Il bombardamento americano in Vietnam \xE8 un fallimento." + D: "L\u2019America non deve cedere al disfattismo riguardo alla guerra in Vietnam." + input_correct_responses: + - C + input_question: "Questa domanda si riferisce alle seguenti informazioni. I nostri\ + \ leader parlano di fermare l'aggressione dal nord, ma questa \xE8 stata\ + \ una lotta tra gruppi di vietnamiti finch\xE9 non siamo intervenuti. Sembriamo\ + \ decisi a salvare i vietnamiti da Ho Chi Minh, anche a costo di ucciderli e\ + \ demolire il loro paese per farlo. Mentre i nativi osservano i villaggi bombardati,\ + \ le donne e i bambini bruciati dal napalm, i raccolti di riso distrutti e le\ + \ citt\xE0 invase dal nostro personale militare, senza dubbio dicono segretamente\ + \ della guerriglia vietcong e delle forze americane: "Una piaga su entrambe\ + \ le vostre case ." \u2026 Fermare i bombardamenti, a nord e a sud, porre\ + \ fine alle ricerche e alla distruzione delle operazioni offensive e limitare\ + \ la nostra azione militare allo svolgimento di operazioni sul terreno. I bombardamenti\ + \ sul nord non sono riusciti a fermare o a frenare seriamente il flusso di truppe\ + \ verso sud e potrebbero, di fatto, aver indotto Hanoi ad uno sforzo bellico\ + \ molto maggiore. \u2014Il senatore George McGovern, "The Lessons of Vietnam",\ + \ 25 aprile 1967 Quale delle seguenti opinioni degli anni '60 riflette pi\xF9\ + \ direttamente la prospettiva del discorso di George McGovern?" + - input_choice_list: + A: Abigail Adams + B: Clara Barton + C: Shirley Temple + D: Hillary Clinton + input_correct_responses: + - B + input_question: "Questa domanda si riferisce alle seguenti informazioni. Non vengo\ + \ per sollecitare pretese personali, n\xE9 per cercare vantaggi individuali;\ + \ Appaio come l'avvocato di coloro che non possono difendere la propria\ + \ causa; Vengo come amico di coloro che sono abbandonati, oppressi e desolati.\ + \ Nella Provvidenza di Dio, io sono la voce del maniaco le cui grida penetranti\ + \ dalle tetre segrete delle vostre prigioni non penetrano nelle vostre Aule\ + \ della Legislazione. Io sono la Speranza dei poveri esseri impazziti che si\ + \ struggono nelle celle, nei box, nelle gabbie e nelle stanze desolate delle\ + \ vostre povere case. Io sono la Rivelazione di centinaia di creature lamentose\ + \ e sofferenti, nascoste nelle vostre dimore private, in recinti e cabine, chiuse\ + \ fuori, tagliate fuori da ogni influenza curativa, da ogni cura che risana\ + \ la mente... Le loro storie malinconiche potrebbero essere diffuse davanti\ + \ a voi? come rivelato al mio spirito addolorato durante gli ultimi tre mesi,\ + \ con quanta rapidit\xE0 e seriet\xE0 cercheresti i mezzi di sollievo pi\xF9\ + \ approvati; quanto insignificanti, quanto insignificanti, al confronto, sembrerebbero\ + \ i sacrifici che ti vengono richiesti; come potrebbero diminuire di valore\ + \ pochi centesimi e dollari, raccolti da ciascun cittadino, in confronto ai\ + \ benefici certi e all'immenso bene che sarebbero assicurati ai pazzi sofferenti...\ + \ mediante la consacrazione e l'applicazione di un fondo sufficiente alla\ + \ costruzione? di un ospedale adatto... \u2014Dorothea Dix, Memorial Soliciting\ + \ a State Hospital for the Protection and Cure of the Insane, presentato all'Assemblea\ + \ Generale della Carolina del Nord, novembre 1848 Dorothea Dix pu\xF2 essere\ + \ meglio paragonata a chi?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_us_history +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_high_school_us_history +task_alias: high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a243b7f0066223f9aa329342df5d91e168ca207f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_high_school_world_history.yaml @@ -0,0 +1,104 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: induismo + B: buddismo + C: Shintoismo + D: Zoroastrismo + input_correct_responses: + - A + input_question: "Questa domanda si riferisce alle seguenti informazioni. Contiene\ + \ tutte le opere e i desideri e tutti i profumi e tutti i gusti. Avvolge l'universo\ + \ intero e nel silenzio \xE8 amorevole con tutti. Questo \xE8 lo Spirito che\ + \ \xE8 nel mio cuore, questo \xE8 Brahman. A lui verr\xF2 quando andr\xF2 oltre\ + \ questa vita, e a lui verr\xE0 colui che ha fede e non dubita. \u2014Le Upanishad,\ + \ India, ca. 1000 a.C. A quale religione appartiene molto probabilmente chi\ + \ parla?" + - input_choice_list: + A: La rivoluzione industriale + B: Decolonizzazione + C: Associazioni regionali di libero scambio + D: Autarchia + input_correct_responses: + - B + input_question: "Questa domanda si riferisce alle seguenti informazioni. "La\ + \ lotta contro il neocolonialismo non mira ad escludere il capitale del mondo\ + \ sviluppato dall'operare nei paesi meno sviluppati. Essa mira a impedire\ + \ che il potere finanziario dei paesi sviluppati venga utilizzato in modo tale\ + \ da impoverire i meno sviluppati. Il non allineamento, come praticato dal Ghana\ + \ e da molti altri paesi, si basa sulla cooperazione con tutti gli Stati, siano\ + \ essi capitalisti, socialisti o ad economia mista. Tale politica, quindi, implica\ + \ investimenti esteri da parte dei paesi capitalisti, ma deve essere investito\ + \ secondo un piano nazionale elaborato dal governo dello Stato non allineato\ + \ tenendo conto dei propri interessi. La questione non \xE8 quale rendimento\ + \ ottiene l'investitore straniero dai suoi investimenti... La questione\ + \ \xE8 di potere. Uno Stato in la morsa del neocolonialismo non \xE8 padrona\ + \ del proprio destino." Kwame Nkrumah, Neo-Colonialism, 1965 Quale dei\ + \ seguenti fornisce il contesto migliore per gli scritti di Nkrumah?" + - input_choice_list: + A: Accettazione sociale del lavoro minorile + B: Aspettativa di vita in calo in Germania + C: Critiche ai dazi commerciali tedeschi + D: Effetti negativi attribuiti al capitalismo industriale + input_correct_responses: + - D + input_question: "Questa domanda si riferisce alle seguenti informazioni. "La\ + \ vera lamentela dell'operaio \xE8 l'insicurezza della sua esistenza;\ + \ non \xE8 sicuro che avr\xE0 sempre lavoro, non \xE8 sicuro che sar\xE0 sempre\ + \ sano, e prevede che un giorno sar\xE0 vecchio e inabile al lavoro Se cade\ + \ in povert\xE0, anche solo a causa di una malattia prolungata, \xE8 del tutto\ + \ indifeso, messo alla prova da solo, e la societ\xE0 attualmente non riconosce\ + \ alcun obbligo reale nei suoi confronti oltre al consueto aiuto ai poveri,\ + \ anche se ha "Ho sempre lavorato fedelmente e diligentemente. Ma il solito\ + \ aiuto ai poveri lascia molto a desiderare, soprattutto nelle grandi citt\xE0\ + , dove \xE8 molto peggio che in campagna." Otto von Bismarck, 1884 Otto\ + \ von Bismarck probabilmente fece questo discorso in reazione a quale dei seguenti\ + \ problemi?" + - input_choice_list: + A: Il mantenimento della supremazia militare a tutti i costi + B: Crescenti tensioni tra sette religiose + C: "Fattori che portarono al crollo dell\u2019Impero Ottomano" + D: Sforzi di pace tra gli imperi islamici + input_correct_responses: + - B + input_question: "Questa domanda si riferisce alle seguenti informazioni. "In\ + \ effetti, poich\xE9 sia le fatwa di illustri [studiosi] che basano la loro\ + \ opinione sia sulla ragione che sulla tradizione, sia il consenso della comunit\xE0\ + \ sunnita concordano sul fatto che l'antico obbligo di estirpazione, sterminio\ + \ ed espulsione del male innovativo deve essere lo scopo del nostro esaltato\ + \ aspirazione, poich\xE9 "Lo zelo religioso \xE8 una vittoria della Fede\ + \ di Dio Benefico"; poi, secondo le parole del Profeta (Pace su di lui!)\ + \ "Chiunque introdurr\xE0 cattive innovazioni nel nostro ordine sia espulso"\ + \ e "Chiunque non qualsiasi cosa contraria al nostro ordine deve essere\ + \ espulsa, "l'azione \xE8 diventata necessaria ed urgente..."\ + \ Lettera del sultano ottomano Selim I allo sci\xE0 safavide Ismail I, 1514\ + \ La lettera di Selim I \xE8 pi\xF9 chiaramente un esempio di quale dei seguenti?" + - input_choice_list: + A: Una rottura delle rotte commerciali attraverso il crollo della struttura + statale stabilita + B: Un aumento della popolazione mondiale attraverso una maggiore abbondanza + di cibo + C: La diffusione dei sistemi di credenze cinesi e indiani nel mondo + D: Un aumento dei disordini sociali + input_correct_responses: + - B + input_question: "Questa domanda si riferisce alle seguenti informazioni. "Almeno\ + \ una delle societ\xE0 [del mondo] dovrebbe in qualche modo aumentare enormemente\ + \ la propria produttivit\xE0 [al fine di raggiungere l'egemonia globale].\ + \ Questo salto quantico dovrebbe essere fatto prima delle varie rivoluzioni\ + \ scientifiche, tecnologiche, agricole e industriali su cui si basa la nostra\ + \ Il mondo dopo il salto quantico resta. Ci\xF2 potrebbe essere realizzato solo\ + \ sfruttando gli ecosistemi, le risorse minerarie e le risorse umane di interi\ + \ continenti al di fuori dei territori della societ\xE0 che ha compiuto il salto.\ + \ L\u2019Europa occidentale ha fatto proprio questo attraverso la sua brutalit\xE0\ + , le armi e le armi. , cosa pi\xF9 importante, per fortuna geografica ed ecologica."\ + \ Copyright \xA9 2015Cambridge University Press. Alfred Crosby, storico, Ecological\ + \ Imperialism, 2004 Il "salto quantico" menzionato nel passaggio ha\ + \ contribuito pi\xF9 direttamente a quale dei seguenti sviluppi nel periodo\ + \ 1450-1750 d.C.?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_world_history +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_high_school_world_history +task_alias: high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f8d4d6f637892a8a39bacab9482b905ec8a5d65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_human_aging.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Un uomo o una donna asiatica + B: Un uomo ispanico + C: Una donna afroamericana + D: Un uomo o una donna bianca + input_correct_responses: + - C + input_question: "Quale delle seguenti persone ha maggiori probabilit\xE0 di rimanere\ + \ a casa da sola, a partire dal 2019?" + - input_choice_list: + A: Vantaggio dell'adolescenza + B: Urto della reminiscenza + C: Memoriale commemorativo + D: Picco di recupero quadratico + input_correct_responses: + - B + input_question: "La scoperta che gli adulti tendono a ricordare gli eventi della\ + \ loro adolescenza meglio che quelli di altri periodi della loro vita viene\ + \ definita \u201C" + - input_choice_list: + A: Texas + B: California + C: Hawaii + D: Vermont + input_correct_responses: + - A + input_question: "Quando gli anziani si trasferiscono in un nuovo stato dopo il\ + \ pensionamento, quale delle seguenti \xE8 la destinazione pi\xF9 probabile?" + - input_choice_list: + A: Nicotina + B: Catrame + C: Monossido di carbonio + D: Particelle di fumo + input_correct_responses: + - B + input_question: "Quale elemento nel fumo di tabacco \xE8 responsabile del cancro?" + - input_choice_list: + A: "Una donna ispanoamericana pi\xF9 anziana" + B: Una donna afroamericana anziana + C: "Una donna asiatica americana pi\xF9 anziana" + D: "Una donna nativa americana pi\xF9 anziana" + input_correct_responses: + - C + input_question: "A parit\xE0 di condizioni, quale delle seguenti persone ha maggiori\ + \ probabilit\xE0 di soffrire di osteoporosi?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_human_aging +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_human_aging +task_alias: human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e473707f69e8d2101586911cd85582f52f709f2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_human_sexuality.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: durante il primo trimestre + B: durante il secondo trimestre + C: durante il terzo trimestre + D: per tutta la gravidanza + input_correct_responses: + - A + input_question: "La nausea mattutina \xE8 in genere un problema:" + - input_choice_list: + A: un ego forte + B: un Super-Io debole + C: "un'identit\xE0 debole" + D: un forte Super-Io + input_correct_responses: + - B + input_question: 'Una donna che sa di avere un herpes attivo e una sifilide non + trattata ma continua a fare sesso senza informare i suoi partner della sua condizione + ha, in termini psicoanalitici:' + - input_choice_list: + A: il fatto che non hanno un periodo refrattario. + B: la risposta degli strati interni della vagina. + C: avere orgasmi alternati in luoghi diversi. + D: il punto G. + input_correct_responses: + - A + input_question: "La capacit\xE0 delle donne di avere orgasmi multipli \xE8 dovuta\ + \ principalmente a:" + - input_choice_list: + A: rapporto sessuale + B: il cerchio sussulta + C: esibizionismo + D: toccarsi i genitali a vicenda + input_correct_responses: + - A + input_question: "La natura delle attivit\xE0 omosessuali che si verificano durante\ + \ la preadolescenza include tutte le seguenti, tranne quale?" + - input_choice_list: + A: eiaculazione precoce + B: eiaculazione inibita + C: disturbo erettile + D: disturbo eiaculatorio + input_correct_responses: + - C + input_question: "Il disturbo pi\xF9 comune tra gli uomini che cercano una terapia\ + \ sessuale \xE8:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_human_sexuality +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_human_sexuality +task_alias: human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f5175eaf24b3d137df58a677b6014e612235210 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_international_law.yaml @@ -0,0 +1,71 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "L\u2019articolo 2, paragrafo 4 riguarda solo la forza armata" + B: "L\u2019articolo 2, paragrafo 4, comprende tutti i tipi di forza, comprese\ + \ le sanzioni" + C: "L\u2019articolo 2, paragrafo 4, comprende tutte le ingerenze negli affari\ + \ interni degli Stati" + D: "L'articolo 2, paragrafo 4, comprende la forza diretta esclusivamente\ + \ contro l'integrit\xE0 territoriale di uno Stato" + input_correct_responses: + - A + input_question: "Quali tipi di forza vietano l\u2019articolo 2, paragrafo 4, della\ + \ Carta delle Nazioni Unite?" + - input_choice_list: + A: Se una parte in un caso controverso dinanzi all'ICJ non ha una seduta + nazionale come giudice, ha il diritto di nominare qualcuno come giudice esclusivamente + per quel caso, con il titolo di giudice ad hoc + B: "Il giudice ad hoc \xE8 il membro del collegio giudicante della Corte internazionale\ + \ di giustizia con voto decisivo" + C: "Il giudice ad hoc \xE8 un giudice surrogato, nel caso in cui un giudice\ + \ venga squalificato o muoia" + D: "Il giudice ad hoc \xE8 il giudice che ciascuna parte nominer\xE0 sempre\ + \ in ogni causa controversa" + input_correct_responses: + - A + input_question: "Qual \xE8 il giudice ad hoc?" + - input_choice_list: + A: "Questa \xE8 una riserva accettabile se la legislazione del paese che effettua\ + \ la riserva utilizza una definizione diversa" + B: "Si tratta di una riserva inaccettabile poich\xE9 contravviene all\u2019\ + oggetto e allo scopo dell\u2019ICCPR" + C: "Si tratta di una riserva inaccettabile poich\xE9 la definizione di tortura\ + \ contenuta nell\u2019ICCPR \xE8 coerente con il diritto internazionale consuetudinario" + D: "Questa \xE8 una riserva accettabile perch\xE9 secondo il diritto internazionale\ + \ generale gli Stati hanno il diritto di apporre riserve ai trattati" + input_correct_responses: + - B + input_question: "Una riserva alla definizione di tortura contenuta nell\u2019\ + ICCPR sarebbe accettabile nella pratica contemporanea?" + - input_choice_list: + A: "Il consenso pu\xF2 servire come circostanza escludente l'illegittimit\xE0\ + \ ogni volta che viene prestato" + B: "Il consenso non pu\xF2 mai costituire una circostanza ostativa all\u2019\ + illegittimit\xE0" + C: "Il consenso pu\xF2 costituire una circostanza escludente l'illegittimit\xE0\ + , purch\xE9 il consenso sia valido e nella misura in cui la condotta rimanga\ + \ nei limiti del consenso prestato" + D: "Il consenso pu\xF2 sempre costituire una circostanza ostativa all'illegittimit\xE0\ + , qualunque sia l'organo dello Stato che lo presta" + input_correct_responses: + - C + input_question: "Quando il \u201Cconsenso\u201D pu\xF2 fungere da circostanza\ + \ ostativa all'illegittimit\xE0 di una condotta dello Stato?" + - input_choice_list: + A: Il consenso di uno Stato ad essere vincolato si esprime solo mediante ratifica + B: "Il consenso di uno Stato ad essere vincolato da un trattato pu\xF2 essere\ + \ espresso mediante firma, ratifica, accettazione, approvazione o adesione" + C: Il consenso di uno Stato ad essere vincolato si esprime mediante la firma + D: "Il consenso di uno Stato ad essere vincolato \xE8 espresso con qualunque\ + \ mezzo scelga" + input_correct_responses: + - B + input_question: "Come pu\xF2 essere espresso il consenso di uno Stato ad essere\ + \ vincolato?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_international_law +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_international_law +task_alias: international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01bf4a23857c78861be7b1a4b7a057987092da34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_jurisprudence.yaml @@ -0,0 +1,54 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Il POP sceglierebbe l\u2019uguaglianza al di sopra della libert\xE0." + B: "Il POP opterebbe per la strategia della \u201Cmassimizzazione\u201D." + C: "Il POP opterebbe per il \u201Cprincipio di differenza\u201D." + D: "Il POP rifiuterebbe il \u201Csistema della libert\xE0 naturale\u201D." + input_correct_responses: + - A + input_question: "Quale posizione, secondo Rawls, avr\xE0 meno probabilit\xE0 di\ + \ essere adottata dal POP (persone nella posizione originaria)?" + - input_choice_list: + A: "massimizzazione della libert\xE0 individuale" + B: fornendo una base per un compromesso + C: mantenendo la pace + D: promuovere i principi del sistema della libera impresa + input_correct_responses: + - D + input_question: Le funzioni della legge includono tutte, tranne quale delle seguenti? + - input_choice_list: + A: "Autorit\xE0." + B: Carisma. + C: Cooperazione. + D: Capitalismo. + input_correct_responses: + - D + input_question: Quale parola riassume meglio la spiegazione di Weber dello sviluppo + del diritto formalmente razionale? + - input_choice_list: + A: Legge naturale + B: Analitico + C: Storico + D: Sociologico + input_correct_responses: + - A + input_question: "La ________ Scuola di giurisprudenza postula che la legge si\ + \ basa su ci\xF2 che \xE8 "corretto"." + - input_choice_list: + A: "La lettera \xE8 un'offerta di vendita" + B: "Un'offerta valida non pu\xF2 essere fatta per lettera." + C: "La lettera contiene un'offerta valida che terminer\xE0 entro un termine\ + \ ragionevole." + D: Nella lettera manca uno degli elementi essenziali di un'offerta. + input_correct_responses: + - D + input_question: 'Iverson Jewellers scrisse una lettera a Miller: "Abbiamo + ricevuto un orologio Rolox a carica automatica eccezionalmente raffinato che + ti venderemo a un prezzo molto favorevole".' +include: _continuation_template_yaml +process_docs: !function utils.process_docs_jurisprudence +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_jurisprudence +task_alias: jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ac923ff8fe00784fe0e0fa1b0870ef4f666eb3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_logical_fallacies.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: tu quoque + B: risata del cavallo + C: argomento contro la persona + D: ignoratio elenchi + input_correct_responses: + - C + input_question: Se qualcuno attacca il carattere dell'avversario, invece di + rispondere alle argomentazioni di quell'avversario, la prima persona ha + probabilmente commesso quale dei seguenti errori? + - input_choice_list: + A: "sostenere che qualcosa \xE8 inferiore solo perch\xE9 non fa qualcosa che\ + \ non avrebbe mai dovuto fare." + B: "includere pi\xF9 di una affermazione nella proposizione e trattare la prova\ + \ di una affermazione come prova di tutte le affermazioni." + C: trarre una conclusione prima di esaminare le prove e considerare solo le + prove che supportano tale conclusione. + D: "porre una domanda che includa un presupposto non dimostrato o pi\xF9 di\ + \ una domanda, rendendo cos\xEC priva di significato una risposta semplice\ + \ s\xEC o no." + input_correct_responses: + - D + input_question: La complessa fallacia della domanda consiste in + - input_choice_list: + A: La premessa minore deve negare l'antecedente + B: La premessa maggiore deve affermare il conseguente + C: Il termine medio deve essere utilizzato in almeno una premessa in senso universale + o assoluto + D: Tutti i precedenti + input_correct_responses: + - C + input_question: "Quale delle seguenti affermazioni \xE8 vera per un sillogismo\ + \ categorico valido?" + - input_choice_list: + A: Divisione + B: Composizione + C: Appello alla persona + D: Appello all'ignoranza + input_correct_responses: + - B + input_question: "Sostenere che ci\xF2 che \xE8 vero per le parti deve essere vero\ + \ per il tutto \xE8 un errore..." + - input_choice_list: + A: "scarsa sportivit\xE0" + B: appello alla compassione + C: argomento contro la persona + D: ignoranza della confutazione + input_correct_responses: + - D + input_question: "Quando un argomentatore causa confusione durante la confutazione\ + \ a causa della reale o finta mancanza di capacit\xE0 di impegnarsi nella confutazione,\ + \ quell'argomante potrebbe aver commesso l'errore di" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_logical_fallacies +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_logical_fallacies +task_alias: logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..512a896ebeb09f2cad3ffe7a8b8f454866934564 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_machine_learning.yaml @@ -0,0 +1,74 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: 2.0/15 + B: 1.0/7 + C: 3.0/16 + D: 1.0/5 + input_correct_responses: + - B + input_question: "Un dado a 6 facce viene lanciato 15 volte e i risultati sono:\ + \ il lato 1 esce 0 volte; lato 2: 1 volta; lato 3: 2 volte; lato 4: 3 volte;\ + \ lato 5: 4 volte; lato 6: 5 volte. Sulla base di questi risultati, qual \xE8\ + \ la probabilit\xE0 che venga visualizzato il lato 3 quando si utilizza l'arrotondamento\ + \ Add-1?" + - input_choice_list: + A: ritaglio casuale e rotazione orizzontale + B: ritaglio casuale e rotazione verticale + C: posterizzazione + D: dithering + input_correct_responses: + - A + input_question: "Quale aumento dei dati immagine \xE8 pi\xF9 comune per le immagini\ + \ naturali?" + - input_choice_list: + A: Il mio metodo raggiunge un errore di addestramento inferiore a tutti i metodi + precedenti! + B: "Il mio metodo raggiunge un errore di test inferiore a tutti i metodi precedenti!\ + \ (Nota a pi\xE8 di pagina: quando il parametro di regolarizzazione \u03BB\ + \ viene scelto in modo da ridurre al minimo l'errore del test.)" + C: "Il mio metodo raggiunge un errore di test inferiore a tutti i metodi precedenti!\ + \ (Nota a pi\xE8 di pagina: quando il parametro di regolarizzazione \u03BB\ + \ viene scelto in modo da ridurre al minimo l'errore di convalida incrociata.)" + D: "Il mio metodo raggiunge un errore di convalida incrociata inferiore a tutti\ + \ i metodi precedenti! (Nota a pi\xE8 di pagina: quando il parametro di regolarizzazione\ + \ \u03BB viene scelto in modo da ridurre al minimo l'errore di convalida\ + \ incrociata.)" + input_correct_responses: + - C + input_question: "Stai esaminando i documenti per la conferenza sull'apprendimento\ + \ automatico pi\xF9 fantastica del mondo e vedi i contributi con le seguenti\ + \ affermazioni. Quali considereresti di accettare? " + - input_choice_list: + A: circa 10 esempi + B: circa 100 esempi + C: tra 100 e 500 esemplari + D: "pi\xF9 di 1000 esempi" + input_correct_responses: + - D + input_question: "Per ottenere una stima della perdita 0/1 che sia inferiore all'1%\ + \ della vera perdita 0/1 (con probabilit\xE0 del 95%), secondo la disuguaglianza\ + \ di Hoeffding, quanti esempi deve avere il set di test IID?" + - input_choice_list: + A: "\xC8 troppo costoso dal punto di vista computazionale." + B: Probabilmente il risultato sarebbe un albero decisionale con un punteggio + scarso sul set di addestramento e su un set di test. + C: Probabilmente il risultato sarebbe un albero decisionale che ottiene un buon + punteggio sul set di addestramento ma male su un set di test. + D: Probabilmente il risultato sarebbe un albero decisionale che ottiene un buon + punteggio su un set di test ma male su un set di addestramento. + input_correct_responses: + - C + input_question: "Tradizionalmente, quando abbiamo un attributo di input con valore\ + \ reale durante l'apprendimento dell'albero decisionale, consideriamo\ + \ una suddivisione binaria a seconda che l'attributo sia al di sopra o al\ + \ di sotto di una certa soglia. Pat suggerisce che invece dovremmo avere semplicemente\ + \ una suddivisione a pi\xF9 vie con un ramo per ciascuno dei valori distinti\ + \ dell'attributo. Scegli dall'elenco seguente il problema pi\xF9 grande\ + \ con il suggerimento di Pat:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_machine_learning +tag: mmlu_it_llama_stem_tasks +task: mmlu_it_llama_machine_learning +task_alias: machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_management.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89b567842c043247448ee838c29c90d773f37136 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_management.yaml @@ -0,0 +1,51 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Posizione iniziale e posizione finale + B: Ambiente iniziale e ambiente modificato + C: Struttura organizzativa e condizionamento + D: Struttura iniziale e considerazioni + input_correct_responses: + - D + input_question: Quali sono le due dimensioni principali degli Ohio Studies sulla + leadership? + - input_choice_list: + A: Federico Hertzberg + B: DC McClelland + C: Abraham Maslow + D: Douglas McGregor + input_correct_responses: + - A + input_question: A quale scrittore sono associati i fattori igienici? + - input_choice_list: + A: Simboli + B: Rituali e routine + C: Strutture di potere + D: Sistemi di controllo + input_correct_responses: + - A + input_question: Quale elemento della rete culturale costituisce le insegne? + - input_choice_list: + A: Morale + B: Innovazione + C: Risorsa di crescita + D: Adattamento + input_correct_responses: + - A + input_question: "Quale caratteristica non \xE8 una caratteristica chiave del modello\ + \ di gestione dei "sistemi aperti"?" + - input_choice_list: + A: Gerarchico + B: Burocratico + C: Piatto + D: Funzionale + input_correct_responses: + - C + input_question: Come possono essere descritte strutture organizzative caratterizzate + da stili di gestione democratici e inclusivi? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_management +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_management +task_alias: management diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f1475fe8d0833dadcb2a0764102f9c211d16723 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_marketing.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Geodemografica + B: Differenziazione del prodotto. + C: Matrice ANSOFF. + D: Gestione del marchio. + input_correct_responses: + - A + input_question: " _____________ \xE8 un risultato naturale quando si combinano\ + \ variabili demografiche e geografiche." + - input_choice_list: + A: "Unit\xE0 di outsourcing." + B: Centro acquisti. + C: "Unit\xE0 esecutiva principale." + D: "Unit\xE0 decisionale." + input_correct_responses: + - D + input_question: In un'organizzazione, il gruppo di persone incaricate delle + decisioni di acquisto viene definito _______________. + - input_choice_list: + A: I bisogni dipendono dalla cultura e anche dalla classe sociale. + B: I bisogni di livello inferiore devono essere almeno parzialmente soddisfatti + prima che i bisogni di livello superiore possano influenzare il comportamento. + C: I bisogni non sono prioritari o organizzati in un ordine particolare. + D: I bisogni soddisfatti sono motivatori e nuovi bisogni emergono quando i bisogni + attuali rimangono insoddisfatti. + input_correct_responses: + - B + input_question: " Quale dei seguenti \xE8 un presupposto nella gerarchia dei bisogni\ + \ di Maslow?" + - input_choice_list: + A: "Il consumatore pi\xF9 anziano che si sente un po\u2019 escluso." + B: "Le donne sposate, molte delle quali sentono il bisogno di stabilit\xE0 nella\ + \ loro vita." + C: Nuovi immigrati che vogliono davvero assimilarsi alla loro nuova cultura. + D: Bambini, che basano la maggior parte delle loro decisioni di acquisto su + influenze esterne. + input_correct_responses: + - D + input_question: "Il singolo gruppo all\u2019interno della societ\xE0 che \xE8\ + \ pi\xF9 vulnerabile all\u2019influenza del gruppo di riferimento \xE8:" + - input_choice_list: + A: Linee di cura. + B: Posta diretta. + C: Inserti. + D: Porta a porta. + input_correct_responses: + - D + input_question: "Sebbene il contenuto e la qualit\xE0 possano essere controllati\ + \ quanto la posta diretta, i tassi di risposta di questo mezzo sono inferiori\ + \ a causa della mancanza di un meccanismo di indirizzo personale. Questo formato\ + \ multimediale \xE8 noto come:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_marketing +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_marketing +task_alias: marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8d55b06bf885f4faadfd372cfdb337b4229a075 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_medical_genetics.yaml @@ -0,0 +1,50 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: reazione a catena della polimerasi. + B: Analisi del polimorfismo conformazionale a singolo filamento. + C: Southernblotting. + D: Blotting occidentale. + input_correct_responses: + - C + input_question: 'Grandi espansioni di ripetizioni di triplette possono essere + rilevate da:' + - input_choice_list: + A: un enzima che unisce i frammenti nella normale replicazione del DNA + B: un enzima di origine batterica che taglia il DNA in sequenze di basi definite + C: un enzima che facilita la trascrizione di geni specifici + D: un enzima che limita il livello raggiunto da un particolare nutriente + input_correct_responses: + - A + input_question: "La DNA ligasi lo \xE8" + - input_choice_list: + A: ha entrambi gli alleli espressi indipendentemente nell'eterozigote + B: ha un allele dominante sull'altro + C: ha alleli strettamente legati sullo stesso cromosoma + D: ha alleli espressi contemporaneamente nello sviluppo + input_correct_responses: + - A + input_question: Un gene che mostra codominanza + - input_choice_list: + A: Stenosi pilorica + B: Schizofrenia + C: Spina bifida (difetti del tubo neurale) + D: Sindrome di Marfan + input_correct_responses: + - D + input_question: "Quale delle seguenti condizioni non mostra ereditariet\xE0 multifattoriale?" + - input_choice_list: + A: profase I + B: metafase I + C: profase II + D: metafase II + input_correct_responses: + - A + input_question: "Lo stadio della meiosi in cui i cromosomi si accoppiano e si\ + \ incrociano \xE8:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_medical_genetics +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_medical_genetics +task_alias: medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6419e5afc7aa8ef550a6a8a0588344150ad4e8a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_miscellaneous.yaml @@ -0,0 +1,51 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: uno + B: due + C: quattro + D: otto + input_correct_responses: + - B + input_question: Quanti assi ha un'automobile standard? + - input_choice_list: + A: Budapest + B: Budokan + C: Bhutan + D: Gran Bretagna + input_correct_responses: + - B + input_question: "Quale luogo \xE8 nominato nel titolo dell'album live del\ + \ 1979 delle leggende del rock Cheap Trick?" + - input_choice_list: + A: Anthony "Spud" Webb + B: Michael "Air" Jordan + C: Tyrone 'Muggsy' Bogue + D: Julius "Dr J" Erving + input_correct_responses: + - A + input_question: "Chi \xE8 l'uomo pi\xF9 basso che abbia mai vinto una gara\ + \ di schiacciata NBA?" + - input_choice_list: + A: idrogeno + B: nylon + C: ossigeno + D: leggero + input_correct_responses: + - C + input_question: Cosa viene prodotto durante la fotosintesi? + - input_choice_list: + A: ''Radio ga ga'' + B: ''Ob-la-di Ob-la-da'' + C: '"De Do Do Do De Da Da Da"' + D: '"In-a-Gadda-Da-Vida"' + input_correct_responses: + - C + input_question: "Quale di queste canzoni \xE8 stata una delle prime 10 hit del\ + \ gruppo rock The Police?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_miscellaneous +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_miscellaneous +task_alias: miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14bf232d8ec9366c8fe4af155305e0cff5066fe8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_moral_disputes.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Non \xE8 ancora chiaro se la pena capitale scoraggi o meno danni alla comunit\xE0\ + ." + B: "Non \xE8 ancora chiaro se la pena capitale scoraggi o meno danni a individui." + C: "Il consequenzialismo non \xE8 supportato dai valori africani." + D: "Anche se il consequenzialismo \xE8 sostenuto dai valori africani, non \xE8\ + \ stato offerto alcun argomento consequenzialista inquadrato in termini di\ + \ valori africani." + input_correct_responses: + - A + input_question: "Secondo Metz, cosa c\u2019\xE8 di sbagliato negli argomenti consequenzialisti\ + \ contro la pena capitale basati sui valori africani?" + - input_choice_list: + A: uno zigote. + B: un embrione. + C: "una viabilit\xE0." + D: una blastocisti. + input_correct_responses: + - A + input_question: "Un ovulo fecondato \xE8 anche noto come" + - input_choice_list: + A: essere nella classe superiore + B: "la capacit\xE0 di apprendere e risolvere problemi" + C: "essere pi\xF9 bello della maggior parte della popolazione" + D: "essere pi\xF9 corto della media" + input_correct_responses: + - B + input_question: "Quale dei seguenti \xE8 un bene intrinseco secondo Singer?" + - input_choice_list: + A: "La SCNT non \xE8 un tipo di riproduzione diverso perch\xE9 non esistono\ + \ differenze moralmente rilevanti tra essa e altri mezzi di riproduzione consentiti." + B: "Poich\xE9 il rischio di danni per il gemellaggio naturale \xE8 basso, il\ + \ rischio di danni per SCNT sar\xE0 basso." + C: sia a che B + D: "n\xE8 a n\xE8 B" + input_correct_responses: + - A + input_question: Pence mette a confronto sei diversi casi di riproduzione, dal + gemellaggio naturale all'SCNT. Quale conclusione trae da questo confronto? + - input_choice_list: + A: "la stupidit\xE0 degli esempi filosofici pi\xF9 tradizionali." + B: una generale mancanza di intelligenza tra le persone con titoli di studio + avanzati. + C: l'ingiustizia della tortura. + D: "la disponibilit\xE0 da parte di molte persone intelligenti a vedere nella\ + \ tortura la migliore soluzione per affrontare il terrorismo." + input_correct_responses: + - D + input_question: "Baron ammette che le versioni dell'ipotetica bomba ad orologeria\ + \ di cui parla sono "incredibilmente stupide", ma sostiene che questa\ + \ \xE8 in realt\xE0 la prova di" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_moral_disputes +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_moral_disputes +task_alias: moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f17c610bc24799b7560a3575552426825c999f91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_moral_scenarios.yaml @@ -0,0 +1,70 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Sbagliato, sbagliato + B: Sbagliato, non sbagliato + C: Non sbagliato, sbagliato + D: Non sbagliato, non sbagliato + input_correct_responses: + - C + input_question: "Per quale di questi due scenari il personaggio principale (che\ + \ usa I/me/my) fa qualcosa di chiaramente moralmente sbagliato, secondo gli\ + \ standard morali ordinari negli Stati Uniti a partire dal 2020? Scenario 1\ + \ | Ho davvero amato il mio bambino appena nato. Scenario 2 | La nuova ragazza\ + \ \xE8 imbarazzata a vivere in un parcheggio per roulotte, quindi invito la\ + \ classe a casa sua." + - input_choice_list: + A: Sbagliato, sbagliato + B: Sbagliato, non sbagliato + C: Non sbagliato, sbagliato + D: Non sbagliato, non sbagliato + input_correct_responses: + - C + input_question: Per quale di questi due scenari il personaggio principale (che + usa I/me/my) fa qualcosa di chiaramente moralmente sbagliato, secondo gli standard + morali ordinari negli Stati Uniti a partire dal 2020? Scenario 1 | Ho chiesto + una sigaretta al ragazzo fuori dal negozio. Scenario 2 | Quando la guardia di + sicurezza ha lasciato il suo posto, ho intascato una moneta rara. + - input_choice_list: + A: Sbagliato, sbagliato + B: Sbagliato, non sbagliato + C: Non sbagliato, sbagliato + D: Non sbagliato, non sbagliato + input_correct_responses: + - D + input_question: Per quale di questi due scenari il personaggio principale (che + usa I/me/my) fa qualcosa di chiaramente moralmente sbagliato, secondo gli standard + morali ordinari negli Stati Uniti a partire dal 2020? Scenario 1 | Ho comprato + un vecchio pick-up per lavoro. Scenario 2 | Ho alzato lo sguardo verso la splendida + cascata e le ho lanciato un sasso. + - input_choice_list: + A: Sbagliato, sbagliato + B: Sbagliato, non sbagliato + C: Non sbagliato, sbagliato + D: Non sbagliato, non sbagliato + input_correct_responses: + - B + input_question: Per quale di questi due scenari il personaggio principale (che + usa I/me/my) fa qualcosa di chiaramente moralmente sbagliato, secondo gli standard + morali ordinari negli Stati Uniti a partire dal 2020? Scenario 1 | Durante la + demolizione della casa, l'ho spinto contro un muro. Scenario 2 | Quando + scelgo una babysitter per le mie nipoti gemelle, non mi interessa il suo background + razziale. + - input_choice_list: + A: Sbagliato, sbagliato + B: Sbagliato, non sbagliato + C: Non sbagliato, sbagliato + D: Non sbagliato, non sbagliato + input_correct_responses: + - A + input_question: Per quale di questi due scenari il personaggio principale (che + usa I/me/my) fa qualcosa di chiaramente moralmente sbagliato, secondo gli standard + morali ordinari negli Stati Uniti a partire dal 2020? Scenario 1 | Ho preso + il suo portafoglio dopo che mi ha molestato al bar. Scenario 2 | Ho fatto l'amore + con il mio bambino appena nato. +include: _continuation_template_yaml +process_docs: !function utils.process_docs_moral_scenarios +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_moral_scenarios +task_alias: moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95f4c8f89521299a4e05a8866c9b4a3ba20fbd7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_nutrition.yaml @@ -0,0 +1,67 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "I bevitori di t\xE8 hanno un rischio minore di sviluppare il diabete." + B: "I bevitori di t\xE8 hanno un rischio maggiore di sviluppare il diabete." + C: Sulla base delle informazioni fornite non possiamo dire se la differenza + osservata nel rischio di malattia sia il risultato del caso. + D: "Il rapporto di rischio \xE8 vicino al valore uno, quindi non vi \xE8 alcuna\ + \ differenza nel rischio di malattia tra i due gruppi." + input_correct_responses: + - C + input_question: "In uno studio di coorte, il rapporto di rischio di sviluppare\ + \ il diabete era 0,86 confrontando i consumatori di t\xE8 (gli esposti) con\ + \ quelli che non bevevano t\xE8 (i non esposti). Quale affermazione \xE8 corretta\ + \ (secondo le conoscenze nel 2020)?\\n" + - input_choice_list: + A: I consumatori affetti da fenilchetonuria devono evitare il consumo del dolcificante + aspartame + B: I consumatori affetti da fenilchetonuria devono evitare il consumo del dolcificante + saccarina + C: I consumatori affetti da fenilchetonuria devono evitare il consumo del dolcificante + sucralosio + D: I consumatori affetti da fenilchetonuria devono evitare il consumo del dolcificante + acesulfame K + input_correct_responses: + - A + input_question: "Quale delle seguenti affermazioni \xE8 corretta (secondo le conoscenze\ + \ del 2020)?\\n" + - input_choice_list: + A: L'acido propionico, formato durante la fermentazione delle fibre del + colon, inibisce la sintesi degli acidi grassi nel fegato + B: L'acido butirrico, formato durante la fermentazione delle fibre del colon, + stimola il "silenziamento" del gene soppressore del tumore SLC5A8 + C: "Nessuna di queste opzioni \xE8 corretta" + D: L'acido butirrico, formato durante la fermentazione delle fibre del colon, + stimola le difese antiossidanti del colon + input_correct_responses: + - D + input_question: "Quale delle seguenti \xE8 la spiegazione pi\xF9 plausibile per\ + \ l'effetto protettivo delle fibre alimentari contro il cancro del colon,\ + \ a partire dal 2020?\\n" + - input_choice_list: + A: Il 50% degli adulti consuma iodio a livelli inferiori all'RNI + B: I latticini sono una scarsa fonte di iodio + C: "Il contenuto di iodio del latte biologico \xE8 generalmente inferiore a\ + \ quello del latte non biologico" + D: "I valori dietetici di riferimento del Regno Unito raccomandano un aumento\ + \ dell\u2019assunzione di iodio in gravidanza" + input_correct_responses: + - C + input_question: "Quale delle seguenti affermazioni sullo iodio \xE8 corretta,\ + \ al 2020?\\n" + - input_choice_list: + A: Acarbosio + B: Metformina + C: Sulfaniluree + D: Insulina + input_correct_responses: + - B + input_question: "Qual \xE8 il farmaco di prima linea per i pazienti con diabete\ + \ di tipo 2 e obesit\xE0, a partire dal 2020?\\n" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_nutrition +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_nutrition +task_alias: nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..754c14a92fd40f09de40aa194a96d9754c2fc6e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_philosophy.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: una teoria etica su come dovremmo comportarci. + B: una generalizzazione riguardante il modo in cui le persone tendono a comportarsi. + C: un'affermazione sulla natura umana e sui modi in cui le persone sono + capaci di comportarsi. + D: nessuna delle precedenti. + input_correct_responses: + - C + input_question: "L\u2019egoismo psicologico \xE8:" + - input_choice_list: + A: piacere. + B: "felicit\xE0." + C: Bene. + D: "virt\xF9." + input_correct_responses: + - C + input_question: "Secondo l\u2019\u201Cutilitarismo ideale\u201D di Moore, l\u2019\ + azione giusta \xE8 quella che produce il maggior numero di:" + - input_choice_list: + A: libere scelte + B: dettami dell'anima + C: leggi naturali necessarie + D: "volont\xE0 indeterminata" + input_correct_responses: + - C + input_question: Secondo d'Holbach le persone agiscono sempre secondo _____. + - input_choice_list: + A: ottimista + B: soddisfatto + C: nominalmente religioso + D: pessimista + input_correct_responses: + - D + input_question: Prima della conversione cristiana di Tolstoj, qual era la sua + prospettiva sul significato della vita? + - input_choice_list: + A: metafisica + B: epistemologia + C: fisica quantistica + D: assiologia + input_correct_responses: + - A + input_question: "Lo studio della realt\xE0 nel senso pi\xF9 ampio, un'indagine\ + \ sulla natura elementare dell'universo e delle cose in esso contenute,\ + \ \xE8 noto come _____." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_philosophy +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_philosophy +task_alias: philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a817cf30a093508d3605b9c369d83fcc8083543f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_prehistory.yaml @@ -0,0 +1,61 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: soddisfare i potenti sacerdoti astronomi Maya. + B: "mostrare la sua generosit\xE0 alla gente comune, poich\xE9 era loro permesso\ + \ di vivere nei templi." + C: spaventare i nemici, in particolare gli spagnoli. + D: "legittimare la sua regalit\xE0, poich\xE9 suo padre non era reale." + input_correct_responses: + - D + input_question: "Il grande re Maya Pacal costru\xEC templi nella citt\xE0 di Palenque\ + \ per:" + - input_choice_list: + A: "un centro della civilt\xE0 del Mississippi con condizioni simili all'ascesa\ + \ dei primi stati." + B: "i limiti dell\u2019autorit\xE0 in una societ\xE0 dei nativi americani di\ + \ raccoglitori egualitari." + C: un chiefdom semplice o forse un chiefdom complesso si era evoluto nel 1500 + d.C. + D: "un centro della civilt\xE0 del Mississippi con condizioni simili alle societ\xE0\ + \ della costa nordoccidentale del Nord America." + input_correct_responses: + - A + input_question: 'Secondo Timothy Pauketat, le prove della stratificazione sociale + e del potere politico a Cahokia suggeriscono:' + - input_choice_list: + A: un cataclisma di qualche tipo, come un terremoto, un vulcano o uno tsunami. + B: degrado ecologico derivante dalle tecniche agricole di taglio e incendio. + C: "guerre infinite tra le vicine citt\xE0-stato Maya." + D: pratiche di incrocio che hanno portato ad un forte aumento dei disturbi congeniti. + input_correct_responses: + - B + input_question: 'I ricercatori ora credono che il declino dei Maya sia stato causato + principalmente da:' + - input_choice_list: + A: "una grande quantit\xE0 di diversit\xE0 di specie o una singola specie che\ + \ mostra molta diversit\xE0." + B: "pochissima diversit\xE0 di specie durante questo periodo e pochissimi ominidi." + C: "diminuzione della diversit\xE0 delle specie a causa di una prolungata era\ + \ glaciale seguita da una grave siccit\xE0." + D: "diminuzione della diversit\xE0 delle specie ma aumento del numero di pietre\ + \ e scaglie, indicando la produzione di utensili in pietra." + input_correct_responses: + - A + input_question: 'Ricerche recenti su specie di ominidi risalenti al Pliocene medio + indicano che esisteva (a partire dal 2020):' + - input_choice_list: + A: sotto i 650 cc + B: circa 800 cc + C: poco meno di 1000 cc + D: 1200 cc + input_correct_responses: + - C + input_question: "Qual \xE8 la capacit\xE0 cranica media approssimativa dell'Homo\ + \ erectus?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_prehistory +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_prehistory +task_alias: prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b08e9944c149c34a9eb6b01c948b9182c2d6e2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_accounting.yaml @@ -0,0 +1,74 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: $ 70.000 + B: $ 75.000 + C: $ 80.000 + D: '100000' + input_correct_responses: + - D + input_question: "Box, un'organizzazione non governativa senza fini di lucro,\ + \ ha effettuato le seguenti transazioni durante l'anno: Proventi dalla vendita\ + \ di investimenti $ 80.000 Acquisto di immobili, impianti e attrezzature $ 10.000\ + \ Proventi da debiti a lungo termine $ 100.000 Perdita sulla vendita di investimenti\ + \ $ 5.000 Quale importo deve essere riportato come netto liquidit\xE0 fornita\ + \ dalle attivit\xE0 di finanziamento nel rendiconto finanziario di Box?" + - input_choice_list: + A: $ 13.000 + B: $ 600 + C: $ 15.000 + D: $ 28.000 + input_correct_responses: + - A + input_question: "Cento anni fa, la tua bis-bisnonna invest\xEC $ 100 con un interesse\ + \ annuo del 5%. Quanto vale l\u2019investimento oggi?" + - input_choice_list: + A: $ 0 + B: $ 500 + C: $ 1.650 + D: $ 16.500 + input_correct_responses: + - A + input_question: "Krete \xE8 un contribuente non sposato con reddito esclusivamente\ + \ salariale. Entro il 31 dicembre, anno 1, il datore di lavoro di Krete ha trattenuto\ + \ $ 16.000 in imposte federali sul reddito e Krete non ha effettuato pagamenti\ + \ fiscali stimati. Il 15 aprile, anno 2, Krete ha presentato tempestivamente\ + \ una richiesta di proroga per presentare la sua dichiarazione dei redditi individuale\ + \ e ha pagato $ 300 di tasse aggiuntive. Il debito fiscale del primo anno di\ + \ Krete era di $ 16.500 quando ha presentato tempestivamente la dichiarazione\ + \ il 30 aprile, anno 2, e ha pagato il saldo del debito fiscale rimanente. Quale\ + \ importo sarebbe soggetto alla sanzione per mancato pagamento delle imposte\ + \ stimate?" + - input_choice_list: + A: $ 5.000 + B: $ 13.500 + C: $ 16.000 + D: $ 20.000 + input_correct_responses: + - B + input_question: "Il 1\xB0 gennaio dell'anno 1, Alpha Co. ha firmato un contratto\ + \ di manutenzione annuale con un fornitore di software per 15.000 dollari e\ + \ il periodo di manutenzione inizia il 1\xB0 marzo dell'anno 2. Alpha ha\ + \ inoltre sostenuto costi di 5.000 dollari il 1\xB0 gennaio dell'anno 1,\ + \ relativi alla modifica del software richieste che aumenteranno la funzionalit\xE0\ + \ del software. Alpha svaluta e ammortizza i suoi beni informatici e software\ + \ in cinque anni utilizzando il metodo a quote costanti. Qual \xE8 l'importo\ + \ totale della spesa che Alpha dovrebbe riconoscere in relazione al contratto\ + \ di manutenzione e alle modifiche del software per l'anno terminato il\ + \ 31 dicembre, anno 1?" + - input_choice_list: + A: Valutazione e allocazione + B: Completezza + C: Diritti e doveri + D: Presentazione e divulgazione + input_correct_responses: + - B + input_question: "Un revisore traccia i numeri di serie sulle apparecchiature nel\ + \ registro secondario di un non emittente. Quale delle seguenti asserzioni del\ + \ management \xE8 supportata da questo test?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_accounting +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_professional_accounting +task_alias: professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1d529268b9a5146e6f6ad08f9fc559e5de0c07d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_law.yaml @@ -0,0 +1,132 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "lo statuto \xE8 nullo per vaghezza ai sensi della clausola del giusto processo\ + \ del Quattordicesimo Emendamento." + B: "lo statuto non \xE8 valido perch\xE9 viola la libert\xE0 di parola del firmatario\ + \ ai sensi del Primo Emendamento." + C: "lo statuto \xE8 una limitazione della libert\xE0 di parola ai sensi del\ + \ Primo Emendamento perch\xE9 sono disponibili mezzi meno restrittivi per\ + \ raggiungere lo stesso scopo." + D: "lo statuto \xE8 troppo ampio e di conseguenza invalido ai sensi del primo\ + \ e del quattordicesimo emendamento." + input_correct_responses: + - D + input_question: "Un legislatore statale ha recentemente promulgato una legge che\ + \ rende un reato imprecare, insultare o usare un linguaggio osceno o dispregiativo\ + \ nei confronti o in riferimento a un agente di polizia che svolge le sue funzioni.\ + \ Uno studente di un'universit\xE0 statale ha organizzato una manifestazione\ + \ nel campus per protestare contro la guerra. Alla manifestazione ha partecipato\ + \ un gruppo di 50 studenti che hanno gridato messaggi contro la guerra alle\ + \ auto che passavano. Per mostrare il suo disprezzo per gli Stati Uniti, lo\ + \ studente ha cucito la bandiera americana sul retro dei suoi jeans. Quando\ + \ un agente di polizia ha visto la bandiera cucita sui jeans dello studente,\ + \ si \xE8 avvicinato e gli ha detto di rimuovere la bandiera altrimenti sarebbe\ + \ stato arrestato. Lo studente si \xE8 arrabbiato e ha gridato all'ufficiale\ + \ di polizia: "Ascolta, bastardo, indosser\xF2 questo straccio ovunque\ + \ mi piaccia". Lo studente \xE8 stato successivamente arrestato e accusato\ + \ di aver violato lo statuto dello stato. Lo studente ha successivamente intentato\ + \ causa dinanzi al tribunale statale contestando la costituzionalit\xE0 della\ + \ norma. L\u2019argomento costituzionale pi\xF9 forte per lo studente \xE8 questo" + - input_choice_list: + A: La clausola commerciale. + B: La clausola di pari protezione del Quattordicesimo Emendamento. + C: "La clausola dei privilegi e delle immunit\xE0 dell\u2019Articolo IV, Sezione\ + \ 2. " + D: La clausola contrattuale. + input_correct_responses: + - A + input_question: "Uno stato ha recentemente promulgato una legge che vieta lo smaltimento\ + \ di qualsiasi scoria nucleare all'interno dello stato. Questa legge non\ + \ contravviene n\xE9 \xE8 in conflitto con alcuno statuto federale. Un uomo\ + \ gestisce un'azienda statale che si occupa dello smaltimento delle scorie\ + \ nucleari. Successivamente all'approvazione dello statuto statale, l'uomo,\ + \ non ancora a conoscenza della nuova legge, ha stipulato contratti con molte\ + \ aziende straniere per smaltire le proprie scorie nucleari nello stato. A causa\ + \ di questa nuova legge, per\xF2, l'uomo non potr\xE0 eseguire questi contratti.\ + \ Supponiamo che l'uomo abbia la legittimazione ad impugnare questa legge\ + \ statale. Quale delle seguenti affermazioni presenta le basi costituzionali\ + \ pi\xF9 forti per contestare la legge statale che vieta lo smaltimento delle\ + \ scorie nucleari all'interno dello Stato?" + - input_choice_list: + A: Fatti indiscutibili. + B: Fatti affermati da singole organizzazioni politiche. + C: Fatti riconosciuti veri dalla conoscenza comune. + D: Fatti suscettibili di verifica scientifica. + input_correct_responses: + - B + input_question: "Il giudice ha preso atto di alcuni fatti all'inizio del processo.\ + \ Quale dei seguenti non \xE8 un fatto appropriato per la notifica giudiziaria?" + - input_choice_list: + A: "concedere uno sgravio perch\xE9 la recinzione violava il vincolo di servit\xF9\ + . " + B: "concedere uno sgravio, perch\xE9 lo sconfinamento della recinzione ha violato\ + \ la restrizione prevista dal piano originale. " + C: "negare lo sgravio, perch\xE9 l'insegnante non ha fatto rispettare la\ + \ restrizione nei confronti del pensionato. " + D: "negare il sollievo, perch\xE9 la recinzione non sarebbe interpretata come\ + \ "una struttura" nei termini della restrizione. " + input_correct_responses: + - B + input_question: "Il 1\xB0 ottobre 1980, un costruttore, proprietario di diverse\ + \ centinaia di acri in una contea rurale, redasse un piano generale di sviluppo\ + \ per l'area. Il piano debitamente registrato imponeva elaborate limitazioni\ + \ e restrizioni al terreno nel piano, che doveva essere sviluppato come quartiere\ + \ residenziale. Le restrizioni dovevano estendersi a tutte le persone che acquistavano\ + \ uno qualsiasi dei lotti e ai loro eredi, aventi causa e locatari. Si prevedeva\ + \ inoltre che tutti i successivi proprietari sarebbero stati incaricati di notificare\ + \ debitamente le restrizioni. Tra queste restrizioni nel piano generale c'erano\ + \ le seguenti: (22) Viene creato un diritto di franchising in una striscia di\ + \ terreno larga 10 piedi lungo il retro di ciascun lotto per l'uso di societ\xE0\ + \ di servizi pubblici con diritto di ingresso e uscita. (23) Nessuna casa o\ + \ struttura di alcun tipo potr\xE0 essere costruita sulla suddetta striscia\ + \ di terreno che attraversa detti blocchi. Nel 2000, un pensionato acquist\xF2\ + \ uno dei lotti, costru\xEC una casa ed eresse una recinzione sul retro della\ + \ sua propriet\xE0 all'interno dell'area riservata. Nel 2004 un insegnante\ + \ acquist\xF2 un terreno adiacente alla propriet\xE0 del pensionato e costru\xEC\ + \ una nuova casa. Due anni dopo, un bibliotecario acquist\xF2 il lotto confinante\ + \ con la propriet\xE0 dell'insegnante. I tre atti relativi a tali propriet\xE0\ + \ contenevano ciascuno riferimenti al libro degli atti in cui era registrato\ + \ il piano generale. Nel 2008, il bibliotecario ha iniziato la costruzione di\ + \ una recinzione con pali e ringhiere alta sette piedi lungo la linea che divide\ + \ il suo lotto da quello dell'insegnante e lungo il centro dell'area\ + \ soggetta al diritto di franchising. Nonostante l'insegnante si fosse opposto\ + \ alla sua costruzione, la recinzione fu completata. Se l'insegnante chiede\ + \ un'ingiunzione obbligatoria per obbligare alla rimozione della recinzione\ + \ del bibliotecario, molto probabilmente il tribunale lo far\xE0." + - input_choice_list: + A: 'La promessa del padre e l'affidamento del creditore su di essa, se provati, + davano luogo ad una valida pretesa del creditore nei confronti del padre basata + sulla dottrina del promissory estoppel. ' + B: "Poich\xE9 era prevedibile che la promessa del padre avrebbe indotto il creditore\ + \ a astenersi da qualsiasi azione contro il figlio, tale indulgenza era, per\ + \ legge, un corrispettivo pattuito per la promessa del padre. " + C: "I cinque pagamenti del padre al creditore per un totale di $ 2.500 manifestavano\ + \ la seria intenzione da parte del padre di essere contrattualmente vincolato,\ + \ e tale manifestazione \xE8 generalmente riconosciuta come un efficace sostituto\ + \ del corrispettivo. " + D: "Facendosi carico del debito antecedente che il figlio aveva nei confronti\ + \ del creditore, il padre diventava un garante la cui promessa al creditore\ + \ era esecutiva, poich\xE9 era scritta e supportata da adeguato corrispettivo. " + input_correct_responses: + - A + input_question: "Un figlio doveva a un creditore 5.000 dollari. Il padre del figlio\ + \ contatt\xF2 il creditore e gli disse che voleva saldare il debito del figlio.\ + \ Il padre firm\xF2 un documento in cui dichiarava che avrebbe pagato il debito\ + \ del figlio ad una tariffa di 500 dollari al mese per 10 mesi. Il creditore\ + \ non ha assunto alcun impegno scritto o orale a astenersi dal citare in giudizio\ + \ il figlio per riscuotere il debito di $ 5.000, e il padre non ha presentato\ + \ alcuna richiesta orale o scritta per tale tolleranza. Per i successivi cinque\ + \ mesi, il padre effettu\xF2 e il creditore accett\xF2 i pagamenti mensili di\ + \ 500 dollari come concordato. Durante tale periodo, il creditore, infatti,\ + \ si astenne dall'intraprendere qualsiasi azione legale nei confronti del\ + \ figlio. Tuttavia, il padre ha poi informato il creditore che non avrebbe effettuato\ + \ ulteriori pagamenti per il debito. Quale dei seguenti \xE8 l'argomento\ + \ pi\xF9 convincente secondo cui il padre \xE8 responsabile nei confronti del\ + \ creditore secondo i termini del contratto?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_law +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_professional_law +task_alias: professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a105671bf3c0601626aa5576debf73a6105f42f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_medicine.yaml @@ -0,0 +1,90 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Labetalolo + B: Una dose di carico di cloruro di potassio + C: Nifedipina + D: Fenossibenzamina + input_correct_responses: + - D + input_question: "Un uomo di 42 anni si presenta in studio per una valutazione\ + \ preoperatoria prima di essere sottoposto ad surrenectomia programmata tra\ + \ 2 settimane. Un mese fa \xE8 stato ricoverato al pronto soccorso per un dolore\ + \ al fianco destro dovuto a uno scontro automobilistico. A quel tempo, la pressione\ + \ sanguigna era di 160/100 mm Hg e la TC dell'addome mostrava una massa\ + \ surrenale sinistra incidentale di 10 cm. I risultati degli studi di laboratorio,\ + \ tra cui emocromo completo, concentrazioni di elettroliti sierici e test di\ + \ funzionalit\xE0 epatica, rientravano negli intervalli di riferimento. Per\ + \ il resto il paziente era sano e non gli era mai stato detto che aveva la pressione\ + \ sanguigna elevata. Non prende farmaci. Una visita di follow-up in studio 2\ + \ settimane fa ha rivelato elevate concentrazioni di normetanefrina e metanefrina\ + \ urinaria e di aldosterone plasmatico. Il paziente \xE8 stato indirizzato a\ + \ un chirurgo, che ha raccomandato la surrenalectomia. Oggi, i segni vitali\ + \ sono la temperatura 36,6\xB0C (97,9\xB0F), il polso 100/min, la respirazione\ + \ 14/min e la pressione sanguigna 170/95 mm Hg. L'esame obiettivo non rivela\ + \ risultati significativi. La preparazione preoperatoria iniziale dovrebbe includere\ + \ il trattamento con quale dei seguenti?" + - input_choice_list: + A: Torsione sacrale sinistra contro sinistra + B: torsione sacrale sinistra-destra + C: flessione sacrale unilaterale destra + D: torsione sacrale dx-dx + input_correct_responses: + - D + input_question: "Un uomo di 36 anni si presenta in studio con un'anamnesi\ + \ di lombalgia da 3 settimane. Nega qualsiasi trauma recente ma dice che entra\ + \ e esce dal suo camion numerose volte al giorno per il suo lavoro. L'esame\ + \ del paziente in posizione prona rivela un profondo solco sacrale a sinistra,\ + \ un angolo laterale postero-inferiore a destra e una giunzione lombosacrale\ + \ che scatta liberamente in compressione. La diagnosi pi\xF9 probabile \xE8" + - input_choice_list: + A: Dopamina + B: Glutammato + C: Noradrenalina + D: Serotonina + input_correct_responses: + - D + input_question: "Una donna di 32 anni, precedentemente sana, si reca dal medico\ + \ 8 mesi dopo la morte del marito in un incidente stradale. Da quel momento\ + \ ha avuto una diminuzione dell'appetito e difficolt\xE0 ad addormentarsi.\ + \ Afferma di essere spesso triste e di piangere spesso. Ha ricontrollato la\ + \ serratura della porta cinque volte prima di uscire di casa e deve contare\ + \ esattamente cinque pezzi di carta igienica prima di usarla. Dice di essere\ + \ sempre stata una perfezionista, ma questi impulsi e rituali sono nuovi. La\ + \ farmacoterapia dovrebbe essere mirata a quale dei seguenti neurotrasmettitori?" + - input_choice_list: + A: Rinite allergica + B: Virus Epstein-Barr + C: Mycoplasma pneumoniae + D: Rinovirus + input_correct_responses: + - D + input_question: "Un uomo di 44 anni si presenta in studio a causa di una storia\ + \ di mal di gola da 3 giorni, tosse non produttiva, naso che cola e mal di testa\ + \ frontale. Dice che il mal di testa peggiora al mattino e l'ibuprofene\ + \ fornisce un certo sollievo. Non ha avuto fiato corto. L'anamnesi medica\ + \ \xE8 irrilevante. Non prende farmaci diversi dall'ibuprofene per il dolore.\ + \ I segni vitali sono temperatura 37,4\xB0C (99,4\xB0F), polso 88/min, respirazione\ + \ 18/min e pressione sanguigna 120/84 mm Hg. L'esame delle narici evidenzia\ + \ mucose eritematose. L'esame della gola mostra eritema e iperplasia linfoide\ + \ follicolare sull'orofaringe posteriore. Non \xE8 presente adenopatia cervicale\ + \ palpabile. I polmoni sono liberi all'auscultazione. Quale delle seguenti\ + \ \xE8 la causa pi\xF9 probabile dei sintomi di questo paziente?" + - input_choice_list: + A: scaleno anteriore + B: latissimus dorsi + C: pettorale minore + D: quadrato dei lombi + input_correct_responses: + - C + input_question: "Un maratoneta di 22 anni si presenta in studio lamentando dolore\ + \ alla costola destra quando corre per lunghe distanze. L'esame obiettivo\ + \ rivela reperti cardiaci e polmonari normali e una disfunzione espiratoria\ + \ alle costole 4-5 a destra. Quale dei seguenti muscoli o gruppi muscolari sar\xE0\ + \ pi\xF9 utile per correggere questa disfunzione utilizzando un metodo diretto?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_medicine +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_professional_medicine +task_alias: professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d17c0d021fecf450a6c60a10df50c8d6403e7e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_professional_psychology.yaml @@ -0,0 +1,62 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: racconta al tuo cliente i pro e i contro di ogni programma che conosci tranne + quello a cui sei iscritto + B: consiglia al tuo cliente il programma a cui sei iscritto e spiega il bonus + di $ 50 che riceverai + C: consigliare al tuo cliente il programma a cui sei iscritto e offrirti di + ricevere il bonus di $ 50 accreditato sul conto del tuo cliente nel programma + D: racconta al tuo cliente i pro e i contro di ogni programma che conosci, ma + non richiedere il bonus di $ 50 se il tuo cliente si iscrive al tuo programma + input_correct_responses: + - D + input_question: "Uno dei tuoi clienti in terapia ti chiede consiglio su un buon\ + \ programma di riduzione del peso. Hai studiato i programmi della comunit\xE0\ + \ e sei iscritto a quello che consideri il migliore. Questo programma offre\ + \ un bonus di $ 50 ai suoi utenti per ogni nuova persona introdotta nel programma.\ + \ In queste circostanze, la risposta pi\xF9 appropriata sarebbe:" + - input_choice_list: + A: meno sensibile ai punteggi estremi rispetto alla media + B: "pi\xF9 utile per distribuzioni asimmetriche" + C: sensibile ai valori estremi e alle distribuzioni altamente asimmetriche + D: "il numero pi\xF9 frequente" + input_correct_responses: + - D + input_question: "Esistono tre modi per misurare la tendenza centrale: la media,\ + \ la mediana e la moda. Dalla tua conoscenza su di loro, qual \xE8 la modalit\xE0\ + ?" + - input_choice_list: + A: individualismo. + B: Individualismo e distanza dal potere. + C: "distanza di potere e mascolinit\xE0." + D: "evitamento dell\u2019incertezza." + input_correct_responses: + - A + input_question: 'In termini di cinque dimensioni culturali di Hofstede (1980), + gli Stati Uniti si collocano in cima alla scala in:' + - input_choice_list: + A: "\xE8 una fantasia che distrae il cliente dalla realt\xE0." + B: "rappresenta \u201Csentimenti contrastanti\u201D nei confronti del terapeuta. " + C: ""\xE8 una forma di ""recitazione."""" + D: riflette l'inconscio personale e collettivo del cliente. + input_correct_responses: + - D + input_question: 'Carl Jung credeva che il transfert del cliente:' + - input_choice_list: + A: non sono correlati tra loro ma sono moderatamente correlati con il criterio + B: hanno basse correlazioni tra loro e basse correlazioni con il criterio + C: sono altamente intercorrelati tra loro e moderatamente correlati con il criterio + D: hanno basse correlazioni con il criterio ma sono moderatamente correlati + tra loro + input_correct_responses: + - A + input_question: "Nella costruzione di un'equazione di regressione multipla\ + \ a fini di previsione, la combinazione ottimale di misure \xE8 quella in cui\ + \ i predittori" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_psychology +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_professional_psychology +task_alias: professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e12d563ad7b2e6890716eec8300e022367335904 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_public_relations.yaml @@ -0,0 +1,55 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Fornisci al giornalista altre informazioni che \xE8 certa siano corrette." + B: Diciamo che le informazioni sono "non registrate" e verranno diffuse + in seguito. + C: Di' "Non lo so" e prometti di fornire le informazioni in seguito. + D: Di' "no comment" piuttosto che sembrare disinformato. + input_correct_responses: + - C + input_question: Cosa dovrebbe fare un professionista dei media nelle pubbliche + relazioni se non conosce la risposta alla domanda di un giornalista? + - input_choice_list: + A: Acquista nomi di dominio che potrebbero essere utilizzati dai gruppi di opposizione. + B: Pubblica commenti anonimi sui blog per combattere queste informazioni. + C: Preparare un comunicato stampa che screditi le informazioni inesatte. + D: Apportare modifiche alle politiche per affrontare i reclami evidenziati su + questi siti. + input_correct_responses: + - D + input_question: "Nella gestione dei problemi, qual \xE8 l'approccio pi\xF9\ + \ proattivo per affrontare le informazioni negative o fuorvianti pubblicate\ + \ online sulla tua organizzazione?" + - input_choice_list: + A: "C\u2019\xE8 stata una risposta coordinata da parte dei media." + B: Sono stati comunicati messaggi coerenti. + C: Le critiche furono interpretate come attacchi alla Chiesa cattolica. + D: "La credibilit\xE0 del Vaticano \xE8 stata mantenuta." + input_correct_responses: + - C + input_question: "Quale di queste affermazioni \xE8 vera per il Vaticano nel 2010,\ + \ al momento delle accuse di insabbiamento degli abusi sui minori?" + - input_choice_list: + A: Definizione del programma + B: Pianificazione del programma + C: Agire e implementare le idee + D: Valutazione del programma + input_correct_responses: + - A + input_question: In quale fase del processo di pianificazione verrebbe effettuata + un'analisi della situazione? + - input_choice_list: + A: Pace verde + B: L'ONU + C: Oxfam + D: Fondo mondiale per la fauna selvatica + input_correct_responses: + - D + input_question: "Earth Hour \xE8 stata una campagna lanciata da quale organizzazione?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_public_relations +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_public_relations +task_alias: public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f7b30d8f1de655056deb6f5c50a096d7831e292 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_security_studies.yaml @@ -0,0 +1,113 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Compellenza \xE8 un altro termine per indicare la diplomazia coercitiva,\ + \ ma copre un insieme pi\xF9 ristretto di criteri; la coercizione copre quelle\ + \ minacce volte ad avviare un'azione avversaria. Una minaccia di costringere\ + \ uno Stato a cedere parte del suo territorio varrebbe come diplomazia coercitiva,\ + \ a condizione che tale minaccia avvii proattivamente un\u2019azione prima\ + \ che venga intrapresa la diplomazia reattiva." + B: "La diplomazia coercitiva costituisce la minaccia di una forza limitata per\ + \ incentivare l'avversario a soddisfare le richieste del coercitivo. \xC8\ + \ una strategia di influenza che mira a ottenere condiscendenza: l\u2019uso\ + \ della forza per sconfiggere prima un avversario non conta. Lascia un elemento\ + \ di scelta con l'obiettivo di conformarsi o continuare." + C: "La forza militare, o la minaccia della forza militare, utilizza la paura\ + \ per raggiungere obiettivi strategici. La diplomazia coercitiva si differenzia\ + \ da questo approccio perch\xE9 non utilizza la paura come strumento per costringere\ + \ un avversario." + D: "La diplomazia coercitiva viene impiegata per usare la forza ma per limitarne\ + \ gli effetti sulla comunit\xE0 internazionale. La diplomazia coercitiva \xE8\ + \ una strategia aggressiva che mira a ottenere la conformit\xE0 attraverso\ + \ la sconfitta. Non lascia un elemento di scelta all'obiettivo, poich\xE9\ + \ l'obiettivo \xE8 costretto a conformarsi o ad impegnarsi in un conflitto.\ + \ Si cerca di controllare imponendo il rispetto e rimuovendo ogni opportunit\xE0\ + \ di negoziazione o concessione." + input_correct_responses: + - B + input_question: Cosa distingue la diplomazia coercitiva dalla forza militare? + - input_choice_list: + A: I bambini soldato sono vittime di combattimenti che necessitano di rieducazione + e riabilitazione. + B: I bambini e le loro madri non sono soggetti attivi nella guerra e sono meglio + considerati come soggetti nella sfera privata. + C: I bambini sono molto spesso spettatori innocenti della guerra e sono meglio + usati come simboli di pace. + D: "I bambini hanno una soggettivit\xE0 politica che viene persa quando vengono\ + \ considerati vittime passive della guerra." + input_correct_responses: + - D + input_question: "Quale delle seguenti \xE8 la lente migliore attraverso la quale\ + \ indagare il ruolo dei bambini soldato?" + - input_choice_list: + A: "Come una minaccia esistenziale che richiede un\u2019azione immediata e straordinaria,\ + \ rappresentando una minaccia per la sopravvivenza dello Stato o per la sicurezza\ + \ sociale." + B: "Richiedendo un\u2019azione immediata e straordinaria da parte dello Stato,\ + \ minacciando la sopravvivenza di un oggetto di riferimento e quindi garantendo\ + \ l\u2019uso di misure normalmente non impiegate in ambito politico." + C: "Come una minaccia urgente alla sopravvivenza dell'oggetto referente,\ + \ cos\xEC grave da legittimare il ricorso ad azioni straordinarie di risposta." + D: Come una minaccia urgente alla sopravvivenza del pubblico che richiede misure + straordinarie o di emergenza. + input_correct_responses: + - C + input_question: Per diventare cartolarizzato, in quale di questi modi deve essere + presentata una minaccia? + - input_choice_list: + A: "Esistono divisioni cos\xEC ampie all\u2019interno del quadro della sicurezza\ + \ umana per quanto riguarda la natura delle minacce e degli oggetti di riferimento\ + \ che non \xE8 possibile tracciare confronti ampiamente applicabili tra gli\ + \ approcci incentrati sullo stato e la sicurezza umana." + B: "Adottando il quadro della sicurezza umana, i limiti dell\u2019approccio\ + \ realista incentrato sullo stato diventano evidenti. Mentre la sicurezza\ + \ umana definisce l\u2019oggetto di riferimento come la persona o la popolazione,\ + \ gli approcci incentrati sullo stato danno priorit\xE0 alla sicurezza dello\ + \ stato, declassando il perseguimento della sicurezza umana." + C: "L\u2019approccio alla sicurezza incentrato sullo stato \xE8 una fazione\ + \ della sicurezza umana, solitamente definita all\u2019interno dell\u2019\ + ampia scuola della sicurezza umana. Essendo incentrato sullo stato, questo\ + \ approccio d\xE0 priorit\xE0 all\u2019individuo come oggetto di riferimento\ + \ negli studi sulla sicurezza." + D: "Sia l\u2019approccio alla sicurezza incentrato sullo stato che quello incentrato\ + \ sull\u2019uomo si escludono a vicenda e offrono un quadro analitico sufficiente\ + \ con cui comprendere il sistema di sicurezza internazionale. \xC8 quindi\ + \ compito degli analisti della sicurezza determinare quale di questi concetti\ + \ sostanziali \xE8 corretto e quale dovrebbe essere scartato." + input_correct_responses: + - B + input_question: "Come possiamo descrivere al meglio la relazione tra l\u2019approccio\ + \ stato-centrico e il concetto di sicurezza umana?" + - input_choice_list: + A: "La competizione tra nazioni pi\xF9 grandi ha portato alcuni paesi a sostenere\ + \ attivamente gruppi terroristici per minare la forza degli stati rivali.\ + \ Le reti terroristiche sono club di clientelismo esteso gestiti e pagati\ + \ dai loro stati donatori e sono concettualizzati come attori statali, da\ + \ affrontare utilizzando la forza militare." + B: "La globalizzazione ha consentito l\u2019internazionalizzazione delle attivit\xE0\ + \ terroristiche aprendone lo spazio operativo, sebbene il coordinamento sia\ + \ ancora gestito su base geografica. Ci\xF2 suggerisce che i gruppi terroristici\ + \ sono strutturati a livello nazionale, il che significa che il terrorismo\ + \ non pu\xF2 essere considerato in termini di una guerra da sconfiggere militarmente\ + \ senza avere gravi implicazioni sulla popolazione indigena." + C: "Il terrorismo pu\xF2 essere visto come un problema da risolvere con mezzi\ + \ militari (guerra al terrorismo), con normali tecniche di polizia (terrorismo\ + \ come crimine) o come un problema medico con cause e sintomi sottostanti\ + \ (terrorismo come malattia)." + D: "Il terrorismo \xE8 visto come un problema criminale. La criminalizzazione\ + \ del terrorismo ha due importanti implicazioni. In primo luogo, suggerisce\ + \ che il terrorismo pu\xF2 essere sradicato \u2013 i terroristi possono essere\ + \ catturati e processati mediante normali procedimenti giudiziari eliminando\ + \ cos\xEC la minaccia per la societ\xE0 \u2013 e in secondo luogo, suggerisce\ + \ che tecniche preventive di criminalit\xE0 siano applicabili per impedirne\ + \ lo sviluppo." + input_correct_responses: + - C + input_question: "Quali sono i quadri di analisi all\u2019interno dei quali \xE8\ + \ stato considerato il terrorismo (a partire dal 2020)?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_security_studies +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_security_studies +task_alias: security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c17fb67476d46834ed2adffe53f954f518e1f8e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_sociology.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: assistenza sanitaria e istruzione gratuite per tutti + B: un salario minimo + C: piena occupazione + D: benessere universale + input_correct_responses: + - B + input_question: 'Quale dei seguenti servizi lo stato sociale del dopoguerra del + 1948 non mirava a fornire:' + - input_choice_list: + A: una giostra da fiera + B: un circo + C: un teatro di marionette + D: un balletto + input_correct_responses: + - C + input_question: "Cosa descrive Berger (1963) come metafora della realt\xE0 sociale?" + - input_choice_list: + A: la crescente burocrazia statale ha reso la religione solo una parte marginale + della nostra vita + B: "nonostante l\u2019indebolimento dell\u2019autorit\xE0 tradizionale, la nostra\ + \ vita quotidiana e il \u201Csenso comune\u201D rimangono plasmati da credenze\ + \ e valori religiosi" + C: "la partecipazione religiosa al culto collettivo pu\xF2 essere diminuita,\ + \ ma le persone continuano a praticare la propria fede in privato" + D: "le persone sono molto pi\xF9 propense a discutere le proprie convinzioni\ + \ religiose in contesti pubblici e informali" + input_correct_responses: + - B + input_question: "Il passaggio dalla \u201Creligione civile\u201D alla \u201Creligione\ + \ comune\u201D significa che:" + - input_choice_list: + A: la tendenza della classe operaia a non realizzare i propri interessi + B: "un\u2019ideologia dominante che legittima il potere economico, politico\ + \ e culturale" + C: una forma di doppia coscienza basata sull'ideologia e sulle esperienze + quotidiane + D: "una modalit\xE0 di pagamento prevista per arte topiaria eccezionale" + input_correct_responses: + - B + input_question: "Il termine \u201Cegemonia\u201D si riferisce a:" + - input_choice_list: + A: la maggior parte degli scioperi passa inosservata ai datori di lavoro e ai + mass media + B: non tutte le controversie industriali verranno denunciate dal datore di lavoro + C: dalla definizione di sciopero sono esclusi quelli che coinvolgono meno di + dieci lavoratori o che durano meno di un giorno + D: "\xE8 difficile confrontare gli scioperi misurati in modi diversi" + input_correct_responses: + - A + input_question: "Quale dei seguenti non \xE8 un problema associato alle statistiche\ + \ ufficiali sugli scioperi?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_sociology +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_sociology +task_alias: sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c34d857d1d56b6beb4bf7c9796e34d818f14102 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_us_foreign_policy.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Ha danneggiato il sostegno al modello americano di economia politica e capitalismo + B: Ha creato rabbia nei confronti degli Stati Uniti per aver esagerato la crisi + C: Ha aumentato il sostegno alla leadership globale americana sotto la presidenza + Obama + D: "Ha ridotto l\u2019uso globale del dollaro USA" + input_correct_responses: + - A + input_question: "In che modo la crisi finanziaria del 2008 ha influito sulla reputazione\ + \ internazionale dell\u2019America?" + - input_choice_list: + A: Ha globalizzato il contenimento. + B: Ha militarizzato il contenimento. + C: Ha chiesto lo sviluppo della bomba all'idrogeno. + D: Tutti i precedenti + input_correct_responses: + - D + input_question: "In che modo l\u2019NSC-68 ha cambiato la strategia degli Stati\ + \ Uniti?" + - input_choice_list: + A: politica del terrorismo. + B: politica economica. + C: politica estera. + D: politica internazionale. + input_correct_responses: + - C + input_question: "L\u2019ambito delle decisioni politiche che riguardano principalmente\ + \ le relazioni tra gli Stati Uniti e il resto del mondo \xE8 noto come" + - input_choice_list: + A: I realisti difensivi pongono maggiore enfasi sul ruolo delle istituzioni + internazionali + B: I realisti difensivi pongono meno enfasi sui fattori geografici + C: "I realisti offensivi danno maggiore priorit\xE0 all\u2019interesse nazionale\ + \ rispetto ai realisti difensivi." + D: I realisti difensivi credono che gli stati massimizzino la sicurezza, mentre + i realisti offensivi credono che gli stati massimizzino il potere + input_correct_responses: + - D + input_question: In che modo il realismo difensivo e il realismo offensivo differiscono + nella loro spiegazione del comportamento statale? + - input_choice_list: + A: La globalizzazione aveva reso uomini come lui troppo ricchi + B: La globalizzazione ha avvantaggiato solo alcuni stati americani, come New + York + C: "Le \xE9lite liberali avevano incoraggiato la globalizzazione, mentre gli\ + \ \u201Camericani comuni\u201D avevano perso il lavoro a causa di essa" + D: La globalizzazione ha incoraggiato guerre commerciali dannose + input_correct_responses: + - C + input_question: In che modo Donald Trump ha attaccato la globalizzazione nella + campagna del 2016? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_us_foreign_policy +tag: mmlu_it_llama_social_sciences_tasks +task: mmlu_it_llama_us_foreign_policy +task_alias: us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23c68eb0c678a416458ba2f1a7f2792fd1877ec9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_virology.yaml @@ -0,0 +1,52 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Perch\xE9 non hanno acido nucleico" + B: Richiedono un virus aiutante + C: Si replicano solo nelle cellule in divisione + D: "Pu\xF2 integrarsi nei cromosomi ospiti" + input_correct_responses: + - A + input_question: "Perch\xE9 i parvovirus sono parassiti ad alto impatto?" + - input_choice_list: + A: Virus fragili spesso visualizzati con RNA che fuoriesce dall'interno + B: Allungare i virus + C: Virus icosaedrici con involucro + D: Virus molto grandi + input_correct_responses: + - A + input_question: "Quale delle seguenti \xE8 una caratteristica morfologica dei\ + \ paramixovirus." + - input_choice_list: + A: Raccolta dati + B: Istituzione di un archivio di campioni biologici + C: Interesse dei partecipanti + D: Somministrazione del questionario da parte del personale + input_correct_responses: + - B + input_question: "Un fattore chiave che ha facilitato l\u2019applicazione degli\ + \ studi caso-controllo annidati dal MACS \xE8 stato:" + - input_choice_list: + A: Cambiamento nel comportamento + B: Copertura completa + C: Uso efficace della teoria comportamentale + D: Cambiamento comportamentale duraturo + input_correct_responses: + - D + input_question: "L\u2019obiettivo pi\xF9 importante di un intervento comportamentale\ + \ \xE8:" + - input_choice_list: + A: Individui sieroprevalenti con infezione da HIV + B: Sieronegativi + C: Sieroconvertitori + D: Sieronegativi ad alto rischio + input_correct_responses: + - C + input_question: "Il tempo medio di sopravvivenza all\u2019AIDS e alla morte \xE8\ + \ stato stabilito come segue:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_virology +tag: mmlu_it_llama_other_tasks +task: mmlu_it_llama_virology +task_alias: virology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da022f127d43420eaa2c6fa4337b618f97c6f9b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/mmlu_it_world_religions.yaml @@ -0,0 +1,50 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: L'Arcobaleno + B: Circoncisione + C: Un figlio + D: Bar mitzvah + input_correct_responses: + - B + input_question: " Qual \xE8 il segno del patto per i maschi ebrei?" + - input_choice_list: + A: Il Dharma + B: Il Sangha + C: Il Budda + D: Il Bodhisattva + input_correct_responses: + - A + input_question: "Qual \xE8 la seconda gemma nel buddismo?" + - input_choice_list: + A: Shang + B: Zhou + C: Han + D: Xia + input_correct_responses: + - B + input_question: ' In quale dinastia fu sviluppato il "Mandato del Cielo" + per legittimare i nuovi sovrani?' + - input_choice_list: + A: Honen + B: Tanaka + C: Tokugawa + D: Meiji + input_correct_responses: + - D + input_question: ' Quale governo giapponese promosse una sorta di culto nazionale + basato sull'imperatore e sulla sua associazione con i kami?' + - input_choice_list: + A: Testi rituali + B: Testi filosofici + C: Inni + D: Storie di origine + input_correct_responses: + - B + input_question: Come si possono caratterizzare le Upanishad? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_world_religions +tag: mmlu_it_llama_humanities_tasks +task: mmlu_it_llama_world_religions +task_alias: world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/utils.py b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e28399e4bb5a0c1b8be3c6d3eec3b0e1cad9b42a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_it/utils.py @@ -0,0 +1,104 @@ +from functools import partial + +import datasets + + +def process_docs(dataset: datasets.Dataset, subtask) -> datasets.Dataset: + return dataset.filter( + lambda example: example["subtask_name"] == f"mmlu_it_chat.{subtask}" + ) + + +process_docs_miscellaneous = partial(process_docs, subtask="miscellaneous") +process_docs_high_school_physics = partial(process_docs, subtask="high_school_physics") +process_docs_high_school_computer_science = partial( + process_docs, subtask="high_school_computer_science" +) +process_docs_high_school_statistics = partial( + process_docs, subtask="high_school_statistics" +) +process_docs_professional_accounting = partial( + process_docs, subtask="professional_accounting" +) +process_docs_machine_learning = partial(process_docs, subtask="machine_learning") +process_docs_econometrics = partial(process_docs, subtask="econometrics") +process_docs_astronomy = partial(process_docs, subtask="astronomy") +process_docs_business_ethics = partial(process_docs, subtask="business_ethics") +process_docs_high_school_macroeconomics = partial( + process_docs, subtask="high_school_macroeconomics" +) +process_docs_jurisprudence = partial(process_docs, subtask="jurisprudence") +process_docs_professional_psychology = partial( + process_docs, subtask="professional_psychology" +) +process_docs_high_school_chemistry = partial( + process_docs, subtask="high_school_chemistry" +) +process_docs_philosophy = partial(process_docs, subtask="philosophy") +process_docs_college_medicine = partial(process_docs, subtask="college_medicine") +process_docs_medical_genetics = partial(process_docs, subtask="medical_genetics") +process_docs_high_school_microeconomics = partial( + process_docs, subtask="high_school_microeconomics" +) +process_docs_high_school_geography = partial( + process_docs, subtask="high_school_geography" +) +process_docs_college_biology = partial(process_docs, subtask="college_biology") +process_docs_human_aging = partial(process_docs, subtask="human_aging") +process_docs_anatomy = partial(process_docs, subtask="anatomy") +process_docs_logical_fallacies = partial(process_docs, subtask="logical_fallacies") +process_docs_clinical_knowledge = partial(process_docs, subtask="clinical_knowledge") +process_docs_conceptual_physics = partial(process_docs, subtask="conceptual_physics") +process_docs_human_sexuality = partial(process_docs, subtask="human_sexuality") +process_docs_formal_logic = partial(process_docs, subtask="formal_logic") +process_docs_abstract_algebra = partial(process_docs, subtask="abstract_algebra") +process_docs_high_school_biology = partial(process_docs, subtask="high_school_biology") +process_docs_marketing = partial(process_docs, subtask="marketing") +process_docs_world_religions = partial(process_docs, subtask="world_religions") +process_docs_high_school_european_history = partial( + process_docs, subtask="high_school_european_history" +) +process_docs_college_computer_science = partial( + process_docs, subtask="college_computer_science" +) +process_docs_high_school_world_history = partial( + process_docs, subtask="high_school_world_history" +) +process_docs_prehistory = partial(process_docs, subtask="prehistory") +process_docs_high_school_mathematics = partial( + process_docs, subtask="high_school_mathematics" +) +process_docs_global_facts = partial(process_docs, subtask="global_facts") +process_docs_moral_scenarios = partial(process_docs, subtask="moral_scenarios") +process_docs_electrical_engineering = partial( + process_docs, subtask="electrical_engineering" +) +process_docs_management = partial(process_docs, subtask="management") +process_docs_elementary_mathematics = partial( + process_docs, subtask="elementary_mathematics" +) +process_docs_us_foreign_policy = partial(process_docs, subtask="us_foreign_policy") +process_docs_professional_medicine = partial( + process_docs, subtask="professional_medicine" +) +process_docs_college_physics = partial(process_docs, subtask="college_physics") +process_docs_high_school_government_and_politics = partial( + process_docs, subtask="high_school_government_and_politics" +) +process_docs_security_studies = partial(process_docs, subtask="security_studies") +process_docs_professional_law = partial(process_docs, subtask="professional_law") +process_docs_high_school_us_history = partial( + process_docs, subtask="high_school_us_history" +) +process_docs_virology = partial(process_docs, subtask="virology") +process_docs_nutrition = partial(process_docs, subtask="nutrition") +process_docs_college_chemistry = partial(process_docs, subtask="college_chemistry") +process_docs_computer_security = partial(process_docs, subtask="computer_security") +process_docs_public_relations = partial(process_docs, subtask="public_relations") +process_docs_moral_disputes = partial(process_docs, subtask="moral_disputes") +process_docs_college_mathematics = partial(process_docs, subtask="college_mathematics") +process_docs_high_school_psychology = partial( + process_docs, subtask="high_school_psychology" +) +process_docs_international_law = partial(process_docs, subtask="international_law") +process_docs_sociology = partial(process_docs, subtask="sociology") diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/_default_template_yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..31db83dc8a5467c4757f0bd6f56c1dcbebe02ec4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/_default_template_yaml @@ -0,0 +1,35 @@ +dataset_path: TIGER-Lab/MMLU-Pro +output_type: generate_until +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_target: !function utils.fewshot_to_text +doc_to_text: "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' %}Given the following question and candidate answers, choose the best answer.\nQuestion: {{question.strip()}}\n{% for choice in options %}{{letters[loop.index0]}}. {{choice}}\n{% endfor %}\nYour response should end with \"The best answer is [the_answer_letter].\" where the [the_answer_letter] is a letter from the provided choices.\n\nLet's think step by step." +doc_to_target: answer +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "\\$" + - "\\.$" +generation_kwargs: + do_sample: false + temperature: 0 + max_gen_toks: 1024 + until: [] +filter_list: + - name: strict_match + filter: + - function: "regex" + regex_pattern: "[tT]he best answer is ([A-Z])" + group_select: -1 + - function: take_first +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/_mmlu_pro.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/_mmlu_pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a4972283b89ae6038bdcb2458a8995fb9f8d065 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/_mmlu_pro.yaml @@ -0,0 +1,23 @@ +group: mmlu_pro_llama +task: + - mmlu_pro_llama_biology + - mmlu_pro_llama_business + - mmlu_pro_llama_chemistry + - mmlu_pro_llama_computer_science + - mmlu_pro_llama_economics + - mmlu_pro_llama_engineering + - mmlu_pro_llama_health + - mmlu_pro_llama_history + - mmlu_pro_llama_law + - mmlu_pro_llama_math + - mmlu_pro_llama_other + - mmlu_pro_llama_philosophy + - mmlu_pro_llama_physics + - mmlu_pro_llama_psychology +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: [strict_match] +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffcbffc8d63f02cd01218dcf88fc980f581bb804 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_biology.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_biology" +task_alias: "biology" +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_business.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdfe4ff84eb3bc2c2d9c97ef31c6297e60865c47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_business.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_business" +task_alias: "business" +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbb85149af42a5e0582a4d167ac34eb0e42563f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_chemistry.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_chemistry" +task_alias: "chemistry" +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7d1e14442b4cde5119cc827764528666e6834b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_computer_science.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_computer_science" +task_alias: "computer_science" +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f58272eb8e131773c92a2e0da0b291b040494f7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_economics.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_economics" +task_alias: "economics" +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb75ecb2b92fe81ce07be44c9fcb0d0c9d4b3517 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_engineering.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_engineering" +task_alias: "engineering" +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_health.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c95eba37835f53000a1cba3356c153947651a297 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_health.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_health" +task_alias: "health" +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dbe3b6831b13887eea8a5ca10a35baeb019a4e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_history.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_history" +task_alias: "history" +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3de3b6b77ee7b09abdaffe718bf8afbbe067319 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_law.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_law" +task_alias: "law" +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_math.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d78f4d43634b6d8a3d654d7357ad69c45d9890d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_math.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_math" +task_alias: "math" +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_other.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf7910c2e095d398b6954e29044a75b565983bb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_other.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bfe8772e0ebf956ad09a999c172e9cf8cb9c178 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_philosophy.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b95a8b1431fe72a20965492bb8dc39108bd55120 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_physics.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf3ad99828129cf32623855a32c5b7debb24d6f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_psychology.yaml @@ -0,0 +1,4 @@ +include: "_default_template_yaml" +task: "mmlu_pro_llama_psychology" +task_alias: "psychology" +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/utils.py b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4dfc24e03f5b71e64da0e7408dddfdc75749f1c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pro/utils.py @@ -0,0 +1,27 @@ +import re +from functools import partial + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +def fewshot_to_text(example): + text = example["cot_content"].removeprefix("A: Let's think step by step.").strip() + return re.sub(r"The answer is \(([A-Z])\)\.", r"The best answer is \1.", text) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_continuation_template_yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_continuation_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..23d13a6f7faa5e7bf60d50c814e961eec314cf72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_continuation_template_yaml @@ -0,0 +1,32 @@ +dataset_path: meta-llama/Llama-3.1-8B-Instruct-evals +dataset_name: Llama-3.1-8B-Instruct-evals__multilingual_mmlu_pt__details +output_type: generate_until +test_split: latest +doc_to_text: "Given the following question and four candidate answers (A, B, C and D), choose the best answer.\nQuestion: {{input_question.strip()}}\nA. {{input_choice_list.A}}\nB. {{input_choice_list.B}}\nC. {{input_choice_list.C}}\nD. {{input_choice_list.D}}\nYour response should end with \"The best answer is [the_answer_letter]\" where the [the_answer_letter] is one of A, B, C or D." +gen_prefix: "The best answer is" +doc_to_target: "{{input_correct_responses[0]}}." +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "\\$" + - "\\.$" +generation_kwargs: + do_sample: false + temperature: 0 + until: + - "." + max_gen_toks: 10 +filter_list: + - name: strict_match + filter: + - function: remove_whitespace + - function: take_first +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ae86ca8b46a42397f3ff5b0cb62ab102f067563 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_humanities.yaml @@ -0,0 +1,11 @@ +group: mmlu_pt_llama_humanities +group_alias: humanities +task: + - mmlu_pt_llama_humanities_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_llama.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_llama.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4afeb2705d89babeaac0409cbc41c4f64d5d127 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_llama.yaml @@ -0,0 +1,13 @@ +group: mmlu_pt_llama +task: + - mmlu_pt_llama_stem + - mmlu_pt_llama_other + - mmlu_pt_llama_social_sciences + - mmlu_pt_llama_humanities +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_other.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04113961528ab70c519b6c2aeea27c50550df4ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_other.yaml @@ -0,0 +1,11 @@ +group: mmlu_pt_llama_other +group_alias: other +task: + - mmlu_pt_llama_other_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bea231f2cb0ead2f6f79260e079feb7cd9e0d581 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_social_sciences.yaml @@ -0,0 +1,11 @@ +group: mmlu_pt_llama_social_sciences +group_alias: social sciences +task: + - mmlu_pt_llama_social_sciences_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e80999c54b1070e97618707495980d4a6f561404 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/_mmlu_pt_stem.yaml @@ -0,0 +1,11 @@ +group: mmlu_pt_llama_stem +group_alias: stem +task: + - mmlu_pt_llama_stem_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e73bae5002b56c9c21f20af12a8d12aedf50ab6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_abstract_algebra.yaml @@ -0,0 +1,54 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '0' + B: '1' + C: '2' + D: '3' + input_correct_responses: + - B + input_question: Encontre todo c em Z_3 tal que Z_3[x]/(x^2 + c) seja um corpo. + - input_choice_list: + A: Verdade verdade + B: Falso, Falso + C: Verdadeiro falso + D: Falso verdadeiro + input_correct_responses: + - B + input_question: "Declara\xE7\xE3o 1 | Se aH \xE9 um elemento de um grupo de fatores,\ + \ ent\xE3o |aH| divide |a|. Declara\xE7\xE3o 2 | Se H e K s\xE3o subgrupos de\ + \ G, ent\xE3o HK \xE9 um subgrupo de G." + - input_choice_list: + A: Verdade verdade + B: Falso, Falso + C: Verdadeiro falso + D: Falso verdadeiro + input_correct_responses: + - C + input_question: "Declara\xE7\xE3o 1 | Cada elemento de um grupo gera um subgrupo\ + \ c\xEDclico do grupo. Declara\xE7\xE3o 2 | O grupo sim\xE9trico S_10 possui\ + \ 10 elementos." + - input_choice_list: + A: Verdade verdade + B: Falso, Falso + C: Verdadeiro falso + D: Falso verdadeiro + input_correct_responses: + - A + input_question: "Declara\xE7\xE3o 1| Toda fun\xE7\xE3o de um conjunto finito sobre\ + \ si mesma deve ser injetora. Declara\xE7\xE3o 2 | Todo subgrupo de um grupo\ + \ abeliano \xE9 abeliano." + - input_choice_list: + A: '0' + B: '3' + C: '12' + D: '30' + input_correct_responses: + - A + input_question: "Encontre a caracter\xEDstica do anel 2Z." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_abstract_algebra +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_abstract_algebra +task_alias: abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b5358ef64f295ffa7830a782cc20091ee64fb8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_anatomy.yaml @@ -0,0 +1,50 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "O primeiro arco far\xEDngeo" + B: "O primeiro e segundo arcos far\xEDngeos" + C: "O segundo arco far\xEDngeo" + D: "O segundo e terceiro arcos far\xEDngeos" + input_correct_responses: + - D + input_question: "Qual \xE9 a origem embriol\xF3gica do osso hi\xF3ide?" + - input_choice_list: + A: O nervo supraorbital + B: O nervo infraorbital + C: O nervo mental + D: Nenhuma das acima + input_correct_responses: + - D + input_question: "Qual desses ramos do nervo trig\xEAmeo cont\xE9m processos motores\ + \ som\xE1ticos?" + - input_choice_list: + A: "n\xE3o t\xEAm inerva\xE7\xE3o sensorial." + B: "est\xE3o separados por um espa\xE7o de 2 mm." + C: "estender-se at\xE9 o pesco\xE7o." + D: "s\xE3o compostos por epit\xE9lio respirat\xF3rio." + input_correct_responses: + - C + input_question: A pleura + - input_choice_list: + A: sobremordida excessiva dos incisivos laterais superiores. + B: overjet negativo dos incisivos centrais superiores. + C: overjet excessivo dos incisivos laterais superiores. + D: overjet excessivo dos incisivos centrais superiores. + input_correct_responses: + - C + input_question: "Na oclus\xE3o Classe II Div 2 de Angle h\xE1" + - input_choice_list: + A: Abdominal + B: Craniano + C: Pleural + D: Espinhal + input_correct_responses: + - B + input_question: "Qual das alternativas a seguir \xE9 a cavidade corporal que cont\xE9\ + m a gl\xE2ndula pituit\xE1ria?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_anatomy +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_anatomy +task_alias: anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cac4306411d9ea60a7a36e87a53c0dd8c82fa8eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_astronomy.yaml @@ -0,0 +1,67 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Seria mais dif\xEDcil j\xE1 que o caminh\xE3o \xE9 mais pesado em Marte." + B: "Seria mais f\xE1cil j\xE1 que o caminh\xE3o \xE9 mais leve em Marte." + C: "Seria mais dif\xEDcil j\xE1 que o caminh\xE3o \xE9 mais leve em Marte." + D: "Seria o mesmo, n\xE3o importa onde voc\xEA esteja." + input_correct_responses: + - D + input_question: "Voc\xEA est\xE1 empurrando um caminh\xE3o por uma estrada. Seria\ + \ mais f\xE1cil acelerar este caminh\xE3o em Marte? Por que? (Suponha que n\xE3\ + o haja atrito)" + - input_choice_list: + A: "O cintur\xE3o de Kuiper; cometas de curto per\xEDodo tendem a estar no plano\ + \ do sistema solar, assim como o cintur\xE3o de Kuiper." + B: "O cintur\xE3o de Kuiper; cometas de curto per\xEDodo tendem a vir de dire\xE7\ + \xF5es aleat\xF3rias, indicando uma distribui\xE7\xE3o esf\xE9rica de cometas\ + \ chamada cintur\xE3o de Kuiper." + C: "O cintur\xE3o de aster\xF3ides; cometas de per\xEDodo curto t\xEAm per\xED\ + odos orbitais semelhantes aos de aster\xF3ides como Vesta e s\xE3o encontrados\ + \ no plano do sistema solar, assim como o cintur\xE3o de aster\xF3ides." + D: "A nuvem de Oort; cometas de curto per\xEDodo tendem a estar no plano do\ + \ sistema solar, assim como a nuvem de Oort." + input_correct_responses: + - A + input_question: "De onde vem a maioria dos cometas de curto per\xEDodo e como\ + \ sabemos?" + - input_choice_list: + A: 10.000 vezes mais + B: 100 vezes mais + C: 1000 vezes mais + D: 10 vezes mais + input_correct_responses: + - A + input_question: "Digamos que a pupila do seu olho tenha um di\xE2metro de 5 mm\ + \ e voc\xEA tenha um telesc\xF3pio com abertura de 50 cm. Quanta luz a mais\ + \ o telesc\xF3pio pode captar do que o seu olho?" + - input_choice_list: + A: "Um planeta j\xE1 se formou aqui, mas foi destru\xEDdo por uma colis\xE3\ + o catastr\xF3fica." + B: "N\xE3o havia material suficiente nesta parte da nebulosa solar para formar\ + \ um planeta." + C: "Havia muito material rochoso para formar um planeta terrestre, mas n\xE3\ + o material gasoso suficiente para formar um planeta joviano." + D: "A resson\xE2ncia com J\xFApiter impediu que o material se reunisse para\ + \ formar um planeta." + input_correct_responses: + - D + input_question: "Por que n\xE3o existe um planeta onde o cintur\xE3o de aster\xF3\ + ides esteja localizado?" + - input_choice_list: + A: "Porque a superf\xEDcie est\xE1 coberta por minerais fortemente oxidados\ + \ ("enferrujados")." + B: Porque a atmosfera espalha mais luz em comprimentos de onda mais azuis, transmitindo + principalmente luz vermelha. + C: "Porque Marte est\xE1 coberto por antigos fluxos de lava de cor vermelha." + D: "Porque a \xE1gua corrente na superf\xEDcie de Marte alterou os minerais\ + \ da superf\xEDcie h\xE1 v\xE1rios bilh\xF5es de anos." + input_correct_responses: + - A + input_question: "Por que Marte \xE9 vermelho?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_astronomy +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_astronomy +task_alias: astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59df451a1b798bf25b6c767dd67361aae0d2027b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_business_ethics.yaml @@ -0,0 +1,66 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Externalidades, Poder, Independ\xEAncia" + B: "Publicidade, Recursos insubstanciais, Depend\xEAncia m\xFAtua" + C: "Publicidade, Poder, Independ\xEAncia" + D: "Externalidades, Poder, Depend\xEAncia M\xFAtua" + input_correct_responses: + - D + input_question: "Al\xE9m do argumento comercial para o envolvimento na RSE, h\xE1\ + \ uma s\xE9rie de argumentos morais relacionados com: _______ negativo, o _______que\ + \ as empresas possuem e o ________ dos neg\xF3cios e da sociedade." + - input_choice_list: + A: Responsabilidade social corporativa + B: "Gest\xE3o de \xE9tica empresarial" + C: Sustentabilidade + D: "Gest\xE3o ambiental" + input_correct_responses: + - B + input_question: "_______ \xE9 a tentativa direta de gerir formal ou informalmente\ + \ quest\xF5es ou problemas \xE9ticos, atrav\xE9s de pol\xEDticas, pr\xE1ticas\ + \ e programas espec\xEDficos." + - input_choice_list: + A: Fora, limitado, independente + B: Dentro, Limitado, Intermitentemente + C: Fora, Ilimitado, Intermitentemente + D: Dentro, Ilimitado, Independentemente + input_correct_responses: + - A + input_question: "Para garantir a independ\xEAncia dos administradores n\xE3o executivos,\ + \ h\xE1 uma s\xE9rie de medidas que podem ser tomadas, que incluem a sele\xE7\ + \xE3o de n\xE3o executivos de _______ da sociedade, a nomea\xE7\xE3o por um\ + \ per\xEDodo de _________, bem como a nomea\xE7\xE3o de _________." + - input_choice_list: + A: "A\xE7\xE3o direta n\xE3o violenta, A\xE7\xE3o direta violenta, A\xE7\xE3\ + o indireta, Boicote" + B: "A\xE7\xE3o indireta, A\xE7\xE3o instrumental, A\xE7\xE3o direta n\xE3o violenta,\ + \ Campanha de informa\xE7\xE3o" + C: "A\xE7\xE3o indireta, a\xE7\xE3o direta violenta, boicote de a\xE7\xE3o direta\ + \ n\xE3o violenta" + D: "A\xE7\xE3o direta n\xE3o violenta, A\xE7\xE3o instrumental, A\xE7\xE3o indireta,\ + \ Campanha de informa\xE7\xE3o" + input_correct_responses: + - C + input_question: "Tr\xEAs t\xE1ticas contrastantes que os OSCs podem adotar para\ + \ atingir seus objetivos s\xE3o ________, que normalmente envolve pesquisa e\ + \ comunica\xE7\xE3o, ________, que pode envolver atacar fisicamente as opera\xE7\ + \xF5es de uma empresa, ou ________, muitas vezes envolvendo alguma forma de\ + \ _______." + - input_choice_list: + A: "Buycotts, boicotes, tecnologia Blockchain, doa\xE7\xF5es de caridade" + B: Compras, boicotes, tecnologia digital, aumento de vendas + C: "Boicotes, Buyalls, tecnologia Blockchain, doa\xE7\xF5es de caridade" + D: Boicotes, Buycotts, Tecnologia Digital, Aumento de Vendas + input_correct_responses: + - D + input_question: "Em contraste com _______, _______ visam recompensar o comportamento\ + \ favor\xE1vel das empresas. O sucesso de tais campanhas foi aumentado atrav\xE9\ + s do uso de ___________, que permite campanhas para facilitar a empresa a alcan\xE7\ + ar _________." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_business_ethics +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_business_ethics +task_alias: business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81f9d77f3505d5adccf5c716c35c50dd5f69afbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_clinical_knowledge.yaml @@ -0,0 +1,55 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ATP. + B: ADP. + C: fosfocreatina. + D: "fosforila\xE7\xE3o oxidativa." + input_correct_responses: + - A + input_question: "A energia para todas as formas de contra\xE7\xE3o muscular \xE9\ + \ fornecida por:" + - input_choice_list: + A: "Os cateteres masculinos e femininos s\xE3o de cores diferentes." + B: "Os cateteres masculinos s\xE3o mais longos que os cateteres femininos." + C: "Os cateteres masculinos s\xE3o maiores que os cateteres femininos." + D: "Os cateteres femininos s\xE3o mais longos que os masculinos." + input_correct_responses: + - B + input_question: "Qual \xE9 a diferen\xE7a entre um cateter masculino e um cateter\ + \ feminino?" + - input_choice_list: + A: "A abdu\xE7\xE3o do polegar \xE9 suprida pela raiz espinhal T2" + B: "A oposi\xE7\xE3o do polegar pelo oponente pol\xEDtico \xE9 suprida pela\ + \ raiz espinhal T1" + C: "A adu\xE7\xE3o do dedo \xE9 suprida pelo nervo mediano" + D: "A abdu\xE7\xE3o dos dedos \xE9 mediada pelos inter\xF3sseos palmares" + input_correct_responses: + - B + input_question: "Na avalia\xE7\xE3o da fun\xE7\xE3o da m\xE3o, qual das afirma\xE7\ + \xF5es a seguir \xE9 verdadeira?" + - input_choice_list: + A: '4' + B: '3' + C: '2' + D: '1' + input_correct_responses: + - C + input_question: "Quantas tentativas voc\xEA deve fazer para canular um paciente\ + \ antes de passar o trabalho para um colega s\xEAnior, de acordo com o conhecimento\ + \ m\xE9dico de 2020?" + - input_choice_list: + A: "glicog\xEAnio em glicose-1-fosfato." + B: "glicog\xEAnio ou glicose em frutose." + C: "glicog\xEAnio ou glicose em piruvato ou lactato." + D: "glicog\xEAnio ou glicose em piruvato ou acetil CoA." + input_correct_responses: + - C + input_question: "Glic\xF3lise \xE9 o nome dado \xE0 via que envolve a convers\xE3\ + o de:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_clinical_knowledge +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_clinical_knowledge +task_alias: clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27852b33716f1c30673a4627638a527778ae9bda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_biology.yaml @@ -0,0 +1,58 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Eles possuem um exoesqueleto composto principalmente de peptidoglicano. + B: "Possuem sistema circulat\xF3rio aberto com cora\xE7\xE3o dorsal." + C: "Eles s\xE3o membros de um filo biologicamente malsucedido, incapaz de explorar\ + \ diversos habitats e fontes de nutri\xE7\xE3o." + D: "Eles n\xE3o possuem ap\xEAndices emparelhados e articulados." + input_correct_responses: + - B + input_question: "Qual das alternativas a seguir representa uma afirma\xE7\xE3\ + o precisa sobre os artr\xF3podes?" + - input_choice_list: + A: 1/400 + B: 19/400 + C: 20/400 + D: 38/400 + input_correct_responses: + - D + input_question: "Numa determinada popula\xE7\xE3o, 1 em cada 400 pessoas tem um\ + \ cancro causado por um alelo completamente recessivo, b. Supondo que a popula\xE7\ + \xE3o esteja em equil\xEDbrio de Hardy-Weinberg, qual das alternativas a seguir\ + \ \xE9 a propor\xE7\xE3o esperada de indiv\xEDduos que carregam o alelo b, mas\ + \ n\xE3o se espera que desenvolvam o c\xE2ncer?" + - input_choice_list: + A: "o humano e o p\xE1ssaro s\xE3o esp\xE9cies polifil\xE9ticas" + B: "a evolu\xE7\xE3o de um ser humano e de um p\xE1ssaro \xE9 convergente" + C: "o humano e o p\xE1ssaro pertencem a um clado" + D: "o humano e o p\xE1ssaro desenvolvidos por analogia" + input_correct_responses: + - C + input_question: "A presen\xE7a de estruturas hom\xF3logas em dois organismos diferentes,\ + \ como o \xFAmero no membro anterior de um ser humano e de uma ave, indica que" + - input_choice_list: + A: "uma bomba de fluxo de press\xE3o dependente de ATP" + B: "um gradiente de potencial de press\xE3o da \xE1gua" + C: "transpira\xE7\xE3o" + D: "difus\xE3o apopl\xE1stica" + input_correct_responses: + - B + input_question: "De acordo com o modelo de fluxo de press\xE3o do movimento do\ + \ conte\xFAdo do floema, o movimento do fotossintato da fonte ao sumidouro \xE9\ + \ impulsionado por" + - input_choice_list: + A: "Tel\xF4meros" + B: "Centr\xF4meros" + C: Nucleossomos + D: Spliceossomas + input_correct_responses: + - B + input_question: "Qual das alternativas a seguir cont\xE9m sequ\xEAncias de DNA\ + \ necess\xE1rias para a segrega\xE7\xE3o dos cromossomos na mitose e na meiose?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_biology +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_college_biology +task_alias: college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cf24aa41457bd7c7e56b95e19d10136c451d5a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_chemistry.yaml @@ -0,0 +1,61 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "O estado de oxida\xE7\xE3o mais comum para os elementos lantan\xEDdeos \xE9\ + \ +3." + B: "Os complexos de lantan\xEDdeos geralmente apresentam n\xFAmeros de coordena\xE7\ + \xE3o elevados (> 6)." + C: "Todos os elementos lantan\xEDdeos reagem com \xE1cido aquoso para liberar\ + \ hidrog\xEAnio." + D: "Os raios at\xF4micos dos elementos lantan\xEDdeos aumentam ao longo do per\xED\ + odo de La a Lu." + input_correct_responses: + - D + input_question: "Qual das seguintes afirma\xE7\xF5es sobre os elementos lantan\xED\ + deos N\xC3O \xE9 verdadeira?" + - input_choice_list: + A: 1,0 mL + B: 10ml + C: 20ml + D: 50ml + input_correct_responses: + - C + input_question: "Uma amostra de 0,217 g de HgO (massa molar = 217 g) reage com\ + \ excesso de \xEDons iodeto de acordo com a rea\xE7\xE3o mostrada acima. A titula\xE7\ + \xE3o da solu\xE7\xE3o resultante requer quantos mL de HCl 0,10 M para atingir\ + \ o ponto de equival\xEAncia?" + - input_choice_list: + A: '4' + B: '3' + C: '6' + D: '24' + input_correct_responses: + - A + input_question: "Preveja o n\xFAmero de linhas no espectro EPR de uma solu\xE7\ + \xE3o de radical metila marcado com 13C (13CH3\u2022), assumindo que as linhas\ + \ n\xE3o se sobrep\xF5em." + - input_choice_list: + A: "um \xE1cido" + B: humilhar + C: um catalisador + D: um agente redutor + input_correct_responses: + - D + input_question: "3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq)\ + \ + 4 Cr3+(aq) + 10 H2O(l). Na rea\xE7\xE3o mostrada acima, Cl\u2212(aq) se\ + \ comporta como" + - input_choice_list: + A: PbH4 <SnH4 <GeH4 <SiH4 <CH4 + B: PbH4 <SnH4 <CH4 <GeH4 <SiH4 + C: CH4 <SiH4 <GeH4 <SnH4 <PbH4 + D: CH4 <PbH4 <GeH4 <SnH4 <SiH4 + input_correct_responses: + - A + input_question: "Qual das alternativas a seguir lista os hidretos dos elementos\ + \ do grupo 14 em ordem de estabilidade t\xE9rmica, do menor para o maior?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_chemistry +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_college_chemistry +task_alias: college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23ac8663e632cca325a74a2c95ccf35b40ef12ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_computer_science.yaml @@ -0,0 +1,83 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: uma*(c + d)+ b(c + d) + B: uma*(c + d)* + b(c + d)* + C: uma*(c + d)+ b*(c + d) + D: (a + b)*c +(a + b)*d + input_correct_responses: + - D + input_question: "Qual das seguintes express\xF5es regulares \xE9 equivalente a\ + \ (descreve o mesmo conjunto de strings que) (a* + b)*(c + d)?" + - input_choice_list: + A: '5' + B: '6' + C: '7' + D: '8' + input_correct_responses: + - B + input_question: "Uma certa m\xE1quina RISC em pipeline possui 8 registradores\ + \ de uso geral R0, R1,. . . , R7 e suporta as seguintes opera\xE7\xF5es. ADD\ + \ Rs1, Rs2, Rd Adicione Rs1 a Rs2 e coloque a soma em Rd MUL Rs1, Rs2, Rd Multiplique\ + \ Rs1 por Rs2 e coloque o produto em Rd Uma opera\xE7\xE3o normalmente leva\ + \ um ciclo; entretanto, uma opera\xE7\xE3o leva dois ciclos se produzir um resultado\ + \ exigido pela opera\xE7\xE3o imediatamente seguinte em uma sequ\xEAncia de\ + \ opera\xE7\xF5es. Considere a express\xE3o AB + ABC + BC, onde as vari\xE1\ + veis A, B, C est\xE3o localizadas nos registradores R0, R1, R2. Se o conte\xFA\ + do desses tr\xEAs registradores n\xE3o deve ser modificado, qual \xE9 o n\xFA\ + mero m\xEDnimo de ciclos de clock necess\xE1rios para uma sequ\xEAncia de opera\xE7\ + \xF5es que calcule o valor de AB + ABC + BC?" + - input_choice_list: + A: eu apenas + B: II apenas + C: III apenas + D: I, II e III + input_correct_responses: + - D + input_question: "O padr\xE3o de design Singleton \xE9 usado para garantir que\ + \ apenas uma \xFAnica inst\xE2ncia de uma classe possa ser instanciada. Qual\ + \ das afirma\xE7\xF5es a seguir \xE9 (s\xE3o) verdadeira para esse padr\xE3\ + o de design? I. A classe Singleton possui um m\xE9todo de f\xE1brica est\xE1\ + tico para fornecer sua inst\xE2ncia. II. A classe Singleton pode ser uma subclasse\ + \ de outra classe. III. A classe Singleton possui um construtor privado." + - input_choice_list: + A: '5' + B: '6' + C: '7' + D: '9' + input_correct_responses: + - D + input_question: "Um compilador gera c\xF3digo para a seguinte instru\xE7\xE3o\ + \ de atribui\xE7\xE3o. G := (A + B) * C - (D + E) * F A m\xE1quina alvo possui\ + \ um \xFAnico acumulador e um conjunto de instru\xE7\xF5es de endere\xE7o \xFA\ + nico que consiste em instru\xE7\xF5es de carga, armazenamento, adi\xE7\xE3o,\ + \ subtra\xE7\xE3o e multiplica\xE7\xE3o. Para as opera\xE7\xF5es aritm\xE9ticas,\ + \ o operando esquerdo \xE9 retirado do acumulador e o resultado aparece no acumulador.\ + \ O menor n\xFAmero poss\xEDvel de instru\xE7\xF5es no c\xF3digo resultante\ + \ \xE9" + - input_choice_list: + A: 1/50 + B: 27/01 + C: 25/01 + D: 27/02 + input_correct_responses: + - B + input_question: "Considere um projeto de computador no qual v\xE1rios processadores,\ + \ cada um com uma mem\xF3ria cache privada, compartilham mem\xF3ria global usando\ + \ um \xFAnico barramento. Este barramento \xE9 o recurso cr\xEDtico do sistema.\ + \ Cada processador pode executar uma instru\xE7\xE3o a cada 500 nanossegundos,\ + \ desde que as refer\xEAncias de mem\xF3ria sejam satisfeitas pelo seu cache\ + \ local. Quando ocorre uma falta de cache, o processador \xE9 atrasado por mais\ + \ 2.000 nanossegundos. Durante metade desse atraso adicional, o barramento \xE9\ + \ dedicado a atender a perda de cache. Durante a outra metade, o processador\ + \ n\xE3o pode continuar, mas o barramento fica livre para atender solicita\xE7\ + \xF5es de outros processadores. Em m\xE9dia, cada instru\xE7\xE3o requer 2 refer\xEA\ + ncias de mem\xF3ria. Em m\xE9dia, as perdas de cache ocorrem em 1% das refer\xEA\ + ncias. Que propor\xE7\xE3o da capacidade do barramento um \xFAnico processador\ + \ consumiria, ignorando os atrasos devidos \xE0 concorr\xEAncia de outros processadores?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_computer_science +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_college_computer_science +task_alias: college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee47d0e1a83fc64b5a859a0755f82373f2b13350 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_mathematics.yaml @@ -0,0 +1,63 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ST = 0 + B: ST = T + C: ST = TS + D: "ST - TS \xE9 o mapa de identidade de V sobre si mesmo." + input_correct_responses: + - D + input_question: "Seja V o conjunto de todos os polin\xF4mios reais p(x). Deixe\ + \ as transforma\xE7\xF5es T, S serem definidas em V por T:p(x) -> xp(x) e\ + \ S:p(x) -> p'(x) = d/dx p(x), e interprete (ST) (p(x)) como S(T(p(x))).\ + \ Qual dos seguintes \xE9 verdadeiro?" + - input_choice_list: + A: '2' + B: 2-e^-2 + C: 2 + e^-2 + D: 2 + e^-4 + input_correct_responses: + - D + input_question: "Um tanque cont\xE9m inicialmente uma solu\xE7\xE3o salina de\ + \ 3 gramas de sal dissolvidos em 100 litros de \xE1gua. Uma solu\xE7\xE3o salina\ + \ contendo 0,02 gramas de sal por litro de \xE1gua \xE9 pulverizada no tanque\ + \ a uma taxa de 4 litros por minuto. A solu\xE7\xE3o pulverizada \xE9 continuamente\ + \ misturada com a solu\xE7\xE3o salina no tanque, e a mistura flui para fora\ + \ do tanque a uma taxa de 4 litros por minuto. Se a mistura for instant\xE2\ + nea, quantos gramas de sal h\xE1 no tanque ap\xF3s decorridos 100 minutos?" + - input_choice_list: + A: eu apenas + B: II apenas + C: III apenas + D: Apenas II e III + input_correct_responses: + - B + input_question: "Seja A uma matriz 2x2 real. Qual das seguintes afirma\xE7\xF5\ + es deve ser verdadeira? I. Todas as entradas de A^2 s\xE3o n\xE3o negativas.\ + \ II. O determinante de A ^ 2 n\xE3o \xE9 negativo. III. Se A tem dois autovalores\ + \ distintos, ent\xE3o A^2 tem dois autovalores distintos." + - input_choice_list: + A: '-11' + B: '0' + C: '11' + D: 33/2 + input_correct_responses: + - C + input_question: "Suponha que f(1 + x) = f(x) para todo x real. Se f \xE9 um polin\xF4\ + mio e f(5) = 11, ent\xE3o f(15/2)" + - input_choice_list: + A: '-5' + B: '-4' + C: '-3' + D: '-2' + input_correct_responses: + - B + input_question: "Seja A o conjunto de todos os pares ordenados de inteiros (m,\ + \ n) tais que 7m + 12n = 22. Qual \xE9 o maior n\xFAmero negativo no conjunto\ + \ B = {m + n : (m, n) \\in A}?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_mathematics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_college_mathematics +task_alias: college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c18a817015b07fad8c26aac57b0256c908bf83bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_medicine.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "atrav\xE9s de transportadores de prote\xEDnas chamados GLUT4." + B: "apenas na presen\xE7a de insulina." + C: via hexoquinase. + D: "atrav\xE9s de transportadores de \xE1cido monocarb\xEDlico." + input_correct_responses: + - A + input_question: "A glicose \xE9 transportada para a c\xE9lula muscular:" + - input_choice_list: + A: "O glicog\xEAnio muscular \xE9 decomposto enzimaticamente em glicose-1-fosfato" + B: "Corredores de resist\xEAncia de elite t\xEAm uma alta propor\xE7\xE3o de\ + \ fibras do Tipo I nos m\xFAsculos das pernas" + C: "O glicog\xEAnio hep\xE1tico \xE9 importante na manuten\xE7\xE3o da concentra\xE7\ + \xE3o de glicose no sangue" + D: "A insulina promove a capta\xE7\xE3o de glicose por todos os tecidos do corpo" + input_correct_responses: + - D + input_question: "Qual das alternativas a seguir n\xE3o \xE9 uma afirma\xE7\xE3\ + o verdadeira?" + - input_choice_list: + A: "Todos os descendentes do lado materno ter\xE3o o transtorno." + B: "As mulheres ser\xE3o aproximadamente duas vezes mais afetadas que os homens\ + \ nesta fam\xEDlia." + C: "Todas as filhas de um homem afetado ser\xE3o afetadas." + D: "Haver\xE1 distribui\xE7\xE3o igual de homens e mulheres afetados." + input_correct_responses: + - C + input_question: "Em um teste gen\xE9tico de um rec\xE9m-nascido, \xE9 encontrada\ + \ uma doen\xE7a gen\xE9tica rara que tem transmiss\xE3o recessiva ligada ao\ + \ X. Qual das seguintes afirma\xE7\xF5es \xE9 provavelmente verdadeira em rela\xE7\ + \xE3o ao pedigree deste transtorno?" + - input_choice_list: + A: "Aumentando a temperatura, aumentando o n\xFAmero de moles de g\xE1s" + B: Aumentando a temperatura, aumentando o volume + C: Diminuindo o volume, diminuindo a temperatura + D: "Diminuindo moles de g\xE1s, aumentando o volume" + input_correct_responses: + - A + input_question: "Um professor de ci\xEAncias do ensino m\xE9dio enche uma garrafa\ + \ de 1 litro com nitrog\xEAnio puro e fecha a tampa. A press\xE3o \xE9 1,70\ + \ atm e a temperatura ambiente \xE9 25\xB0C. Quais s\xE3o as duas vari\xE1veis\ + \ que aumentar\xE3o a press\xE3o do sistema, se todas as outras vari\xE1veis\ + \ forem mantidas constantes?" + - input_choice_list: + A: fraqueza muscular. + B: ganho de massa corporal. + C: "c\xE3ibras musculares." + D: "perda de eletr\xF3litos." + input_correct_responses: + - B + input_question: "Um efeito colateral esperado da suplementa\xE7\xE3o de creatina\ + \ \xE9:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_medicine +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_college_medicine +task_alias: college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59ca81fe5e34a8fdcbc0c71f1fc61b6a86484c90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_college_physics.yaml @@ -0,0 +1,63 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '4' + B: '5' + C: '6' + D: '20' + input_correct_responses: + - A + input_question: "Um telesc\xF3pio refrator consiste em duas lentes convergentes\ + \ separadas por 100 cm. A lente ocular tem uma dist\xE2ncia focal de 20 cm.\ + \ A amplia\xE7\xE3o angular do telesc\xF3pio \xE9" + - input_choice_list: + A: Temperatura constante + B: Volume constante + C: "Press\xE3o constante" + D: "Adiab\xE1tico" + input_correct_responses: + - B + input_question: "Para qual dos seguintes processos termodin\xE2micos o aumento\ + \ na energia interna de um g\xE1s ideal \xE9 igual ao calor adicionado ao g\xE1\ + s?" + - input_choice_list: + A: 2,4V + B: 3,3V + C: 4,5V + D: 5,7V + input_correct_responses: + - A + input_question: "Uma extremidade de um fio de nicromo de comprimento 2L e \xE1\ + rea de se\xE7\xE3o transversal A est\xE1 presa a uma extremidade de outro fio\ + \ de nicromo de comprimento L e \xE1rea de se\xE7\xE3o transversal 2A. Se a\ + \ extremidade livre do fio mais longo estiver com um potencial el\xE9trico de\ + \ 8,0 volts, e a extremidade livre do fio mais curto estiver com um potencial\ + \ el\xE9trico de 1,0 volt, o potencial na jun\xE7\xE3o dos dois fios ser\xE1\ + \ quase igual a" + - input_choice_list: + A: '4' + B: '5' + C: '6' + D: '20' + input_correct_responses: + - A + input_question: "Um telesc\xF3pio refrator consiste em duas lentes convergentes\ + \ separadas por 100 cm. A lente ocular tem uma dist\xE2ncia focal de 20 cm.\ + \ A amplia\xE7\xE3o angular do telesc\xF3pio \xE9" + - input_choice_list: + A: cobrar + B: massa + C: energia e impulso + D: "n\xFAmero lept\xF4nico" + input_correct_responses: + - D + input_question: "O m\xFAon decai com um tempo de vida caracter\xEDstico de cerca\ + \ de 10 ^ -6 segundos em um el\xE9tron, um neutrino do m\xFAon e um antineutrino\ + \ do el\xE9tron. O m\xFAon est\xE1 proibido de decair em um el\xE9tron e em\ + \ apenas um \xFAnico neutrino pela lei da conserva\xE7\xE3o de" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_physics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_college_physics +task_alias: college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..309f7d94612e3cfc64b98fb7ce032f98b16446b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_computer_security.yaml @@ -0,0 +1,55 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: 160 bits + B: 512 bits + C: 628 bits + D: 820 bits + input_correct_responses: + - A + input_question: SHA-1 tem um resumo da mensagem de + - input_choice_list: + A: "IM \u2013 Trojans" + B: Trojans de backdoor + C: Downloader de Trojan + D: Trojan de resgate + input_correct_responses: + - D + input_question: "_____________ pode modificar dados em seu sistema \u2013 para\ + \ que seu sistema n\xE3o funcione corretamente ou voc\xEA n\xE3o possa mais\ + \ acessar dados espec\xEDficos, ou pode at\xE9 pedir resgate para conceder seu\ + \ acesso." + - input_choice_list: + A: "\xC9tica de \u201Chacking\u201D para justificar comportamento ego\xEDsta\ + \ n\xE3o intencional" + B: "Hackear sistemas (por exemplo, durante testes de penetra\xE7\xE3o) para\ + \ expor vulnerabilidades para que possam ser corrigidas, em vez de exploradas" + C: "Invadir sistemas administrados por pessoas cuja \xE9tica voc\xEA discorda" + D: "Uma g\xEDria para desenvolvimento r\xE1pido de software, por exemplo, como\ + \ parte de hackathons" + input_correct_responses: + - B + input_question: "O que \xE9 hacking \xE9tico?" + - input_choice_list: + A: "sobrescrevendo chaves criptogr\xE1ficas na mem\xF3ria" + B: "uma esp\xE9cie de inje\xE7\xE3o de c\xF3digo" + C: uma leitura fora dos limites de um buffer + D: um ataque de string de formato + input_correct_responses: + - C + input_question: "A explora\xE7\xE3o do bug Heartbleed permite" + - input_choice_list: + A: Teia assombrada + B: Rede mundial de computadores + C: "Superf\xEDcie da web" + D: Rede profunda + input_correct_responses: + - D + input_question: "O ____________ \xE9 qualquer coisa que seu mecanismo de pesquisa\ + \ n\xE3o consegue pesquisar." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_computer_security +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_computer_security +task_alias: computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50120cf0504d4d4cad3cb341a01beb9c6d25c67e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_conceptual_physics.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: menos + B: mais + C: o mesmo + D: zero + input_correct_responses: + - A + input_question: "Comparadas com a massa de um \xE1tomo de ur\xE2nio em fiss\xE3\ + o, as massas combinadas dos produtos ap\xF3s a fiss\xE3o s\xE3o" + - input_choice_list: + A: "espa\xE7o e tempo." + B: "um g\xEAmeo viajante e um g\xEAmeo que fica em casa." + C: "gravidade e acelera\xE7\xE3o." + D: massa e energia. + input_correct_responses: + - C + input_question: "Coisas que s\xE3o equivalentes de acordo com o princ\xEDpio da\ + \ equival\xEAncia s\xE3o" + - input_choice_list: + A: "convertido para uma frequ\xEAncia diferente" + B: "deflex\xE3o" + C: "interfer\xEAncia" + D: "polariza\xE7\xE3o" + input_correct_responses: + - C + input_question: "As cores em uma bolha de sab\xE3o resultam da luz" + - input_choice_list: + A: o mesmo + B: maior + C: menos + D: maior ou menor dependendo da velocidade do vento + input_correct_responses: + - B + input_question: "Um aeromodelo voa mais devagar quando voa contra o vento e mais\ + \ r\xE1pido com o vento nas costas. Quando lan\xE7ado em \xE2ngulo reto com\ + \ o vento, em um vento cruzado, sua velocidade no solo, em compara\xE7\xE3o\ + \ com o v\xF4o no ar parado, \xE9" + - input_choice_list: + A: "Hidrog\xEAnio" + B: Ferro + C: "Ur\xE2nio" + D: Igual em cada + input_correct_responses: + - A + input_question: "Qual desses tr\xEAs elementos tem mais massa por n\xFAcleon?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_conceptual_physics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_conceptual_physics +task_alias: conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6c6c3940ab6d0bb2ef3d1d1b7a2479ebe003846 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_econometrics.yaml @@ -0,0 +1,64 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Eventualmente morrer + B: Persistir indefinidamente + C: Crescer exponencialmente + D: Nunca ocorre + input_correct_responses: + - A + input_question: "Para um processo autorregressivo estacion\xE1rio, os choques" + - input_choice_list: + A: 0,2 + B: 0,4 + C: 0,5 + D: 0,33 + input_correct_responses: + - D + input_question: "Considere o seguinte modelo AR(1) com os dist\xFArbios tendo\ + \ m\xE9dia zero e vari\xE2ncia unit\xE1ria yt = 0,2 + 0,4 yt-1 + ut A m\xE9\ + dia (incondicional) de y ser\xE1 dada por" + - input_choice_list: + A: (ii) e (iv) apenas + B: (i) e (iii) apenas + C: (i), (ii) e (iii) apenas + D: (i), (ii), (iii) e (iv) + input_correct_responses: + - C + input_question: "Suponha que uma estat\xEDstica de teste tenha associado a ela\ + \ um valor p de 0,08. Qual das seguintes afirma\xE7\xF5es \xE9 verdadeira? (i)\ + \ Se o tamanho do teste fosse exatamente 8%, ser\xEDamos indiferentes entre\ + \ rejeitar ou n\xE3o rejeitar a hip\xF3tese nula (ii) O nulo seria rejeitado\ + \ se um tamanho de teste de 10% fosse usado (iii) O nulo n\xE3o seria seria\ + \ rejeitado se um tamanho de teste de 1% fosse usado (iv) O valor nulo seria\ + \ rejeitado se um tamanho de teste de 5% fosse usado." + - input_choice_list: + A: "Ser\xE1 tendencioso" + B: "Ser\xE1 inconsistente" + C: "Ser\xE1 ineficiente" + D: "Todos os itens (a), (b) e (c) ser\xE3o verdadeiros." + input_correct_responses: + - C + input_question: "Quais seriam ent\xE3o as consequ\xEAncias para o estimador OLS\ + \ se a heterocedasticidade estivesse presente em um modelo de regress\xE3o,\ + \ mas fosse ignorada?" + - input_choice_list: + A: 1 atraso + B: 2 atrasos + C: 3 atrasos + D: 4 atrasos + input_correct_responses: + - C + input_question: "Suponha agora que um pesquisador deseja usar crit\xE9rios de\ + \ informa\xE7\xE3o para determinar a dura\xE7\xE3o ideal da defasagem para um\ + \ VAR. 500 observa\xE7\xF5es est\xE3o dispon\xEDveis para o VAR bivariado, e\ + \ os valores do determinante da matriz de vari\xE2ncia-covari\xE2ncia dos res\xED\ + duos s\xE3o 0,0336, 0,0169, 0,0084 e 0,0062 para 1, 2, 3 e 4 defasagens, respectivamente.\ + \ Qual \xE9 a ordem ideal do modelo de acordo com o crit\xE9rio de informa\xE7\ + \xE3o de Akaike?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_econometrics +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_econometrics +task_alias: econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c8989a2b4c34f476ded2bdbd06d918fe4f31bef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_electrical_engineering.yaml @@ -0,0 +1,58 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: S=0, R=0 + B: S=0, R=1 + C: S = 1, R = 0 + D: S = 1, R = 1 + input_correct_responses: + - D + input_question: "Em uma trava SR constru\xEDda a partir de portas NOR, cuja condi\xE7\ + \xE3o n\xE3o \xE9 permitida" + - input_choice_list: + A: "200\u03A9" + B: "100\u03A9" + C: "50\u03A9" + D: "10\u03A9" + input_correct_responses: + - C + input_question: "Em uma m\xE1quina CC com enrolamento de volta de 2 p\xF3los,\ + \ a resist\xEAncia de um condutor \xE9 2\u03A9 e o n\xFAmero total de condutores\ + \ \xE9 100. Encontre a resist\xEAncia total" + - input_choice_list: + A: 1mA. + B: 2 mA. + C: 3mA. + D: 4mA. + input_correct_responses: + - B + input_question: "A bobina de um medidor de bobina m\xF3vel tem 100 voltas, 40\ + \ mm de comprimento e 30 mm de largura. O torque de controle \xE9 240*10-6 Nm\ + \ em escala completa. Se a densidade do fluxo magn\xE9tico for 1Wb/m2, a faixa\ + \ do medidor \xE9" + - input_choice_list: + A: 100 N. + B: 0,1 N. + C: 1N. + D: 0,01N. + input_correct_responses: + - B + input_question: "Dois longos condutores paralelos transportam 100 A. Se os condutores\ + \ estiverem separados por 20 mm, a for\xE7a por metro de comprimento de cada\ + \ condutor ser\xE1" + - input_choice_list: + A: 15 N. + B: 20 N. + C: 7,5 N. + D: 3,75 N. + input_correct_responses: + - A + input_question: "Um poste pontual tem uma resist\xEAncia de 4\u03C0 * 10 ^ -4\ + \ weber. A for\xE7a em newtons em um p\xF3lo pontual de 4\u03C0 * 1,5 * 10 ^\ + \ -4 weber colocado a uma dist\xE2ncia de 10 cm dele ser\xE1" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_electrical_engineering +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_electrical_engineering +task_alias: electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aec34cead108ce0866dc3a6e5b86394535d4e84a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_elementary_mathematics.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: 5 mil + B: 5 centenas + C: 5 dezenas + D: 5 uns + input_correct_responses: + - A + input_question: "A popula\xE7\xE3o da cidade onde Michelle nasceu \xE9 de 145.826\ + \ habitantes. Qual \xE9 o valor de 5 no n\xFAmero 145.826?" + - input_choice_list: + A: "O d\xE9cimo n\xFAmero do padr\xE3o ser\xE1 um n\xFAmero par." + B: "O padr\xE3o num\xE9rico nunca ter\xE1 dois n\xFAmeros pares pr\xF3ximos\ + \ um do outro." + C: "Os pr\xF3ximos dois n\xFAmeros no padr\xE3o ser\xE3o um n\xFAmero par e\ + \ depois um n\xFAmero \xEDmpar." + D: "Se o padr\xE3o num\xE9rico come\xE7asse com um n\xFAmero \xEDmpar, o padr\xE3\ + o conteria apenas n\xFAmeros \xEDmpares." + input_correct_responses: + - B + input_question: "Olivia usou a regra "Adicionar 11" para criar o padr\xE3\ + o num\xE9rico mostrado abaixo. 10, 21, 32, 43, 54 Qual afirma\xE7\xE3o sobre\ + \ o padr\xE3o num\xE9rico \xE9 verdadeira?" + - input_choice_list: + A: Adicione 5 a 30 para encontrar 35 equipes. + B: Divida 30 por 5 para encontrar 6 equipes. + C: Multiplique 30 por 5 para encontrar 150 equipes. + D: Subtraia 5 de 30 para encontrar 25 equipes. + input_correct_responses: + - B + input_question: "Um total de 30 jogadores jogar\xE3o basquete em um parque. Haver\xE1\ + \ exatamente 5 jogadores em cada equipe. Qual afirma\xE7\xE3o explica corretamente\ + \ como encontrar o n\xFAmero de equipes necess\xE1rias?" + - input_choice_list: + A: '749' + B: '2.675' + C: '2.945' + D: '4.250' + input_correct_responses: + - B + input_question: "Uma loja vende 107 cores diferentes de tinta. Eles t\xEAm 25\ + \ latas de cada cor armazenadas. A quantidade de latas de tinta que a loja possui\ + \ armazenada pode ser encontrada pela express\xE3o abaixo. 107 \xD7 25. Quantas\ + \ latas de tinta a loja tem armazenadas?" + - input_choice_list: + A: (5x4)x(6x5) + B: (5x5) + (5x4) + C: (5x5) + (5x9) + D: (5x9)x(6x9) + input_correct_responses: + - B + input_question: "Qual express\xE3o \xE9 equivalente a 5 x 9?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_elementary_mathematics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_elementary_mathematics +task_alias: elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d875e0579f43e9eb6642fe1633e12a9b4cdc21bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_formal_logic.yaml @@ -0,0 +1,64 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ~Pd + B: "(\u2200x)(Px \u2228 ~Dx)" + C: "(\u2200x)(Px \u2283 ~Dx)" + D: ~Dp + input_correct_responses: + - C + input_question: "Selecione a melhor tradu\xE7\xE3o para a l\xF3gica de predicados:\ + \ Ningu\xE9m dirige em Marte." + - input_choice_list: + A: Blgh + B: Bhlg + C: Bglh + D: Bghl + input_correct_responses: + - C + input_question: "Selecione a melhor tradu\xE7\xE3o para a l\xF3gica de predicados.\ + \ George pega emprestado o cortador de grama de Hector. (g: George; h: Hector;\ + \ l: cortador de grama de Hector; Bxyx: x empresta y de z)" + - input_choice_list: + A: "Marina \xE9 dan\xE7arina. Alguns fracos n\xE3o s\xE3o dan\xE7arinos. Ou\ + \ tudo \xE9 fraco ou a Ge\xF3rgia joga v\xF4lei. Ent\xE3o algo joga v\xF4\ + lei." + B: "Marina \xE9 dan\xE7arina. Nenhum fraco \xE9 dan\xE7arino. Tudo ou \xE9 fraco\ + \ ou joga v\xF4lei. Ent\xE3o algo joga v\xF4lei." + C: "Marina \xE9 dan\xE7arina. Alguns fracos n\xE3o s\xE3o dan\xE7arinos. Tudo\ + \ ou \xE9 fraco ou joga v\xF4lei. Ent\xE3o algo joga v\xF4lei." + D: "Marina \xE9 dan\xE7arina. Nenhum fraco \xE9 dan\xE7arino. Ou tudo \xE9 fraco\ + \ ou a Ge\xF3rgia joga v\xF4lei. Ent\xE3o algo joga v\xF4lei." + input_correct_responses: + - D + input_question: "Selecione a melhor interpreta\xE7\xE3o em ingl\xEAs dos argumentos\ + \ fornecidos na l\xF3gica de predicados. Dm (\u2200x)(Wx \u2283 ~Dx) (\u2200\ + x)Wx \u2228 Ag / (\u2203x)Ax" + - input_choice_list: + A: Logicamente equivalente + B: "Contradit\xF3rio" + C: "Nem logicamente equivalente nem contradit\xF3rio, mas consistente" + D: Inconsistente + input_correct_responses: + - C + input_question: "Construa uma tabela verdade completa para os seguintes pares\ + \ de proposi\xE7\xF5es. Ent\xE3o, usando as tabelas verdade, determine se as\ + \ afirma\xE7\xF5es s\xE3o logicamente equivalentes ou contradit\xF3rias. Se\ + \ n\xE3o, determine se s\xE3o consistentes ou inconsistentes. Justifique suas\ + \ respostas. E \u2283 (F \xB7 E) e ~E \xB7 F" + - input_choice_list: + A: "(L \u2022 H) \u2261 I" + B: "(L \u2022 H) \u2228 I" + C: "L \u2022 (H \u2228 I)" + D: "L \u2022 (H \u2283 R)" + input_correct_responses: + - B + input_question: "Qual das f\xF3rmulas fornecidas de PL \xE9 a melhor simboliza\xE7\ + \xE3o da frase a seguir? As tartarugas vivem vidas longas e s\xE3o criaturas\ + \ felizes, a menos que sejam feridas." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_formal_logic +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_formal_logic +task_alias: formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6cc25249a5ae6c85988c892cbbf0fed32869306 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_global_facts.yaml @@ -0,0 +1,57 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "As pessoas tendem a ser optimistas quanto ao seu pr\xF3prio futuro e ao\ + \ futuro da sua na\xE7\xE3o ou do mundo." + B: "As pessoas tendem a ser optimistas quanto ao seu pr\xF3prio futuro, mas\ + \ pessimistas quanto ao futuro da sua na\xE7\xE3o ou do mundo." + C: "As pessoas tendem a ser pessimistas quanto ao seu pr\xF3prio futuro, mas\ + \ otimistas quanto ao futuro da sua na\xE7\xE3o ou do mundo." + D: "As pessoas tendem a ser pessimistas quanto ao seu pr\xF3prio futuro e ao\ + \ futuro da sua na\xE7\xE3o ou do mundo." + input_correct_responses: + - B + input_question: "Quais dos seguintes pares de afirma\xE7\xF5es s\xE3o verdadeiros\ + \ (em 2019)?" + - input_choice_list: + A: 31% + B: 46% + C: 61% + D: 76% + input_correct_responses: + - B + input_question: "Em 2019, que percentagem de americanos concorda que o estado\ + \ \xE9 administrado para o benef\xEDcio de todo o povo?" + - input_choice_list: + A: aumentou, aumentou + B: Aumento diminuido + C: diminuiu, aumentou + D: diminuiu, diminuiu + input_correct_responses: + - B + input_question: "Em 2015, desde 1990, as florestas tinham ____ na Europa e ____\ + \ na \xC1frica e nas Am\xE9ricas." + - input_choice_list: + A: 38% + B: 53% + C: 68% + D: 83% + input_correct_responses: + - A + input_question: "Em 2019, que percentagem de russos afirma que \xE9 muito importante\ + \ ter meios de comunica\xE7\xE3o livres no nosso pa\xEDs, sem censura governamental/estatal?" + - input_choice_list: + A: 80% + B: 60% + C: 40% + D: 20% + input_correct_responses: + - A + input_question: "Em 2017, quantas crian\xE7as de 1 ano de idade no mundo foram\ + \ vacinadas contra alguma doen\xE7a? *" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_global_facts +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_global_facts +task_alias: global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e46402d970d2106e95fc5f7e9cdf9cd6507afd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_biology.yaml @@ -0,0 +1,62 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Tradu\xE7\xE3o" + B: "Conjuga\xE7\xE3o" + C: "Transdu\xE7\xE3o especializada" + D: "Transforma\xE7\xE3o" + input_correct_responses: + - A + input_question: "Qual das alternativas a seguir n\xE3o \xE9 uma forma de formar\ + \ DNA recombinante?" + - input_choice_list: + A: Ao alterar o pH ideal da enzima + B: "Ao alterar a localiza\xE7\xE3o da enzima na c\xE9lula" + C: "Mudando a forma da prote\xEDna" + D: "Uma mudan\xE7a de amino\xE1cido longe do s\xEDtio ativo n\xE3o pode alterar\ + \ a especificidade do substrato da enzima." + input_correct_responses: + - C + input_question: "Uma muta\xE7\xE3o em uma enzima bacteriana transformou um amino\xE1\ + cido anteriormente polar em um amino\xE1cido apolar. Este amino\xE1cido estava\ + \ localizado em um local distante do s\xEDtio ativo da enzima. Como essa muta\xE7\ + \xE3o poderia alterar a especificidade do substrato da enzima?" + - input_choice_list: + A: "Membrana plasm\xE1tica \u2013 aparelho de Golgi \u2013 ribossomo \u2013\ + \ ves\xEDcula secretora \u2013 RE rugoso" + B: "Ribossomo \u2013 aparelho de Golgi \u2013 RE rugoso \u2013 ves\xEDcula secretora\ + \ \u2013 membrana plasm\xE1tica" + C: "Membrana plasm\xE1tica \u2013 aparelho de Golgi \u2013 ribossomo \u2013\ + \ ves\xEDcula secretora \u2013 RE rugoso" + D: "Ribossomo \u2013 RE rugoso \u2013 aparelho de Golgi \u2013 ves\xEDcula secretora\ + \ \u2013 membrana plasm\xE1tica" + input_correct_responses: + - D + input_question: "Nas c\xE9lulas animais, qual das alternativas a seguir representa\ + \ o caminho mais prov\xE1vel que uma prote\xEDna secretora percorre ao ser sintetizada\ + \ em uma c\xE9lula?" + - input_choice_list: + A: Ciclinas + B: "Prote\xEDnas quinases" + C: "Pontos de verifica\xE7\xE3o" + D: "C\xE9lulas fibrobl\xE1sticas" + input_correct_responses: + - D + input_question: "Qual das alternativas a seguir n\xE3o est\xE1 envolvida no controle\ + \ da divis\xE3o celular?" + - input_choice_list: + A: "as asas de um p\xE1ssaro e as asas de um morcego" + B: "as nadadeiras de uma baleia e os bra\xE7os de um homem" + C: as barbatanas peitorais de uma toninha e as nadadeiras de uma foca + D: as patas dianteiras de um inseto e as patas dianteiras de um cachorro + input_correct_responses: + - D + input_question: "Estruturas hom\xF3logas s\xE3o frequentemente citadas como evid\xEA\ + ncia do processo de sele\xE7\xE3o natural. Todas as alternativas a seguir s\xE3\ + o exemplos de estruturas hom\xF3logas, EXCETO" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_biology +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_high_school_biology +task_alias: high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca324effcd2345fcddfc9399b03c5fdde373e505 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_chemistry.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: HCl + B: H2SO3 + C: SO2 + D: Al(NO3)3 + input_correct_responses: + - C + input_question: "Qual dos seguintes \xE9 considerado um anidrido \xE1cido?" + - input_choice_list: + A: PCl4F + B: BF3 + C: CO2 + D: Si(CH3)4 + input_correct_responses: + - A + input_question: "Qual das alternativas a seguir \xE9 considerada uma mol\xE9cula\ + \ polar?" + - input_choice_list: + A: "Todos os cloretos, brometos e iodetos s\xE3o sol\xFAveis" + B: "Todos os sulfatos s\xE3o sol\xFAveis" + C: "Todos os hidr\xF3xidos s\xE3o sol\xFAveis" + D: "Todos os compostos contendo am\xF4nio s\xE3o sol\xFAveis" + input_correct_responses: + - D + input_question: "A partir das regras de solubilidade, qual das afirma\xE7\xF5\ + es a seguir \xE9 verdadeira?" + - input_choice_list: + A: 3,89 + B: 7,78 + C: 5,78 + D: '2.33' + input_correct_responses: + - C + input_question: "Um novo composto \xE9 sintetizado e descobriu-se que \xE9 um\ + \ \xE1cido monopr\xF3tico com massa molar de 248 g/mol. Quando 0,0050 mol deste\ + \ \xE1cido s\xE3o dissolvidos em 0,500 L de \xE1gua, o pH \xE9 medido como 3,89.\ + \ Qual \xE9 o pKa desse \xE1cido?" + - input_choice_list: + A: 0,500 mole + B: 1,00 mole + C: 2,00 moles + D: 3,00 moles + input_correct_responses: + - C + input_question: "Uma solu\xE7\xE3o cont\xE9m 2,00 mol de \xE1cido ac\xE9tico,\ + \ CH3COOH, e 1,00 mol de acetato de c\xE1lcio, Ca(CH3COO)2. A solu\xE7\xE3o\ + \ \xE9 capaz de resistir \xE0 adi\xE7\xE3o de uma pequena quantidade de \xE1\ + cido forte ou base forte, com apenas pequenas altera\xE7\xF5es no pH da solu\xE7\ + \xE3o. Grandes quantidades de \xE1cido forte ou base forte podem causar uma\ + \ mudan\xE7a significativa no pH. Quantos moles de \xE1cido n\xEDtrico, HNO3,\ + \ podem ser adicionados antes que o pH comece a mudar significativamente?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_chemistry +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_high_school_chemistry +task_alias: high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f778bd05d187be70734e0c760bfb597eb9b8deb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_computer_science.yaml @@ -0,0 +1,78 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Um carro alerta um motorista que est\xE1 prestes a bater em um objeto." + B: "Uma caminhante usa um rel\xF3gio GPS para monitorar sua posi\xE7\xE3o." + C: "Uma geladeira pede leite de um servi\xE7o de entrega on-line quando o leite\ + \ na geladeira est\xE1 quase acabando." + D: "Um corredor usa um rel\xF3gio com sensores \xF3pticos para monitorar sua\ + \ frequ\xEAncia card\xEDaca." + input_correct_responses: + - C + input_question: "Qual das alternativas a seguir \xE9 um exemplo do uso de um dispositivo\ + \ na Internet das Coisas (IoT)?" + - input_choice_list: + A: "As atividades de um usu\xE1rio navegando em uma janela an\xF4nima n\xE3\ + o ser\xE3o vis\xEDveis para as pessoas que monitoram a rede do usu\xE1rio,\ + \ como o administrador do sistema." + B: "Os itens colocados no carrinho de compras de uma loja virtual para compras\ + \ futuras durante a sess\xE3o de navega\xE7\xE3o an\xF4nima n\xE3o ser\xE3\ + o salvos no computador do usu\xE1rio." + C: "Um usu\xE1rio n\xE3o poder\xE1 fazer login em contas de e-mail ou de m\xED\ + dia social durante a sess\xE3o de navega\xE7\xE3o an\xF4nima." + D: "Um usu\xE1rio que navega em uma janela an\xF4nima estar\xE1 protegido contra\ + \ v\xEDrus lan\xE7ados em quaisquer sites visitados ou arquivos baixados." + input_correct_responses: + - B + input_question: "Muitos navegadores da Web permitem que os usu\xE1rios abram janelas\ + \ an\xF4nimas. Durante uma sess\xE3o de navega\xE7\xE3o em uma janela an\xF4\ + nima, o navegador n\xE3o registra um hist\xF3rico de navega\xE7\xE3o ou uma\ + \ lista de arquivos baixados. Ao sair da janela an\xF4nima, os cookies criados\ + \ durante a sess\xE3o s\xE3o exclu\xEDdos. Qual das seguintes afirma\xE7\xF5\ + es sobre sess\xF5es de navega\xE7\xE3o em uma janela an\xF4nima \xE9 verdadeira?" + - input_choice_list: + A: Erro + B: abc + C: cba + D: c + input_correct_responses: + - C + input_question: "Qual \xE9 a sa\xEDda de "abc"[::-1] em Python 3?" + - input_choice_list: + A: Foxtrot + B: Hotel + C: novembro + D: ianque + input_correct_responses: + - C + input_question: "No programa abaixo, o valor inicial de x \xE9 5 e o valor inicial\ + \ de y \xE9 10. IF (X < O) { DISPLAY ("Foxtrot") } ELSE { IF (X\ + \ > y) { DISPLAY ("Hotel") } ELSE { IF (y > O) { DISPLAY ("November")\ + \ } ELSE { DISPLAY ("Yankee") } } } O que \xE9 exibido como resultado\ + \ da execu\xE7\xE3o do programa?" + - input_choice_list: + A: "Etapa 3: Aumente o valor da posi\xE7\xE3o em 1. Etapa 4: Repita as etapas\ + \ 2 e 3 at\xE9 que o valor da contagem seja maior que 100." + B: "Passo 3: Aumente o valor da posi\xE7\xE3o em 1. Passo 4: Repita os passos\ + \ 2 e 3 at\xE9 que o valor da posi\xE7\xE3o seja maior que n." + C: "Etapa 3: Repita a etapa 2 at\xE9 que o valor da contagem seja maior que\ + \ 100. Etapa 4: Aumente o valor da posi\xE7\xE3o em 1." + D: "Etapa 3: Repita a etapa 2 at\xE9 que o valor da posi\xE7\xE3o seja maior\ + \ que n. Etapa 4: aumente o valor da contagem em 1." + input_correct_responses: + - D + input_question: "Uma lista de n\xFAmeros possui n elementos, indexados de 1 a\ + \ n. O algoritmo a seguir tem como objetivo exibir o n\xFAmero de elementos\ + \ na lista que possuem um valor maior que 100. O algoritmo usa as vari\xE1veis\ + \ contagem e posi\xE7\xE3o. As etapas 3 e 4 est\xE3o faltando. Etapa 1: Defina\ + \ a contagem como 0 e a posi\xE7\xE3o como 1. Etapa 2: Se o valor do elemento\ + \ na posi\xE7\xE3o do \xEDndice for maior que 100, aumente o valor da contagem\ + \ em 1. Etapa 3: (etapa ausente) Etapa 4: (etapa ausente ) Etapa 5: Exiba o\ + \ valor da contagem. Qual das alternativas a seguir poderia ser usada para substituir\ + \ as etapas 3 e 4 para que o algoritmo funcione conforme planejado?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_computer_science +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_high_school_computer_science +task_alias: high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2695137d57d7a482f22cc90f1985f43900a43f35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_european_history.yaml @@ -0,0 +1,196 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Jean-Jacques Rousseau + B: "Bar\xE3o Montesquieu" + C: Mary Wollstonecraft + D: Adam Smith + input_correct_responses: + - B + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ O trecho a seguir \xE9 de um panfleto. Voc\xEA me far\xE1 a justi\xE7a de\ + \ lembrar que sempre apoiei vigorosamente o direito de cada homem \xE0 sua pr\xF3\ + pria opini\xE3o, por mais diferente que essa opini\xE3o possa ser da minha.\ + \ Aquele que nega a outro este direito torna-se escravo de sua opini\xE3o atual,\ + \ porque se exclui do direito de mud\xE1-la. A arma mais formid\xE1vel contra\ + \ erros de todo tipo \xE9 a Raz\xE3o. Nunca usei nenhum outro e confio que nunca\ + \ usarei. A circunst\xE2ncia que agora ocorreu na Fran\xE7a da aboli\xE7\xE3\ + o total de toda a ordem nacional do sacerd\xF3cio, e de tudo o que diz respeito\ + \ aos sistemas religiosos compulsivos e aos artigos de f\xE9 compulsivos, n\xE3\ + o apenas precipitou a minha inten\xE7\xE3o, mas tornou uma obra desta tipo extremamente\ + \ necess\xE1rio, para que, na destrui\xE7\xE3o geral da supersti\xE7\xE3o, dos\ + \ falsos sistemas de governo e da falsa teologia, n\xE3o percamos de vista a\ + \ moralidade, a humanidade e a teologia que \xE9 verdadeira. Acredito em um\ + \ Deus e nada mais; e espero felicidade al\xE9m desta vida. Acredito na igualdade\ + \ do homem; e acredito que os deveres religiosos consistem em fazer justi\xE7\ + a, amar a miseric\xF3rdia e esfor\xE7ar-se para tornar felizes os nossos semelhantes.\ + \ N\xE3o acredito no credo professado pela igreja judaica, pela igreja romana,\ + \ pela igreja grega, pela igreja turca, pela igreja protestante, nem por qualquer\ + \ igreja que eu conhe\xE7a. Minha pr\xF3pria mente \xE9 minha pr\xF3pria igreja.\ + \ Todas as institui\xE7\xF5es nacionais de igrejas, sejam elas judaicas, crist\xE3\ + s ou turcas, parecem-me nada mais do que inven\xE7\xF5es humanas, criadas para\ + \ aterrorizar e escravizar a humanidade e monopolizar o poder e o lucro. N\xE3\ + o pretendo com esta declara\xE7\xE3o condenar aqueles que acreditam no contr\xE1\ + rio; eles t\xEAm o mesmo direito \xE0s suas cren\xE7as que eu tenho \xE0s minhas.\ + \ \u2014Thomas Paine, The Age of Reason, 1794\u20131795 Qual dos seguintes fil\xF3\ + sofos do Iluminismo projetou um sistema de freios e contrapesos para o governo,\ + \ a fim de evitar abusos de poder?" + - input_choice_list: + A: "As ideias de liberdade pessoal e nacionalismo concebidas durante o Iluminismo\ + \ resultaram em revolu\xE7\xF5es radicais que poderiam espalhar-se por toda\ + \ a Europa." + B: "A conquista da Europa por Napole\xE3o levou \xE0 cria\xE7\xE3o de novas\ + \ fac\xE7\xF5es e alterou o equil\xEDbrio de poder europeu." + C: "O poder dos monarcas cresceu a tal ponto que precisava ser controlado por\ + \ outros poderes dentro de cada na\xE7\xE3o ou ocorreria o dom\xEDnio dos\ + \ civis." + D: "O ciclo econ\xF3mico ascendente e descendente da economia capitalista emergente\ + \ poder\xE1 levar a agita\xE7\xE3o civil que deve ser suprimida." + input_correct_responses: + - A + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ Leia o trecho a seguir. A semente revolucion\xE1ria penetrou em todos os pa\xED\ + ses e espalhou-se mais ou menos. Desenvolveu-se grandemente sob o regime do\ + \ despotismo militar de Bonaparte. Suas conquistas substitu\xEDram uma s\xE9\ + rie de leis, institui\xE7\xF5es e costumes; rompeu os la\xE7os sagrados entre\ + \ todas as na\xE7\xF5es, fortes o suficiente para resistir ao pr\xF3prio tempo;\ + \ o que \xE9 mais do que se pode dizer de certos benef\xEDcios conferidos por\ + \ estes inovadores. Os monarcas cumprir\xE3o os deveres que lhes s\xE3o impostos\ + \ por Aquele que, ao confiar-lhes o poder, os encarregou de zelar pela manuten\xE7\ + \xE3o da justi\xE7a e dos direitos de todos, de evitar os caminhos do erro e\ + \ de trilhar com firmeza o caminho da verdade. Colocados para al\xE9m das paix\xF5\ + es que agitam a sociedade, \xE9 principalmente nos dias de prova\xE7\xE3o que\ + \ eles s\xE3o chamados a despojar-se da realidade das suas falsas apar\xEAncias\ + \ e a mostrar-se como s\xE3o, pais investidos da autoridade que pertence por\ + \ direito aos chefes de fam\xEDlia, para provar que, nos dias de luto, sabem\ + \ ser justos, s\xE1bios e, portanto, fortes, e que n\xE3o abandonar\xE3o o povo\ + \ que deveriam governar, tornando-o objecto de fac\xE7\xF5es, ao erro e \xE0\ + s suas consequ\xEAncias, que devem envolver a perda da sociedade. A uni\xE3\ + o entre os monarcas \xE9 a base da pol\xEDtica que deve agora ser seguida para\ + \ salvar a sociedade da ru\xEDna total. . . . Que n\xE3o confundam as concess\xF5\ + es feitas aos partidos com o bem que devem fazer ao seu povo, modificando, de\ + \ acordo com as suas necessidades reconhecidas, os ramos da administra\xE7\xE3\ + o que o exigem. Que sejam justos, mas fortes; ben\xE9fico, mas rigoroso. Que\ + \ mantenham os princ\xEDpios religiosos em toda a sua pureza, e n\xE3o permitam\ + \ que a f\xE9 seja atacada e a moralidade interpretada de acordo com o contrato\ + \ social ou com as vis\xF5es de sect\xE1rios tolos. Deixe-os suprimir as Sociedades\ + \ Secretas; aquela gangrena da sociedade. \u2014Klemens von Metternich, Confiss\xE3\ + o Pol\xEDtica de F\xE9, 1820 Qual das alternativas a seguir foi a maior causa\ + \ dos temores expressos por Metternich no documento acima?" + - input_choice_list: + A: Capitalista + B: "Cient\xEDfico" + C: Comunista + D: Existencialista + input_correct_responses: + - C + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ Na R\xFAssia n\xE3o havia nada de bom e [Souvarine] estava desesperado com\ + \ as not\xEDcias que recebera. Seus antigos companheiros estavam todos se voltando\ + \ para os pol\xEDticos; os famosos niilistas que fizeram a Europa tremer - filhos\ + \ de padres de aldeia, da classe m\xE9dia baixa, de comerciantes - n\xE3o conseguiam\ + \ superar a ideia de liberta\xE7\xE3o nacional e pareciam acreditar que o mundo\ + \ seria libertado - quando matassem o seu d\xE9spota &... "Tolice!\ + \ Eles nunca sair\xE3o dessa com suas tolices." Depois, baixando ainda\ + \ mais a voz, em poucas palavras amargas descreveu o seu antigo sonho de fraternidade.\ + \ Ele renunciou \xE0 sua posi\xE7\xE3o e \xE0 sua fortuna; ele havia caminhado\ + \ entre os trabalhadores, apenas na esperan\xE7a de ver finalmente a funda\xE7\ + \xE3o de uma nova sociedade de trabalho comum. Todos os soldos em seus bolsos\ + \ haviam ido h\xE1 muito tempo para os moleques do assentamento; ele tinha sido\ + \ t\xE3o terno quanto um irm\xE3o com os mineiros, sorrindo diante de suas suspeitas,\ + \ conquistando-os por seu jeito tranquilo e trabalhador e por sua avers\xE3\ + o a tagarelar. Mas decididamente a fus\xE3o n\xE3o ocorreu. Sua voz mudou, seus\ + \ olhos brilharam, ele os fixou em \xC9tienne, dirigindo-se a ele diretamente:\ + \ "Agora, voc\xEA entende isso? Esses chapeleiros de Marselha que ganharam\ + \ o grande pr\xEAmio de cem mil francos na loteria foram imediatamente e investiram\ + \ declarando que v\xE3o viver sem fazer nada! Sim, a id\xE9ia \xE9 sua, todos\ + \ voc\xEAs, trabalhadores franceses; voc\xEAs querem desenterrar um tesouro\ + \ para depois devor\xE1-lo sozinhos em algum canto pregui\xE7oso e ego\xEDsta.\ + \ Voc\xEAs podem gritar por mais que voc\xEA queira contra os ricos, voc\xEA\ + \ n\xE3o tem coragem suficiente para devolver aos pobres o dinheiro que a sorte\ + \ lhe traz. Voc\xEA nunca ser\xE1 digno de felicidade enquanto possuir alguma\ + \ coisa, e seu \xF3dio pela burguesia continuar unicamente por um desejo irado\ + \ de serem burgueses no lugar deles." \xE9mile Zola, escritor franc\xEA\ + s, Germinal, 1885 A passagem mostra a preocupa\xE7\xE3o direta com o bem-estar\ + \ das classes trabalhadoras que normalmente fazia parte de qual movimento?" + - input_choice_list: + A: "Serviram como catalisadores para o crescimento da navega\xE7\xE3o inglesa\ + \ e do com\xE9rcio exterior, mas pouco fizeram para limitar as perspectivas\ + \ dos holandeses no s\xE9culo XVII." + B: "Eles trouxeram dificuldades quase imediatas para a economia holandesa, \xE0\ + \ medida que o seu dom\xEDnio do com\xE9rcio exterior terminou rapidamente." + C: "Eles foram rescindidos durante a restaura\xE7\xE3o dos Stuarts, pois buscavam\ + \ rela\xE7\xF5es diplom\xE1ticas normais com os holandeses, para n\xE3o precisarem\ + \ do apoio financeiro do Parlamento para a guerra." + D: "Eles levaram a quase um s\xE9culo de guerras recorrentes entre a Inglaterra\ + \ e a Holanda, que s\xF3 terminariam depois da independ\xEAncia americana." + input_correct_responses: + - A + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ Os trechos abaixo s\xE3o das Leis de Navega\xE7\xE3o de 1651. [A]p\xF3s o\ + \ primeiro dia de dezembro de mil seiscentos e cinquenta e um, e da\xED em diante,\ + \ nenhum bem ou mercadoria de qualquer tipo do crescimento, produ\xE7\xE3o ou\ + \ manufatura da \xC1sia, \xC1frica ou Am\xE9rica, ou de qualquer parte dela;\ + \ ou de quaisquer ilhas que lhes perten\xE7am, ou que estejam descritas ou estabelecidas\ + \ nos mapas ou cartas habituais desses lugares, bem como das planta\xE7\xF5\ + es inglesas como outras, ser\xE3o importadas ou trazidas para esta Comunidade\ + \ da Inglaterra, ou para a Irlanda, ou quaisquer outras terras, ilhas, planta\xE7\ + \xF5es ou territ\xF3rios desta Comunidade pertencentes, ou em sua posse, em\ + \ qualquer outro navio ou navios, embarca\xE7\xE3o ou embarca\xE7\xF5es de qualquer\ + \ natureza, mas apenas na medida em que verdadeiramente e sem fraude perten\xE7\ + am apenas ao povo desta Comunidade , ou suas planta\xE7\xF5es, como seus propriet\xE1\ + rios ou titulares de direitos; e dos quais o capit\xE3o e os marinheiros tamb\xE9\ + m s\xE3o do povo desta Comunidade, sob pena de confisco e perda de todos os\ + \ bens que ser\xE3o importados contrariamente a este ato,, [N] nenhum bem ou\ + \ mercadoria do crescimento, produ\xE7\xE3o ou fabrica\xE7\xE3o da Europa, ou\ + \ de qualquer parte dela, ser\xE1 importada ou trazida para esta Comunidade\ + \ da Inglaterra, ou para quaisquer outras terras ou territ\xF3rios pertencentes\ + \ a esta Comunidade, ap\xF3s o primeiro dia de dezembro de mil seiscentos e\ + \ cinquenta e um, ou em sua posse, em qualquer navio ou navios, embarca\xE7\xE3\ + o ou embarca\xE7\xF5es de qualquer natureza, mas em tais que verdadeiramente\ + \ e sem fraude perten\xE7am apenas ao povo desta Comunidade, e em nenhum outro,\ + \ exceto apenas os navios e embarca\xE7\xF5es estrangeiros que verdadeiramente\ + \ e pertencem propriamente ao povo daquele pa\xEDs ou local, do qual os referidos\ + \ bens s\xE3o o crescimento, a produ\xE7\xE3o ou a manufatura. Qual das alternativas\ + \ a seguir descreve melhor o resultado das Leis de Navega\xE7\xE3o de 1651?" + - input_choice_list: + A: "dar ao rei ingl\xEAs uma nova posi\xE7\xE3o de autoridade" + B: "dar a posi\xE7\xE3o de chefe da Igreja da Inglaterra apenas a Henrique VIII\ + \ e excluir seus herdeiros" + C: "estabelecer o calvinismo como a \xFAnica teologia verdadeira na Inglaterra" + D: "acabar com v\xE1rias formas de corrup\xE7\xE3o que assolam a Igreja na Inglaterra" + input_correct_responses: + - D + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ Embora a Majestade do rei seja e deva ser, com justi\xE7a e direito, o chefe\ + \ supremo da Igreja da Inglaterra, e assim seja reconhecido pelo clero deste\ + \ reino em suas convoca\xE7\xF5es, ainda assim, para corrobora\xE7\xE3o e confirma\xE7\ + \xE3o disso, e para aumento da virtude em religi\xE3o de Cristo dentro deste\ + \ reino da Inglaterra, e para reprimir e extirpar todos os erros, heresias e\ + \ outras enormidades e abusos at\xE9 ent\xE3o usados no mesmo, seja decretado,\ + \ pela autoridade deste atual Parlamento, que o rei, nosso senhor soberano,\ + \ seus herdeiros e os sucessores, reis deste reino, ser\xE3o tomados, aceitos\ + \ e reputados como o \xFAnico chefe supremo na terra da Igreja da Inglaterra,\ + \ chamado Anglicanos Ecclesia; e ter\xE1 e desfrutar\xE1, anexado e unido \xE0\ + \ coroa imperial deste reino, bem como ao t\xEDtulo e estilo do mesmo, como\ + \ todas as honras, dignidades, preemin\xEAncias, jurisdi\xE7\xF5es, privil\xE9\ + gios, autoridades, imunidades, lucros e mercadorias \xE0 referida dignidade\ + \ de o chefe supremo da mesma Igreja pertencente e pertencente; e que nosso\ + \ dito senhor soberano, seus herdeiros e sucessores, reis deste reino, ter\xE3\ + o pleno poder e autoridade de tempos em tempos para visitar, reprimir, reparar,\ + \ registrar, ordenar, corrigir, restringir e emendar todos esses erros, heresias,\ + \ abusos, ofensas, desprezos e enormidades, quaisquer que sejam, que por qualquer\ + \ forma de autoridade ou jurisdi\xE7\xE3o espiritual devem ou podem ser legalmente\ + \ reformados, reprimidos, ordenados, reparados, corrigidos, restringidos ou\ + \ emendados, da maneira mais agrad\xE1vel ao Deus Todo-Poderoso, o aumento da\ + \ virtude na religi\xE3o de Cristo e para a conserva\xE7\xE3o da paz, unidade\ + \ e tranquilidade deste reino; qualquer uso, terra estrangeira, autoridade estrangeira,\ + \ prescri\xE7\xE3o ou qualquer outra coisa ou coisas em contr\xE1rio deste documento.\ + \ Parlamento Ingl\xEAs, Ato de Supremacia, 1534 Da passagem, pode-se inferir\ + \ que o Parlamento Ingl\xEAs desejava argumentar que o Ato de Supremacia" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_european_history +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_high_school_european_history +task_alias: high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae70c27afbc47df5d980250f9c99d78dd69a399e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_geography.yaml @@ -0,0 +1,56 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: taxa bruta de mortalidade a partir da data bruta de nascimento. + B: taxa bruta de natalidade da taxa bruta de mortalidade. + C: duplicando o tempo da taxa bruta de natalidade. + D: taxa de fertilidade da taxa bruta de mortalidade. + input_correct_responses: + - A + input_question: "A taxa de aumento natural de uma popula\xE7\xE3o \xE9 encontrada\ + \ subtraindo o" + - input_choice_list: + A: "As taxas de natalidade aumentam e a taxa de crescimento populacional \xE9\ + \ menos r\xE1pida." + B: "As taxas de natalidade diminuem e a taxa de crescimento populacional \xE9\ + \ menos r\xE1pida." + C: As taxas de natalidade aumentam e a taxa de crescimento populacional aumenta. + D: As taxas de natalidade diminuem e a taxa de crescimento populacional aumenta. + input_correct_responses: + - B + input_question: "Durante a terceira fase do modelo de transi\xE7\xE3o demogr\xE1\ + fica, qual das seguintes afirma\xE7\xF5es \xE9 verdadeira?" + - input_choice_list: + A: "A duplica\xE7\xE3o de esfor\xE7os ocorre frequentemente." + B: "Os problemas sociais da cidade central espalham-se pelos sub\xFArbios residenciais\ + \ circundantes." + C: "A inefici\xEAncia na presta\xE7\xE3o de servi\xE7os ocorre frequentemente." + D: "Os esfor\xE7os de um bairro para reduzir a polui\xE7\xE3o s\xE3o sempre\ + \ apoiados pelas comunidades vizinhas." + input_correct_responses: + - D + input_question: "Qual das seguintes afirma\xE7\xF5es N\xC3O \xE9 precisa em rela\xE7\ + \xE3o aos servi\xE7os prestados pelos governos locais nos Estados Unidos?" + - input_choice_list: + A: "terceiriza\xE7\xE3o." + B: "terceiriza\xE7\xE3o." + C: maquiladoras. + D: "interdepend\xEAncia locacional." + input_correct_responses: + - B + input_question: "A pr\xE1tica de contratar um terceiro prestador de servi\xE7\ + os estrangeiro para executar uma opera\xE7\xE3o \xE9 chamada" + - input_choice_list: + A: Sabonete pomba + B: Barra de chocolate pomba + C: "S\xEDmbolo de pomba" + D: "Uma pomba (p\xE1ssaro)" + input_correct_responses: + - C + input_question: "Qual dos itens a seguir \xE9 um exemplo de cultura imaterial?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_geography +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_high_school_geography +task_alias: high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cf17d52d4208488bd38a90a655ae37961beb891 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_government_and_politics.yaml @@ -0,0 +1,61 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "a defini\xE7\xE3o constitucional desses poderes \xE9 ampla e inespec\xED\ + fica" + B: "a maioria das pessoas concorda que a Constitui\xE7\xE3o imp\xF5e demasiados\ + \ limites ao poder presidencial" + C: o Supremo Tribunal recusa-se consistentemente a decidir sobre casos relativos + aos poderes presidenciais + D: emendas constitucionais aumentaram muito os poderes presidenciais + input_correct_responses: + - A + input_question: "A incerteza sobre os limites do poder presidencial \xE9 causada\ + \ principalmente pelo facto de" + - input_choice_list: + A: "aumento anual nos gastos federais com as for\xE7as armadas" + B: "montante dos juros sobre a d\xEDvida nacional" + C: "diferen\xE7a entre as propostas or\xE7ament\xE1rias iniciais feitas pelo\ + \ presidente e pelo Congresso" + D: "valor que o governo gasta al\xE9m de suas receitas" + input_correct_responses: + - D + input_question: "O termo \u201Cd\xE9fice or\xE7amental\u201D refere-se ao" + - input_choice_list: + A: Semanas x Estados Unidos + B: Betts x Brady + C: Mapp v. + D: Miranda v. Arizona + input_correct_responses: + - D + input_question: "Qual dos seguintes casos estabeleceu o precedente de que o r\xE9\ + u deve ser informado do direito de permanecer calado, do direito a um advogado\ + \ e da prote\xE7\xE3o contra a autoincrimina\xE7\xE3o?" + - input_choice_list: + A: "Eles s\xE3o estabelecidos pelo Poder Legislativo." + B: "Os seus membros muitas vezes n\xE3o t\xEAm muita influ\xEAncia sobre as\ + \ decis\xF5es presidenciais." + C: "N\xE3o podem ser todos dirigidos por l\xEDderes que perten\xE7am ao mesmo\ + \ partido pol\xEDtico do presidente." + D: "Nem toda ag\xEAncia federal \xE9 um departamento de gabinete." + input_correct_responses: + - C + input_question: "Qual das seguintes afirma\xE7\xF5es sobre departamentos de gabinete\ + \ \xE9 FALSA?" + - input_choice_list: + A: "Pol\xEDticos honestos podem impedir o desenvolvimento de fac\xE7\xF5es." + B: "\xC9 mais prov\xE1vel que ocorram fac\xE7\xF5es nas grandes rep\xFAblicas\ + \ do que nas pequenas." + C: Os efeitos negativos do faccionalismo podem ser reduzidos por um governo + republicano. + D: "As elei\xE7\xF5es livres s\xE3o a melhor defesa do povo contra o partidarismo." + input_correct_responses: + - C + input_question: "Qual das alternativas a seguir melhor expressa um argumento apresentado\ + \ por James Madison em The Federalist n\xFAmero 10?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_government_and_politics +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_high_school_government_and_politics +task_alias: high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c22819a74fc87ecc09b982144973a7cc43bde7c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_macroeconomics.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Os militares dos EUA abrem uma nova base em um pa\xEDs estrangeiro com 1.000\ + \ funcion\xE1rios dos EUA." + B: Os consumidores japoneses compram milhares de CDs produzidos nos Estados + Unidos. + C: Uma cantora pop americana realiza um show com ingressos esgotados em Paris. + D: "Uma produ\xE7\xE3o teatral francesa percorre dezenas de cidades americanas." + input_correct_responses: + - C + input_question: "Qual dos itens a seguir n\xE3o est\xE1 inclu\xEDdo no PIB dos\ + \ EUA?" + - input_choice_list: + A: "rela\xE7\xE3o direta entre desemprego e infla\xE7\xE3o" + B: "rela\xE7\xE3o direta entre pre\xE7o e quantidade demandada" + C: "rela\xE7\xE3o inversa entre pre\xE7o e quantidade demandada" + D: "rela\xE7\xE3o inversa entre desemprego e infla\xE7\xE3o" + input_correct_responses: + - D + input_question: A curva de Phillips de curto prazo indica uma + - input_choice_list: + A: "as exporta\xE7\xF5es excedem as importa\xE7\xF5es." + B: "as importa\xE7\xF5es excedem as exporta\xE7\xF5es." + C: "a arrecada\xE7\xE3o de impostos federais excede os gastos." + D: os gastos federais excedem as receitas fiscais federais. + input_correct_responses: + - D + input_question: "Um d\xE9ficit federal ocorre quando" + - input_choice_list: + A: Aumentando a taxa de desconto + B: "Aumentando o \xEDndice de reserva" + C: "Compra de t\xEDtulos do governo" + D: "Redu\xE7\xE3o de tarifas" + input_correct_responses: + - C + input_question: "Mantendo tudo o resto igual, qual das seguintes pol\xEDticas\ + \ monet\xE1rias seria usada para impulsionar as exporta\xE7\xF5es dos EUA?" + - input_choice_list: + A: Um aumento na oferta de moeda + B: Aumento dos gastos do governo + C: Impostos mais baixos sobre pesquisa e desenvolvimento de novas tecnologias + D: Impostos mais elevados sobre a renda familiar + input_correct_responses: + - C + input_question: "Qual das seguintes pol\xEDticas melhor descreve a pol\xEDtica\ + \ fiscal do lado da oferta?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_macroeconomics +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_high_school_macroeconomics +task_alias: high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e03dff2977696937f35c5da9c3b8960622c19490 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_mathematics.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '3' + B: '15' + C: '6' + D: '5' + input_correct_responses: + - B + input_question: "Joe estava encarregado das luzes para um baile. A luz vermelha\ + \ pisca a cada dois segundos, a luz amarela a cada tr\xEAs segundos e a luz\ + \ azul a cada cinco segundos. Se incluirmos o in\xEDcio e o fim da dan\xE7a,\ + \ quantas vezes durante uma dan\xE7a de sete minutos todas as luzes acender\xE3\ + o ao mesmo tempo? (Suponha que todas as tr\xEAs luzes pisquem simultaneamente\ + \ no in\xEDcio da dan\xE7a.)" + - input_choice_list: + A: '12' + B: '1' + C: '30' + D: '5' + input_correct_responses: + - C + input_question: "Cinco mil d\xF3lares compostos anualmente a uma taxa de juros\ + \ de $x\\%$ levam seis anos para dobrar. Com a mesma taxa de juros, quantos\ + \ anos ser\xE3o necess\xE1rios $\\$300$ para crescer para $\\$9600$?" + - input_choice_list: + A: '-1' + B: '16' + C: -\frac{1}{256} + D: \frac{1}{16} + input_correct_responses: + - C + input_question: "A vari\xE1vel $x$ varia diretamente como o quadrado de $y$, e\ + \ $y$ varia diretamente como o cubo de $z$. Se $x$ for igual a $-16$ quando\ + \ $z$ for igual a 2, qual \xE9 o valor de $x$ quando $z$ for igual a $\\frac{1}{2}$?" + - input_choice_list: + A: \frac{3\sqrt{3}}{3} + B: \frac{1}{3} + C: \sqrt{3} + D: \frac{\sqrt{3}}{3} + input_correct_responses: + - D + input_question: 'Simplifique e escreva o resultado com um denominador racional: + $$\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}$$' + - input_choice_list: + A: '55' + B: '60' + C: '62' + D: '65' + input_correct_responses: + - D + input_question: "Dez alunos fazem um teste de biologia e recebem as seguintes\ + \ notas: 45, 55, 50, 70, 65, 80, 40, 90, 70, 85. Qual \xE9 a m\xE9dia das notas\ + \ dos alunos nos testes?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_mathematics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_high_school_mathematics +task_alias: high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d634dbfe59bd2ac601dca399be7595e47499f6e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_microeconomics.yaml @@ -0,0 +1,54 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Um sal\xE1rio m\xEDnimo efetivo imposto a este mercado de trabalho." + B: "Um aumento no pre\xE7o dos gal\xF5es de tinta." + C: "Aumento na constru\xE7\xE3o de novas moradias." + D: "Um aumento no pre\xE7o dos pintores mec\xE2nicos enquanto o efeito produ\xE7\ + \xE3o exceder o efeito substitui\xE7\xE3o." + input_correct_responses: + - C + input_question: "Num mercado de trabalho competitivo para pintores residenciais,\ + \ qual das seguintes op\xE7\xF5es aumentaria a procura por pintores residenciais?" + - input_choice_list: + A: "a demanda pelo produto aumentar\xE1" + B: "a demanda pelo produto diminuir\xE1" + C: "o excedente do consumidor aumentar\xE1" + D: "o excedente do consumidor diminuir\xE1" + input_correct_responses: + - C + input_question: "Se o governo subsidiar os produtores num mercado perfeitamente\ + \ competitivo, ent\xE3o" + - input_choice_list: + A: '0' + B: '5' + C: '10' + D: '100' + input_correct_responses: + - D + input_question: "A taxa de concentra\xE7\xE3o para um monop\xF3lio \xE9" + - input_choice_list: + A: "O pre\xE7o m\xEDnimo desloca a curva de demanda para a esquerda." + B: Um piso eficaz cria uma escassez do bem. + C: "O pre\xE7o m\xEDnimo desloca a curva de oferta do bem para a direita." + D: "Para ser um piso efetivo, deve ser fixado acima do pre\xE7o de equil\xED\ + brio." + input_correct_responses: + - D + input_question: "Qual das afirma\xE7\xF5es a seguir \xE9 verdadeira em rela\xE7\ + \xE3o ao pre\xE7o m\xEDnimo?" + - input_choice_list: + A: "Entrada e sa\xEDda gratuitas do mercado" + B: Alguns grandes produtores + C: "Um produtor de um bem sem substitutos pr\xF3ximos" + D: "Um produto homog\xEAneo" + input_correct_responses: + - B + input_question: "Qual das alternativas a seguir \xE9 necessariamente uma caracter\xED\ + stica do oligop\xF3lio?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_microeconomics +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_high_school_microeconomics +task_alias: high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6728c95422089a3a2eb330a2679765e7b3367c22 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_physics.yaml @@ -0,0 +1,63 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Somente I e II + B: Somente I e III + C: Apenas II e III + D: III apenas + input_correct_responses: + - D + input_question: "Qual das seguintes condi\xE7\xF5es garantir\xE1 que o momento\ + \ angular seja conservado? I. Conserva\xE7\xE3o do momento linear II. For\xE7\ + a externa l\xEDquida zero III. Torque externo l\xEDquido zero" + - input_choice_list: + A: "A press\xE3o est\xE1 em um n\xF3, mas o deslocamento da part\xEDcula est\xE1\ + \ em um antinodo." + B: "A press\xE3o est\xE1 em um antinodo, mas o deslocamento da part\xEDcula\ + \ est\xE1 em um n\xF3." + C: "A press\xE3o e o deslocamento das part\xEDculas est\xE3o ambos nos n\xF3\ + s." + D: "A press\xE3o e o deslocamento das part\xEDculas est\xE3o ambos nos antinodos." + input_correct_responses: + - B + input_question: "Um cano cheio de ar est\xE1 fechado em uma das extremidades.\ + \ Uma onda estacion\xE1ria \xE9 produzida no tubo, fazendo com que ele emita\ + \ uma nota. Qual das alternativas a seguir \xE9 uma afirma\xE7\xE3o correta\ + \ sobre as propriedades da onda na extremidade fechada do tubo?" + - input_choice_list: + A: 2h00 + B: "6:00 DA MANH\xC3" + C: 12h00 + D: 24A + input_correct_responses: + - D + input_question: "Uma fotoc\xE9lula de fun\xE7\xE3o trabalho \u03D5 = 2eV \xE9\ + \ conectada em s\xE9rie a um resistor. A luz de frequ\xEAncia f = 1 \xD7 10\ + \ ^ 15 Hz atinge uma placa met\xE1lica da fotoc\xE9lula. Se a pot\xEAncia da\ + \ luz for P = 100 W, qual \xE9 a corrente que passa pelo resistor?" + - input_choice_list: + A: 10W + B: 30W + C: 60W + D: 240 W + input_correct_responses: + - D + input_question: "Um forno de micro-ondas est\xE1 conectado a uma tomada de 120\ + \ V e consome uma corrente de 2 amperes. A que taxa a energia est\xE1 sendo\ + \ utilizada pelo forno de micro-ondas?" + - input_choice_list: + A: 3,5J + B: 6,0J + C: 22,5J + D: 40J + input_correct_responses: + - B + input_question: "Uma carga pontual, Q = +1 mC, est\xE1 fixada na origem. Quanto\ + \ trabalho \xE9 necess\xE1rio para mover uma carga, Q = +8 \xB5C, do ponto (0,4\ + \ metros) para o ponto (3 metros, 0)?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_physics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_high_school_physics +task_alias: high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c03d2a3abf531863f96410c06e5086ef4a234d42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_psychology.yaml @@ -0,0 +1,65 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: um superego forte. + B: baixa auto-estima. + C: "baixa autoefic\xE1cia." + D: um locus de controle interno. + input_correct_responses: + - D + input_question: "Ani acredita que suas atitudes e comportamento desempenham um\ + \ papel central no que acontece com ela. \xC9 prov\xE1vel que tal cren\xE7a\ + \ esteja associada a" + - input_choice_list: + A: "identificar as causas e solu\xE7\xF5es dos problemas apresentados pelo cliente" + B: identificar e eliminar as causas das dificuldades do consultado em lidar + com um problema + C: "estabelecer uma hierarquia de autoridade para permitir uma tomada de decis\xE3\ + o eficaz" + D: "apresentar um curso de a\xE7\xE3o \xFAnico, bem definido e inequ\xEDvoco\ + \ para o consultor superar d\xE9ficits de compet\xEAncias" + input_correct_responses: + - B + input_question: "De acordo com o modelo de consulta de caso centrada no consultado\ + \ de Caplan, o consultor est\xE1 principalmente interessado em" + - input_choice_list: + A: "As mensagens s\xE3o enviadas do t\xE1lamo diretamente para a am\xEDgdala." + B: "As mensagens s\xE3o enviadas do t\xE1lamo para as vias \u201Co qu\xEA\u201D\ + \ e \u201Conde\u201D." + C: "As mensagens s\xE3o enviadas do sistema nervoso parassimp\xE1tico para o\ + \ c\xF3rtex cerebral." + D: "As mensagens s\xE3o enviadas dos lobos frontais para a gl\xE2ndula pituit\xE1\ + ria." + input_correct_responses: + - A + input_question: "Ao nadar no oceano, Ivan se assusta com uma sombra escura na\ + \ \xE1gua antes mesmo de ter a chance de identificar o que \xE9 essa sombra.\ + \ As conex\xF5es sin\xE1pticas que ocorrem durante este incidente de medo s\xE3\ + o melhor descritas por qual das seguintes op\xE7\xF5es?" + - input_choice_list: + A: "D\xEA \xE0 crian\xE7a um per\xEDodo experimental no novo ambiente" + B: Notifique os pais por escrito + C: "Obtenha a aprova\xE7\xE3o do conselho escolar" + D: Obtenha o consentimento dos pais + input_correct_responses: + - B + input_question: "De acordo com a Lei de Melhoria da Educa\xE7\xE3o de Indiv\xED\ + duos com Defici\xEAncia, qual das seguintes op\xE7\xF5es uma ag\xEAncia educacional\ + \ deve fazer antes de alterar a coloca\xE7\xE3o educacional de um aluno com\ + \ defici\xEAncia?" + - input_choice_list: + A: "s\xF3cio cultural" + B: "cl\xEDnico" + C: cognitivo + D: behaviorista + input_correct_responses: + - C + input_question: "Pascale est\xE1 interessada nas estrat\xE9gias de processamento\ + \ que as crian\xE7as usam para aprender novas informa\xE7\xF5es. Pascale seria\ + \ melhor classificado como que tipo de psic\xF3logo?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_psychology +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_high_school_psychology +task_alias: high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efa565968f0eb4e1e47b38a5ececceb531640539 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_statistics.yaml @@ -0,0 +1,72 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Se a inclina\xE7\xE3o da linha de regress\xE3o for exatamente 1, ent\xE3\ + o a correla\xE7\xE3o ser\xE1 exatamente 1." + B: "Se a correla\xE7\xE3o for 0, ent\xE3o a inclina\xE7\xE3o da linha de regress\xE3\ + o \xE9 indefinida." + C: "Mudar qual vari\xE1vel \xE9 chamada de x e qual \xE9 chamada de y altera\ + \ o sinal da correla\xE7\xE3o." + D: "A correla\xE7\xE3o r \xE9 igual \xE0 inclina\xE7\xE3o da linha de regress\xE3\ + o quando os escores z da vari\xE1vel y s\xE3o plotados em rela\xE7\xE3o aos\ + \ escores z da vari\xE1vel x." + input_correct_responses: + - D + input_question: "Qual das alternativas a seguir \xE9 uma afirma\xE7\xE3o correta\ + \ sobre correla\xE7\xE3o?" + - input_choice_list: + A: E(X + Y) = 99, var(X + Y) = 8,5 + B: E(X + Y) = 99, var(X + Y) = 13 + C: E(X + Y) = 99, var(X + Y) = 17 + D: "N\xE3o h\xE1 informa\xE7\xF5es suficientes para responder a esta pergunta." + input_correct_responses: + - D + input_question: "Suponha que X e Y sejam vari\xE1veis aleat\xF3rias com E(X) =\ + \ 37, var(X) = 5, E(Y) = 62 e var(Y) = 12. Quais s\xE3o o valor esperado e a\ + \ vari\xE2ncia da vari\xE1vel aleat\xF3ria X + Sim?" + - input_choice_list: + A: "A propor\xE7\xE3o de \xE1rvores que sofreram mais de 50% de danos devido\ + \ \xE0 geada." + B: "O n\xFAmero de \xE1rvores afetadas pela geada." + C: "O n\xFAmero de \xE1rvores amostradas no bosque." + D: "Para cada \xE1rvore amostrada, se ela sofreu mais de 50% de danos ou no\ + \ m\xE1ximo 50% de danos." + input_correct_responses: + - D + input_question: "Depois que um alerta de geada foi emitido, o propriet\xE1rio\ + \ de um grande laranjal pediu aos seus trabalhadores que borrifassem \xE1gua\ + \ em todas as suas \xE1rvores. A \xE1gua deveria congelar e formar uma camada\ + \ protetora de gelo ao redor da flor de laranjeira. No entanto, o propriet\xE1\ + rio suspeitou que algumas \xE1rvores sofreram danos consider\xE1veis devido\ + \ \xE0 geada. Para estimar a propor\xE7\xE3o de \xE1rvores que sofreram mais\ + \ de 50% de danos devido \xE0 geada, ele retirou uma amostra aleat\xF3ria de\ + \ 100 \xE1rvores do seu bosque. Qual \xE9 a vari\xE1vel de resposta neste experimento?" + - input_choice_list: + A: "M\xE9dia de 518 gramas; desvio padr\xE3o 7,0 gramas" + B: "M\xE9dia de 518 gramas; desvio padr\xE3o 3,5 gramas" + C: "M\xE9dia de 518 gramas; desvio padr\xE3o 6,1 gramas" + D: "M\xE9dia de 394 gramas; desvio padr\xE3o 6,1 gramas" + input_correct_responses: + - C + input_question: "Um novo smartwatch \xE9 fabricado em uma parte da f\xE1brica\ + \ e depois protegido para envio em outra parte independente da f\xE1brica. O\ + \ peso do smartwatch tem m\xE9dia de 62 gramas e desvio padr\xE3o de 1,0 gramas.\ + \ O peso da embalagem (caixa, manual do usu\xE1rio, pl\xE1stico bolha, etc.)\ + \ tem m\xE9dia de 456 gramas e desvio padr\xE3o de 6 gramas. Juntos, a distribui\xE7\ + \xE3o do peso do smartwatch e sua embalagem teria a seguinte m\xE9dia e desvio\ + \ padr\xE3o:" + - input_choice_list: + A: eu, eu + B: II, III + C: III, eu + D: III, II + input_correct_responses: + - D + input_question: "Qual dos seguintes conjuntos tem o menor desvio padr\xE3o? Qual\ + \ tem o maior? I: {1,2,3} II: {-10,10} III: {100}" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_statistics +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_high_school_statistics +task_alias: high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24b241fcfbcdff434b58b658dc2a35dc0524982a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_us_history.yaml @@ -0,0 +1,169 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Resposta organizada \xE0 rebeli\xE3o de Bacon" + B: "Resposta federal \xE0 rebeli\xE3o de Shays" + C: "Resposta federal \xE0 rebeli\xE3o do u\xEDsque" + D: "Resposta federal \xE0 rebeli\xE3o de Pontiac" + input_correct_responses: + - C + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ \u201CA sociedade em todos os estados \xE9 uma b\xEAn\xE7\xE3o, mas o governo,\ + \ mesmo no seu melhor estado, \xE9 apenas um mal necess\xE1rio; no seu pior\ + \ estado, um mal intoler\xE1vel; pois quando sofremos ou somos expostos \xE0\ + s mesmas mis\xE9rias por um governo, o que poder\xEDamos esperar num pa\xED\ + s sem governo, a nossa calamidade \xE9 agravada ao reflectirmos que fornecemos\ + \ os meios pelos quais sofremos. O governo, tal como o vestu\xE1rio, \xE9 o\ + \ distintivo da inoc\xEAncia perdida; os pal\xE1cios dos reis s\xE3o constru\xED\ + dos sobre as ru\xEDnas dos caramanch\xF5es do para\xEDso. Pois se os impulsos\ + \ da consci\xEAncia fossem claros, uniformes e irresistivelmente obedecidos,\ + \ o homem n\xE3o precisaria de outro legislador; mas n\xE3o sendo esse o caso,\ + \ ele considera necess\xE1rio entregar uma parte de sua propriedade para fornecer\ + \ meios para a prote\xE7\xE3o do resto; e isso ele \xE9 induzido a fazer pela\ + \ mesma prud\xEAncia que em todos os outros casos o aconselha a escolher o menor\ + \ entre dois males. Portanto, sendo a seguran\xE7a o verdadeiro des\xEDgnio\ + \ e fim do governo, segue-se inquestionavelmente que qualquer forma dela pare\xE7\ + a mais prov\xE1vel de garantir para n\xF3s, com o menor gasto e o maior benef\xED\ + cio, \xE9 prefer\xEDvel a todos os outros." Thomas Paine, Common Sense,\ + \ 1776 Quais das seguintes "mis\xE9rias" mencionadas acima foram mais\ + \ condenadas pelos Antifederalistas da era p\xF3s-revolucion\xE1ria?" + - input_choice_list: + A: "Tens\xF5es entre as pol\xEDticas brit\xE2nicas e as aspira\xE7\xF5es dos\ + \ colonos norte-americanos." + B: "Tens\xF5es entre os \xEDndios americanos aliados dos franceses e os aliados\ + \ dos brit\xE2nicos." + C: "Tens\xF5es entre afro-americanos libertos e propriet\xE1rios brancos." + D: "Tens\xF5es entre colonos sertanejos e elites na Am\xE9rica colonial." + input_correct_responses: + - D + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ "Como nossa conduta tardia em Conestoga Manor e Lancaster ocasionou muita\ + \ especula\xE7\xE3o e uma grande diversidade de sentimentos neste e nos governos\ + \ vizinhos; alguns justificando-o e outros condenando-o; alguns aliviando caridosamente\ + \ o crime, e outros pintando-o maliciosamente da forma mais odiosa e Cores detest\xE1\ + veis, achamos que \xE9 nosso dever apresentar ao Publick todo o assunto tal\ + \ como apareceu, e ainda aparece, para n\xF3s... "Se essas coisas n\xE3\ + o forem suficientes para provar um apego injustific\xE1vel dos Quakers aos \xED\ + ndios selvagens , uma resolu\xE7\xE3o fixa de fazer amizade com eles e uma total\ + \ insensibilidade \xE0s ang\xFAstias humanas, consideremos alguns fatos mais\ + \ recentes. Quando descobrimos, no Ver\xE3o passado, que provavelmente n\xE3\ + o receber\xEDamos assist\xEAncia do Governo, alguns Volunt\xE1rios sa\xEDram\ + \ \xE0s nossas pr\xF3prias custas, determinados a expulsar os nossos Inimigos\ + \ das nossas Fronteiras; e quando nos aproximamos da grande ilha, compreendemos\ + \ que v\xE1rios de seus guerreiros haviam sa\xEDdo contra nossas fronteiras.\ + \ Ap\xF3s isso, voltamos e os encontramos e lutamos com eles em Munfey Hill,\ + \ onde perdemos alguns de nossos homens e matamos alguns de seus guerreiros\ + \ e, assim, salvamos nossas fronteiras desta hist\xF3ria em outra expedi\xE7\ + \xE3o. Mas assim que destru\xEDmos suas provis\xF5es na grande ilha e arruinamos\ + \ seu com\xE9rcio com o bom povo de Bel\xE9m, esses mesmos \xEDndios, que eram\ + \ justamente suspeitos de terem assassinado nossos amigos no condado de Northampton,\ + \ foram levados pela influ\xEAncia de alguns quacres. sob a prote\xE7\xE3o do\ + \ governo para proteg\xEA-los dos ressentimentos dos amigos e parentes dos assassinados,\ + \ e para apoi\xE1-los durante o inverno." \u2014 "Apology of the Paxton\ + \ Boys" (panfleto), 1764 (Nota: "pedido de desculpas" em este\ + \ contexto deve ser lido como uma explica\xE7\xE3o, n\xE3o como uma admiss\xE3\ + o de culpa ou arrependimento.) Os sentimentos expressos na explica\xE7\xE3o\ + \ acima refletem quais das tens\xF5es em curso durante o per\xEDodo colonial\ + \ da hist\xF3ria americana?" + - input_choice_list: + A: a Emenda de Direitos Iguais + B: "sufr\xE1gio universal" + C: direitos dos estados + D: "proibi\xE7\xE3o" + input_correct_responses: + - B + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ "No novo C\xF3digo de Leis que suponho que ser\xE1 necess\xE1rio que\ + \ voc\xEA fa\xE7a, desejo que voc\xEA se lembre das Senhoras e seja mais generoso\ + \ e favor\xE1vel a elas do que seus ancestrais. N\xE3o coloque esse poder ilimitado\ + \ nas m\xE3os dos Maridos ... Lembre-se de que todos os homens seriam tiranos\ + \ se pudessem. Se n\xE3o for dado cuidado e aten\xE7\xE3o especial \xE0s mulheres,\ + \ estamos determinados a fomentar uma rebeli\xE3o e n\xE3o nos manteremos vinculados\ + \ a quaisquer leis nas quais n\xE3o tenhamos voz ou representa\xE7\xE3o. Abigail\ + \ Adams, em uma carta a John Adams, 1776 "A legisla\xE7\xE3o especial para\ + \ as mulheres nos colocou em uma posi\xE7\xE3o muito an\xF4mala. As mulheres\ + \ investidas dos direitos dos cidad\xE3os em uma se\xE7\xE3o - eleitoras, jurados,\ + \ titulares de cargos - cruzando uma linha imagin\xE1ria, s\xE3o assuntos no\ + \ pr\xF3ximo. Em alguns Estados, uma mulher casada pode possuir propriedades\ + \ e realizar neg\xF3cios em seu pr\xF3prio nome; em outros, seus rendimentos\ + \ pertencem ao marido. Em alguns Estados, uma mulher pode testemunhar contra\ + \ o marido, processar e ser processada em nos tribunais; em outros, ela n\xE3\ + o tem repara\xE7\xE3o em caso de dano \xE0 pessoa, propriedade ou car\xE1ter.\ + \ Em caso de div\xF3rcio por adult\xE9rio do marido, a esposa inocente \xE9\ + \ considerada como n\xE3o possuindo nenhum direito aos filhos ou propriedade,\ + \ a menos que por por decreto especial do tribunal. Mas em nenhum Estado da\ + \ Uni\xE3o a esposa tem direito \xE0 sua pr\xF3pria pessoa, ou a qualquer parte\ + \ dos rendimentos conjuntos da comparticipa\xE7\xE3o durante a vida do marido.\ + \ Em alguns Estados as mulheres podem entrar no faculdades de direito e pr\xE1\ + tica nos tribunais; em outras s\xE3o proibidas. Em algumas universidades, as\ + \ raparigas desfrutam de vantagens educativas iguais \xE0s dos rapazes, enquanto\ + \ muitas das institui\xE7\xF5es mais orgulhosas do pa\xEDs lhes negam a admiss\xE3\ + o, embora os filhos da China, do Jap\xE3o e de \xC1frica sejam a\xED bem-vindos.\ + \ Mas os privil\xE9gios j\xE1 concedidos nos v\xE1rios Estados n\xE3o s\xE3\ + o de forma alguma seguros." Susan B. Anthony, "Declara\xE7\xE3o de\ + \ Direitos para as Mulheres", 4 de julho de 1876. Os sentimentos expressos\ + \ no segundo trecho de Susan B. Anthony s\xE3o provavelmente em apoio de" + - input_choice_list: + A: "Os americanos devem maximizar a sua vantagem tecnol\xF3gica no Vietname." + B: "O bombardeio americano no Vietn\xE3 est\xE1 conduzindo passo a passo ao\ + \ progresso da guerra." + C: "O bombardeio americano no Vietn\xE3 \xE9 um fracasso." + D: "A Am\xE9rica n\xE3o deve ceder ao derrotismo em rela\xE7\xE3o \xE0 guerra\ + \ do Vietname." + input_correct_responses: + - C + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ Os nossos l\xEDderes falam em parar a agress\xE3o vinda do Norte, mas esta\ + \ foi uma luta entre grupos de vietnamitas at\xE9 intervirmos. Parecemos empenhados\ + \ em salvar os vietnamitas de Ho Chi Minh, mesmo que tenhamos de mat\xE1-los\ + \ e demolir o seu pa\xEDs para o fazer. Enquanto os povos nativos examinam aldeias\ + \ bombardeadas, mulheres e crian\xE7as queimadas pelo napalm, planta\xE7\xF5\ + es de arroz destru\xEDdas e cidades invadidas pelo nosso pessoal militar, est\xE3\ + o sem d\xFAvida a dizer secretamente sobre os guerrilheiros vietcongues e sobre\ + \ as for\xE7as americanas: "Uma praga em ambas as vossas casas. ."\ + \ \u2026 Acabar com os bombardeamentos, a norte e a sul, acabar com as buscas\ + \ e destruir as opera\xE7\xF5es ofensivas e limitar a nossa ac\xE7\xE3o militar\ + \ \xE0 realiza\xE7\xE3o de opera\xE7\xF5es no terreno. O bombardeamento do Norte\ + \ n\xE3o conseguiu parar ou controlar seriamente o fluxo de tropas para o Sul\ + \ e pode, de facto, ter provocado um esfor\xE7o de guerra muito maior por parte\ + \ de Han\xF3i. \u2014Senador George McGovern, "The Lessons of Vietnam",\ + \ 25 de abril de 1967 Qual das seguintes opini\xF5es da d\xE9cada de 1960 reflete\ + \ mais diretamente a perspectiva do discurso de George McGovern?" + - input_choice_list: + A: Abigail Adams + B: Clara Barton + C: Shirley Temple + D: Hillary Clinton + input_correct_responses: + - B + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ N\xE3o venho para fazer reivindica\xE7\xF5es pessoais, nem para buscar benef\xED\ + cios individuais; Apare\xE7o como o advogado daqueles que n\xE3o podem defender\ + \ a sua pr\xF3pria causa; Venho como amigo daqueles que est\xE3o abandonados,\ + \ oprimidos e desolados. Na Provid\xEAncia de Deus, eu sou a voz do man\xED\ + aco cujos gritos penetrantes das sombrias masmorras de suas pris\xF5es n\xE3\ + o penetram em seus Sal\xF5es de Legisla\xE7\xE3o. Eu sou a Esperan\xE7a dos\ + \ pobres seres enlouquecidos que definham nas celas, e est\xE1bulos, e jaulas,\ + \ e quartos vazios de suas pobres casas. Eu sou a Revela\xE7\xE3o de centenas\ + \ de criaturas chorosas e sofredoras, escondidas em suas resid\xEAncias particulares,\ + \ e em cercados e cabanas \u2013 exclu\xEDdas, isoladas de todas as influ\xEA\ + ncias curativas, de todos os cuidados restauradores da mente... Poderiam suas\ + \ hist\xF3rias melanc\xF3licas ser espalhadas diante de voc\xEAs? conforme revelado\ + \ ao meu esp\xEDrito entristecido durante os \xFAltimos tr\xEAs meses, com que\ + \ rapidez e zelo voc\xEA procuraria os meios de al\xEDvio mais aprovados; qu\xE3\ + o insignificantes, qu\xE3o insignificantes, em compara\xE7\xE3o, pareceriam\ + \ os sacrif\xEDcios que voc\xEA \xE9 solicitado a fazer; como \xE9 que algumas\ + \ moedas de dez centavos e d\xF3lares, recolhidas de cada cidad\xE3o, diminuiriam\ + \ em valor como posse, em compara\xE7\xE3o com os certos benef\xEDcios e vasto\ + \ bem a ser assegurado para os insanos sofredores... pela consagra\xE7\xE3o\ + \ e aplica\xE7\xE3o de um fundo suficiente para a constru\xE7\xE3o de um hospital\ + \ adequado.\u2026 \u2014Dorothea Dix, Memorial Solicitando um Hospital Estadual\ + \ para a Prote\xE7\xE3o e Cura dos Insanos, apresentado \xE0 Assembleia Geral\ + \ da Carolina do Norte, novembro de 1848 Dorothea Dix pode ser melhor comparada\ + \ a quem?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_us_history +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_high_school_us_history +task_alias: high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eb9b4a461b7992e68648b89b27cb48993618559 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_high_school_world_history.yaml @@ -0,0 +1,107 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Hindu\xEDsmo" + B: budismo + C: "Xinto\xEDsmo" + D: Zoroastrismo + input_correct_responses: + - A + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ Ele cont\xE9m todas as obras e desejos e todos os perfumes e todos os gostos.\ + \ Ele envolve todo o universo e em sil\xEAncio \xE9 amoroso com todos. Este\ + \ \xE9 o Esp\xEDrito que est\xE1 em meu cora\xE7\xE3o, este \xE9 Brahman. A\ + \ ele irei quando for al\xE9m desta vida, e a ele vir\xE1 aquele que tiver f\xE9\ + \ e n\xE3o duvidar. \u2014Os Upanishads, \xCDndia, c. 1000 aC A que religi\xE3\ + o o orador provavelmente pertence?" + - input_choice_list: + A: "A revolu\xE7\xE3o industrial" + B: "Descoloniza\xE7\xE3o" + C: "Associa\xE7\xF5es Regionais de Livre Com\xE9rcio" + D: Autarquia + input_correct_responses: + - B + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ \u201CA luta contra o neocolonialismo n\xE3o visa excluir o capital do mundo\ + \ desenvolvido de operar nos pa\xEDses menos desenvolvidos. Visa evitar que\ + \ o poder financeiro dos pa\xEDses desenvolvidos seja utilizado de forma a empobrecer\ + \ os menos desenvolvidos. O n\xE3o-alinhamento, tal como praticado pelo Gana\ + \ e por muitos outros pa\xEDses, baseia-se na coopera\xE7\xE3o com todos os\ + \ Estados, sejam eles capitalistas, socialistas ou de economia mista. Tal pol\xED\ + tica, portanto, envolve investimento estrangeiro de pa\xEDses capitalistas,\ + \ mas deve ser investido de acordo com um plano nacional elaborado pelo governo\ + \ do Estado n\xE3o alinhado com os seus pr\xF3prios interesses em mente. A quest\xE3\ + o n\xE3o \xE9 qual o retorno que o investidor estrangeiro recebe dos seus investimentos...\ + \ A quest\xE3o \xE9 de poder. as garras do neocolonialismo n\xE3o s\xE3o donas\ + \ do seu pr\xF3prio destino." Kwame Nkrumah, Neo-Colonialismo, 1965 Qual\ + \ das alternativas a seguir fornece o melhor contexto para os escritos de Nkrumah?" + - input_choice_list: + A: "Aceita\xE7\xE3o social do trabalho infantil" + B: "Diminui\xE7\xE3o da esperan\xE7a de vida na Alemanha" + C: "Cr\xEDticas \xE0s tarifas comerciais alem\xE3s" + D: "Efeitos negativos atribu\xEDdos ao capitalismo industrial" + input_correct_responses: + - D + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ \u201CA verdadeira queixa do trabalhador \xE9 a inseguran\xE7a da sua exist\xEA\ + ncia; ele n\xE3o tem certeza de que sempre ter\xE1 trabalho, n\xE3o tem certeza\ + \ de que sempre ter\xE1 sa\xFAde e prev\xEA que um dia ficar\xE1 velho e incapacitado\ + \ para trabalhar Se ele cair na pobreza, mesmo que seja apenas devido a uma\ + \ doen\xE7a prolongada, ele ficar\xE1 completamente desamparado, por conta pr\xF3\ + pria, e a sociedade n\xE3o reconhece atualmente qualquer obriga\xE7\xE3o real\ + \ para com ele al\xE9m da ajuda habitual aos pobres, mesmo que ele tenha tenho\ + \ trabalhado o tempo todo com muita fidelidade e dilig\xEAncia. A ajuda habitual\ + \ aos pobres, por\xE9m, deixa muito a desejar, especialmente nas grandes cidades,\ + \ onde \xE9 muito pior do que no campo." Otto von Bismarck, 1884 Otto von\ + \ Bismarck provavelmente fez este discurso em rea\xE7\xE3o a qual das seguintes\ + \ quest\xF5es?" + - input_choice_list: + A: "A manuten\xE7\xE3o da supremacia militar a todo custo" + B: "Expans\xE3o das tens\xF5es entre seitas religiosas" + C: "Fatores que provocaram o colapso do Imp\xE9rio Otomano" + D: "Esfor\xE7os de pacifica\xE7\xE3o entre os imp\xE9rios isl\xE2micos" + input_correct_responses: + - B + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ "Na verdade, como tanto as fatwas de ilustres [estudiosos] que baseiam\ + \ as suas opini\xF5es tanto na raz\xE3o como na tradi\xE7\xE3o, e o consenso\ + \ da comunidade sunita concordam que a antiga obriga\xE7\xE3o de extirpa\xE7\ + \xE3o, exterm\xEDnio e expuls\xE3o da inova\xE7\xE3o maligna deve ser o objectivo\ + \ da nossa exaltada aspira\xE7\xE3o, pois "O zelo religioso \xE9 uma vit\xF3\ + ria para a F\xE9 de Deus, o Beneficente"; ent\xE3o, de acordo com as palavras\ + \ do Profeta (que a paz esteja com ele!) "Todo aquele que introduz inova\xE7\ + \xF5es malignas em nossa ordem deve ser expulso" e "Quem o faz qualquer\ + \ coisa contra a nossa ordem deve ser expulsa," a ac\xE7\xE3o tornou-se\ + \ necess\xE1ria e exigente\u2026" Carta do Sult\xE3o Otomano Selim I ao\ + \ X\xE1 Saf\xE1vida Ismail I, 1514 A carta de Selim I \xE9 mais claramente um\ + \ exemplo de qual das seguintes op\xE7\xF5es?" + - input_choice_list: + A: "Uma ruptura nas rotas comerciais atrav\xE9s do colapso da estrutura estatal\ + \ estabelecida" + B: "Um aumento na popula\xE7\xE3o do mundo atrav\xE9s de suprimentos mais abundantes\ + \ de alimentos" + C: "A dissemina\xE7\xE3o dos sistemas de cren\xE7as chineses e indianos em todo\ + \ o mundo" + D: "Um aumento na agita\xE7\xE3o social" + input_correct_responses: + - B + input_question: "Esta quest\xE3o refere-se \xE0s seguintes informa\xE7\xF5es.\ + \ "Pelo menos uma das sociedades [do mundo] teria de, de alguma forma,\ + \ aumentar enormemente a sua produtividade [para alcan\xE7ar a hegemonia global].\ + \ Esse salto qu\xE2ntico teria de ser dado antes das v\xE1rias revolu\xE7\xF5\ + es cient\xEDficas, tecnol\xF3gicas, agr\xEDcolas e industriais em que o nosso\ + \ o mundo p\xF3s-salto qu\xE2ntico repousa. Isso s\xF3 poderia ser alcan\xE7\ + ado atrav\xE9s da explora\xE7\xE3o dos ecossistemas, recursos minerais e ativos\ + \ humanos de continentes inteiros fora das terras da sociedade que deu o salto.\ + \ A Europa Ocidental fez exatamente isso por meio de sua brutalidade e armas\ + \ e , mais importante, por sorte geogr\xE1fica e ecol\xF3gica." Direitos\ + \ autorais \xA9 2015 Cambridge University Press. Alfred Crosby, historiador,\ + \ Imperialismo Ecol\xF3gico, 2004 O "salto qu\xE2ntico" mencionado\ + \ na passagem contribuiu mais diretamente para qual dos seguintes desenvolvimentos\ + \ no per\xEDodo 1450-1750 dC?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_world_history +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_high_school_world_history +task_alias: high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4faa2c0b82205361fdd6de31fba75e7d7cce826a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_human_aging.yaml @@ -0,0 +1,54 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Um homem ou mulher asi\xE1tica" + B: "Um homem hisp\xE2nico" + C: Uma mulher afro-americana + D: Um homem ou mulher branca + input_correct_responses: + - C + input_question: Qual das seguintes pessoas tem maior probabilidade de permanecer + sozinha em casa, a partir de 2019? + - input_choice_list: + A: "Vantagem da adolesc\xEAncia" + B: "Aumento de reminisc\xEAncia" + C: Memorial comemorativo + D: "Pico de recupera\xE7\xE3o quadr\xE1tica" + input_correct_responses: + - B + input_question: "A descoberta de que os adultos tendem a lembrar-se melhor dos\ + \ acontecimentos da sua adolesc\xEAncia do que de outros per\xEDodos da sua\ + \ vida \xE9 referida como a" + - input_choice_list: + A: Texas + B: "Calif\xF3rnia" + C: "Hava\xED" + D: Vermonte + input_correct_responses: + - A + input_question: "Quando os idosos se mudam para um novo estado ap\xF3s a reforma,\ + \ qual das seguintes op\xE7\xF5es \xE9 o destino mais prov\xE1vel?" + - input_choice_list: + A: Nicotina + B: "Alcatr\xE3o" + C: "Mon\xF3xido de carbono" + D: "Part\xEDculas de fuma\xE7a" + input_correct_responses: + - B + input_question: "Qual elemento da fuma\xE7a do tabaco \xE9 respons\xE1vel pelo\ + \ c\xE2ncer?" + - input_choice_list: + A: Uma mulher hispano-americana mais velha + B: Uma mulher afro-americana mais velha + C: "Uma mulher asi\xE1tico-americana mais velha" + D: Uma mulher nativa americana mais velha + input_correct_responses: + - C + input_question: "Mantendo todas as outras condi\xE7\xF5es iguais, qual das seguintes\ + \ pessoas tem maior probabilidade de apresentar osteoporose?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_human_aging +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_human_aging +task_alias: human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62fd11c9955009abc533c20e055631bf7fb1b216 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_human_sexuality.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: durante o primeiro trimestre + B: durante o segundo trimestre + C: durante o terceiro trimestre + D: durante toda a gravidez + input_correct_responses: + - A + input_question: "O enj\xF4o matinal costuma ser um problema:" + - input_choice_list: + A: um ego forte + B: um superego fraco + C: "uma identifica\xE7\xE3o fraca" + D: um superego forte + input_correct_responses: + - B + input_question: "Uma mulher que sabe que tem herpes activo e s\xEDfilis n\xE3\ + o tratada, mas continua a ter rela\xE7\xF5es sexuais sem informar os seus parceiros\ + \ sobre a sua condi\xE7\xE3o, tem, em termos psicanal\xEDticos:" + - input_choice_list: + A: "o fato de n\xE3o possu\xEDrem per\xEDodo refrat\xE1rio." + B: a resposta das camadas internas da vagina. + C: tendo orgasmos alternados em locais diferentes. + D: o ponto G. + input_correct_responses: + - A + input_question: "A capacidade das mulheres de ter orgasmos m\xFAltiplos se deve\ + \ principalmente a:" + - input_choice_list: + A: "rela\xE7\xE3o sexual" + B: "c\xEDrculo empurr\xF5es" + C: exibicionismo + D: "tocando os \xF3rg\xE3os genitais um do outro" + input_correct_responses: + - A + input_question: "A natureza das atividades homossexuais que ocorrem durante a\ + \ pr\xE9-adolesc\xEAncia inclui todas as op\xE7\xF5es a seguir, exceto qual?" + - input_choice_list: + A: "ejacula\xE7\xE3o precoce" + B: "ejacula\xE7\xE3o inibida" + C: "dist\xFArbio er\xE9til" + D: "dist\xFArbio ejaculat\xF3rio" + input_correct_responses: + - C + input_question: "O dist\xFArbio mais comum entre homens que procuram terapia sexual\ + \ \xE9:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_human_sexuality +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_human_sexuality +task_alias: human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a87ccd20141edcea9c690569acd84dbe0f5b57fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_international_law.yaml @@ -0,0 +1,70 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "O Artigo 2(4) abrange apenas a for\xE7a armada" + B: "O Artigo 2(4) abrange todos os tipos de for\xE7a, incluindo san\xE7\xF5\ + es" + C: "O Artigo 2(4) abrange todas as interfer\xEAncias nos assuntos internos dos\ + \ Estados" + D: "O Artigo 2(4) abrange a for\xE7a dirigida apenas contra a integridade territorial\ + \ de um Estado" + input_correct_responses: + - A + input_question: "Que tipos de for\xE7a o Artigo 2(4) da Carta das Na\xE7\xF5es\ + \ Unidas pro\xEDbe?" + - input_choice_list: + A: "Se uma parte em um caso contencioso perante a CIJ n\xE3o tiver um cidad\xE3\ + o nacional como juiz, ela ter\xE1 o direito de nomear algu\xE9m como juiz\ + \ exclusivamente para esse caso, com o t\xEDtulo de juiz ad hoc." + B: "Juiz ad hoc \xE9 membro da bancada da CIJ com voto de qualidade" + C: "O juiz ad hoc \xE9 um juiz substituto, caso um juiz seja desqualificado\ + \ ou fale\xE7a" + D: "Juiz ad hoc \xE9 o juiz que cada parte sempre nomear\xE1 em cada caso contencioso" + input_correct_responses: + - A + input_question: "O que \xE9 o juiz ad hoc?" + - input_choice_list: + A: "Esta \xE9 uma reserva aceit\xE1vel se a legisla\xE7\xE3o do pa\xEDs que\ + \ fez a reserva empregar uma defini\xE7\xE3o diferente" + B: "Esta \xE9 uma reserva inaceit\xE1vel porque contraria o objecto e a finalidade\ + \ do PIDCP" + C: "Esta \xE9 uma reserva inaceit\xE1vel porque a defini\xE7\xE3o de tortura\ + \ no PIDCP \xE9 consistente com o direito internacional consuetudin\xE1rio" + D: "Esta \xE9 uma reserva aceit\xE1vel porque, de acordo com o direito internacional\ + \ geral, os Estados t\xEAm o direito de inserir reservas aos tratados" + input_correct_responses: + - B + input_question: "Ser\xE1 que uma reserva \xE0 defini\xE7\xE3o de tortura constante\ + \ do PIDCP seria aceit\xE1vel na pr\xE1tica contempor\xE2nea?" + - input_choice_list: + A: "O consentimento pode servir como circunst\xE2ncia que exclui a ilicitude\ + \ sempre que for dado" + B: "O consentimento nunca pode servir como uma circunst\xE2ncia que exclui a\ + \ ilicitude" + C: "O consentimento pode servir como circunst\xE2ncia que exclui a ilicitude,\ + \ desde que o consentimento seja v\xE1lido e na medida em que a conduta permane\xE7\ + a dentro dos limites do consentimento dado" + D: "O consentimento pode sempre servir como circunst\xE2ncia que exclui a ilicitude,\ + \ independentemente do \xF3rg\xE3o do Estado que o conceda." + input_correct_responses: + - C + input_question: "Quando o \u201Cconsentimento\u201D pode servir como uma circunst\xE2\ + ncia que exclui a ilicitude de uma conduta estatal?" + - input_choice_list: + A: "O consentimento de um Estado em ficar vinculado \xE9 expresso apenas por\ + \ ratifica\xE7\xE3o" + B: "O consentimento de um Estado em ficar vinculado por um tratado pode ser\ + \ expresso por assinatura, ratifica\xE7\xE3o, aceita\xE7\xE3o, aprova\xE7\xE3\ + o ou ades\xE3o." + C: "O consentimento de um Estado em ficar vinculado \xE9 expresso pela assinatura" + D: "O consentimento de um Estado em ficar vinculado \xE9 expresso por qualquer\ + \ meio que escolher" + input_correct_responses: + - B + input_question: Como pode ser expresso o consentimento de um Estado em obrigar-se? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_international_law +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_international_law +task_alias: international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec39d94f47a56113aec8e9e304fe472b28a6d249 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_jurisprudence.yaml @@ -0,0 +1,55 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: O POP escolheria a igualdade acima da liberdade. + B: "O POP optaria pela estrat\xE9gia 'maximin'." + C: "O POP optaria pelo \u201Cprinc\xEDpio da diferen\xE7a\u201D." + D: "O POP rejeitaria o \u201Csistema de liberdade natural\u201D." + input_correct_responses: + - A + input_question: "Qual posi\xE7\xE3o Rawls afirma ser a menos prov\xE1vel de ser\ + \ adotada pelo POP (pessoas na posi\xE7\xE3o original)?" + - input_choice_list: + A: maximizando a liberdade individual + B: fornecendo uma base para um compromisso + C: mantendo a paz + D: "promovendo os princ\xEDpios do sistema de livre iniciativa" + input_correct_responses: + - D + input_question: "As fun\xE7\xF5es da lei incluem todas as op\xE7\xF5es a seguir,\ + \ exceto quais?" + - input_choice_list: + A: Autoridade. + B: Carisma. + C: "Coopera\xE7\xE3o." + D: Capitalismo. + input_correct_responses: + - D + input_question: "Qual palavra resume melhor a explica\xE7\xE3o de Weber sobre\ + \ o desenvolvimento do direito formalmente racional?" + - input_choice_list: + A: Lei natural + B: "Anal\xEDtico" + C: "Hist\xF3rico" + D: "Sociol\xF3gico" + input_correct_responses: + - A + input_question: "A Escola ________ de jurisprud\xEAncia postula que a lei se baseia\ + \ no que \xE9 "correto"." + - input_choice_list: + A: "A carta \xE9 uma oferta de venda" + B: "Uma oferta v\xE1lida n\xE3o pode ser feita por carta." + C: "A carta cont\xE9m uma oferta v\xE1lida que ser\xE1 encerrada dentro de um\ + \ prazo razo\xE1vel." + D: A carta carece de um dos elementos essenciais de uma oferta. + input_correct_responses: + - D + input_question: "A Iverson Jewelers escreveu uma carta a Miller: 'Recebemos\ + \ um rel\xF3gio Rolox de corda autom\xE1tica excepcionalmente fino que venderemos\ + \ a voc\xEA por um pre\xE7o muito favor\xE1vel.'" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_jurisprudence +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_jurisprudence +task_alias: jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a826cbe4352f6fc57bb9875ba1b49fe53f5b210 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_logical_fallacies.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "voc\xEA \xE9 quem" + B: risada de cavalo + C: argumento contra a pessoa + D: ignoratio elenchi + input_correct_responses: + - C + input_question: "Se algu\xE9m atacar o car\xE1ter de um argumentador advers\xE1\ + rio, em vez de responder aos argumentos desse oponente, a primeira pessoa provavelmente\ + \ cometeu qual das seguintes fal\xE1cias?" + - input_choice_list: + A: "argumentar que algo \xE9 inferior s\xF3 porque n\xE3o faz algo que nunca\ + \ foi planejado." + B: "incluir mais de uma afirma\xE7\xE3o na proposi\xE7\xE3o e tratar a prova\ + \ de uma afirma\xE7\xE3o como prova de todas as afirma\xE7\xF5es." + C: "tirar uma conclus\xE3o antes de examinar as evid\xEAncias e considerar apenas\ + \ as evid\xEAncias que apoiam essa conclus\xE3o." + D: "fazer uma pergunta que inclua uma suposi\xE7\xE3o n\xE3o comprovada ou mais\ + \ de uma pergunta, tornando assim uma resposta direta, sim ou n\xE3o, sem\ + \ sentido." + input_correct_responses: + - D + input_question: "A fal\xE1cia da quest\xE3o complexa consiste em" + - input_choice_list: + A: A premissa menor deve negar o antecedente + B: A premissa maior deve afirmar o consequente + C: "O termo m\xE9dio deve ser usado em pelo menos uma premissa em um sentido\ + \ universal ou n\xE3o qualificado" + D: Tudo o que precede + input_correct_responses: + - C + input_question: "Qual das afirma\xE7\xF5es a seguir \xE9 verdadeira sobre um silogismo\ + \ categ\xF3rico v\xE1lido?" + - input_choice_list: + A: "Divis\xE3o" + B: "Composi\xE7\xE3o" + C: "Apelo \xE0 pessoa" + D: "Apelo \xE0 ignor\xE2ncia" + input_correct_responses: + - B + input_question: "Argumentar que o que \xE9 verdade para as partes deve ser verdade\ + \ para o todo \xE9 a fal\xE1cia de..." + - input_choice_list: + A: "mau esp\xEDrito esportivo" + B: "apelo \xE0 compaix\xE3o" + C: argumento contra a pessoa + D: "ignor\xE2ncia da refuta\xE7\xE3o" + input_correct_responses: + - D + input_question: "Quando um argumentador causa confus\xE3o durante a refuta\xE7\ + \xE3o devido \xE0 falta real ou fingida de capacidade de se envolver na refuta\xE7\ + \xE3o, esse argumentador pode ter cometido a fal\xE1cia de" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_logical_fallacies +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_logical_fallacies +task_alias: logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..090f1857916a65589fb2835a6b8509d6b6e67b41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_machine_learning.yaml @@ -0,0 +1,72 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: 2,0/15 + B: 1,0/7 + C: 3.0/16 + D: 1,0/5 + input_correct_responses: + - B + input_question: "Um dado de 6 faces \xE9 lan\xE7ado 15 vezes e os resultados s\xE3\ + o: o lado 1 aparece 0 vezes; lado 2: 1 vez; lado 3: 2 vezes; lado 4: 3 vezes;\ + \ lado 5: 4 vezes; lado 6: 5 vezes. Com base nesses resultados, qual \xE9 a\ + \ probabilidade de o lado 3 aparecer ao usar a suaviza\xE7\xE3o Add-1?" + - input_choice_list: + A: "corte aleat\xF3rio e invers\xE3o horizontal" + B: "corte aleat\xF3rio e invers\xE3o vertical" + C: "posteriza\xE7\xE3o" + D: hesitante + input_correct_responses: + - A + input_question: "Qual aumento de dados de imagem \xE9 mais comum para imagens\ + \ naturais?" + - input_choice_list: + A: "Meu m\xE9todo atinge um erro de treinamento menor que todos os m\xE9todos\ + \ anteriores!" + B: "Meu m\xE9todo atinge um erro de teste menor que todos os m\xE9todos anteriores!\ + \ (Nota de rodap\xE9: Quando o par\xE2metro de regulariza\xE7\xE3o \u03BB\ + \ \xE9 escolhido de modo a minimizar o erro de teste.)" + C: "Meu m\xE9todo atinge um erro de teste menor que todos os m\xE9todos anteriores!\ + \ (Nota de rodap\xE9: Quando o par\xE2metro de regulariza\xE7\xE3o \u03BB\ + \ \xE9 escolhido de modo a minimizar o erro de valida\xE7\xE3o cruzada.)" + D: "Meu m\xE9todo atinge um erro de valida\xE7\xE3o cruzada inferior a todos\ + \ os m\xE9todos anteriores! (Nota de rodap\xE9: Quando o par\xE2metro de regulariza\xE7\ + \xE3o \u03BB \xE9 escolhido de modo a minimizar o erro de valida\xE7\xE3o\ + \ cruzada.)" + input_correct_responses: + - C + input_question: "Voc\xEA est\xE1 revisando artigos para a Confer\xEAncia de Aprendizado\ + \ de M\xE1quina mais sofisticada do mundo e v\xEA submiss\xF5es com as seguintes\ + \ afirma\xE7\xF5es. Quais voc\xEA consideraria aceitar?" + - input_choice_list: + A: cerca de 10 exemplos + B: cerca de 100 exemplos + C: entre 100 e 500 exemplos + D: mais de 1000 exemplos + input_correct_responses: + - D + input_question: Para obter uma estimativa de perda 0/1 inferior a 1 por cento + da perda 0/1 verdadeira (com probabilidade de 95%), de acordo com a desigualdade + de Hoeffding, o conjunto de testes IID deve ter quantos exemplos? + - input_choice_list: + A: "\xC9 muito caro computacionalmente." + B: "Provavelmente resultaria em uma \xE1rvore de decis\xE3o com pontua\xE7\xE3\ + o ruim no conjunto de treinamento e no conjunto de teste." + C: "Provavelmente resultaria em uma \xE1rvore de decis\xE3o com boa pontua\xE7\ + \xE3o no conjunto de treinamento, mas ruim em um conjunto de teste." + D: "Provavelmente resultaria em uma \xE1rvore de decis\xE3o com boa pontua\xE7\ + \xE3o em um conjunto de teste, mas ruim em um conjunto de treinamento." + input_correct_responses: + - C + input_question: "Tradicionalmente, quando temos um atributo de entrada com valor\ + \ real durante o aprendizado da \xE1rvore de decis\xE3o, consideramos uma divis\xE3\ + o bin\xE1ria dependendo se o atributo est\xE1 acima ou abaixo de algum limite.\ + \ Pat sugere que, em vez disso, dever\xEDamos apenas ter uma divis\xE3o multidirecional\ + \ com uma ramifica\xE7\xE3o para cada um dos valores distintos do atributo.\ + \ Na lista abaixo, escolha o maior problema com a sugest\xE3o de Pat:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_machine_learning +tag: mmlu_pt_llama_stem_tasks +task: mmlu_pt_llama_machine_learning +task_alias: machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_management.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6beae30405ac6c051ba4d8453a594e1dafbf04e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_management.yaml @@ -0,0 +1,51 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Posi\xE7\xE3o inicial e posi\xE7\xE3o final" + B: Ambiente inicial e ambiente alterado + C: Estrutura organizacional e condicionamento + D: "Estrutura inicial e considera\xE7\xF5es" + input_correct_responses: + - D + input_question: "Quais s\xE3o as duas dimens\xF5es principais dos Estudos de Ohio\ + \ sobre lideran\xE7a?" + - input_choice_list: + A: Frederico Hertzberg + B: DC McClelland + C: Abraham Maslow + D: Douglas McGregor + input_correct_responses: + - A + input_question: "Fatores de higiene est\xE3o associados a qual escritor?" + - input_choice_list: + A: "S\xEDmbolos" + B: Rituais e rotinas + C: Estruturas de poder + D: Sistemas de controle + input_correct_responses: + - A + input_question: Qual elemento da teia cultural forma os trajes? + - input_choice_list: + A: Moral + B: "Inova\xE7\xE3o" + C: Recurso de crescimento + D: "Adapta\xE7\xE3o" + input_correct_responses: + - A + input_question: "Que caracter\xEDstica n\xE3o \xE9 fundamental no modelo de gest\xE3\ + o de \u201Csistemas abertos\u201D?" + - input_choice_list: + A: "Hier\xE1rquico" + B: "Burocr\xE1tico" + C: Plano + D: Funcional + input_correct_responses: + - C + input_question: "Como podem ser descritas as estruturas organizacionais caracterizadas\ + \ por estilos de gest\xE3o democr\xE1ticos e inclusivos?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_management +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_management +task_alias: management diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb2191a4b78fa2fe1b889fab326b620cace22900 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_marketing.yaml @@ -0,0 +1,60 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Geodemografia + B: "Diferencia\xE7\xE3o do produto." + C: Matriz ANSOFF. + D: "Gest\xE3o de marca." + input_correct_responses: + - A + input_question: "_____________ \xE9 um resultado natural quando se combinam vari\xE1\ + veis demogr\xE1ficas e geogr\xE1ficas." + - input_choice_list: + A: "Unidade de terceiriza\xE7\xE3o." + B: Centro de compras. + C: Unidade executiva principal. + D: "Unidade de tomada de decis\xE3o." + input_correct_responses: + - D + input_question: "Em uma organiza\xE7\xE3o, o grupo de pessoas encarregadas de\ + \ tomar decis\xF5es de compra \xE9 denominado _______________." + - input_choice_list: + A: "As necessidades dependem da cultura e tamb\xE9m da classe social." + B: "As necessidades de n\xEDvel inferior devem ser pelo menos parcialmente satisfeitas\ + \ antes que necessidades superiores possam afetar o comportamento." + C: "As necessidades n\xE3o s\xE3o priorizadas ou organizadas em nenhuma ordem\ + \ espec\xEDfica." + D: "As necessidades satisfeitas s\xE3o motivadores e novas necessidades surgem\ + \ quando as necessidades atuais permanecem n\xE3o satisfeitas." + input_correct_responses: + - B + input_question: "Qual das alternativas a seguir \xE9 uma suposi\xE7\xE3o na hierarquia\ + \ de necessidades de Maslow?" + - input_choice_list: + A: "O consumidor mais velho que se sente um tanto exclu\xEDdo." + B: As mulheres casadas, muitas das quais sentem necessidade de estabilidade + nas suas vidas. + C: Novos imigrantes que realmente desejam assimilar sua nova cultura. + D: "Crian\xE7as, que baseiam a maior parte das suas decis\xF5es de compra em\ + \ influ\xEAncias externas." + input_correct_responses: + - D + input_question: "O \xFAnico grupo dentro da sociedade que \xE9 mais vulner\xE1\ + vel \xE0 influ\xEAncia do grupo de refer\xEAncia \xE9:" + - input_choice_list: + A: Linhas de cuidado. + B: Mala direta. + C: "Inser\xE7\xF5es." + D: De porta em porta. + input_correct_responses: + - D + input_question: "Embora o conte\xFAdo e a qualidade possam ser t\xE3o controlados\ + \ quanto a mala direta, as taxas de resposta desse meio s\xE3o mais baixas devido\ + \ \xE0 falta de um mecanismo de endere\xE7o pessoal. Este formato de m\xEDdia\ + \ \xE9 conhecido como:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_marketing +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_marketing +task_alias: marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c589ffe1120f366f73ee83a9de4be920d318b00d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_medical_genetics.yaml @@ -0,0 +1,51 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "rea\xE7\xE3o em cadeia da polimerase." + B: "an\xE1lise de polimorfismo conformacional de fita simples." + C: Mancha do sul. + D: Mancha ocidental. + input_correct_responses: + - C + input_question: "Grandes expans\xF5es de repeti\xE7\xE3o tripla podem ser detectadas\ + \ por:" + - input_choice_list: + A: "uma enzima que une fragmentos na replica\xE7\xE3o normal do DNA" + B: "uma enzima de origem bacteriana que corta o DNA em sequ\xEAncias de bases\ + \ definidas" + C: "uma enzima que facilita a transcri\xE7\xE3o de genes espec\xEDficos" + D: "uma enzima que limita o n\xEDvel a que um determinado nutriente atinge" + input_correct_responses: + - A + input_question: "DNA ligase \xE9" + - input_choice_list: + A: tem ambos os alelos expressos independentemente no heterozigoto + B: tem um alelo dominante para o outro + C: tem alelos fortemente ligados no mesmo cromossomo + D: tem alelos expressos ao mesmo tempo no desenvolvimento + input_correct_responses: + - A + input_question: "Um gene mostrando codomin\xE2ncia" + - input_choice_list: + A: "Estenose pil\xF3rica" + B: Esquizofrenia + C: "Espinha b\xEDfida (defeitos do tubo neural)" + D: "s\xEDndrome de Marfan" + input_correct_responses: + - D + input_question: "Qual das seguintes condi\xE7\xF5es n\xE3o mostra heran\xE7a multifatorial?" + - input_choice_list: + A: "pr\xF3fase I" + B: "met\xE1fase I" + C: "pr\xF3fase II" + D: "met\xE1fase II" + input_correct_responses: + - A + input_question: "O est\xE1gio da meiose em que os cromossomos se emparelham e\ + \ se cruzam \xE9:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_medical_genetics +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_medical_genetics +task_alias: medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19ccdacc0cd3b099d8999659c1a97e0c61282712 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_miscellaneous.yaml @@ -0,0 +1,51 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: um + B: dois + C: quatro + D: oito + input_correct_responses: + - B + input_question: "Quantos eixos tem um autom\xF3vel padr\xE3o?" + - input_choice_list: + A: Budapeste + B: Budokan + C: "But\xE3o" + D: "Gr\xE3-Bretanha" + input_correct_responses: + - B + input_question: "Que lugar \xE9 nomeado no t\xEDtulo do \xE1lbum ao vivo de 1979\ + \ das lendas do rock Cheap Trick?" + - input_choice_list: + A: Anthony 'Spud' Webb + B: Michael 'Air' Jordan + C: Tyrone 'Muggsy' Bogues + D: Julius 'Dr. J' Erving + input_correct_responses: + - A + input_question: "Quem \xE9 o homem mais baixo a vencer uma competi\xE7\xE3o de\ + \ enterradas da NBA?" + - input_choice_list: + A: "hidrog\xEAnio" + B: nylon + C: "oxig\xEAnio" + D: luz + input_correct_responses: + - C + input_question: "O que \xE9 produzido durante a fotoss\xEDntese?" + - input_choice_list: + A: "'R\xE1dio Ga Ga'" + B: ''Ob-la-di Ob-la-da'' + C: ''De Do Do Do De Da Da Da'' + D: ''In-a-Gadda-Da-Vida'' + input_correct_responses: + - C + input_question: "Qual dessas m\xFAsicas foi um hit Top 10 da banda de rock The\ + \ Police?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_miscellaneous +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_miscellaneous +task_alias: miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22274cdf2857f3e8bf0a7122b0292139e9b0bc41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_moral_disputes.yaml @@ -0,0 +1,62 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Ainda n\xE3o est\xE1 claro se a pena capital impede ou n\xE3o danos \xE0\ + \ comunidade." + B: "Ainda n\xE3o est\xE1 claro se a pena capital impede ou n\xE3o danos a quaisquer\ + \ indiv\xEDduos." + C: "O consequencialismo n\xE3o \xE9 apoiado pelos valores africanos." + D: "Embora o consequencialismo seja apoiado por valores africanos, n\xE3o foram\ + \ apresentados quaisquer argumentos consequencialistas enquadrados em termos\ + \ de valores africanos." + input_correct_responses: + - A + input_question: "De acordo com Metz, o que h\xE1 de errado com os argumentos consequencialistas\ + \ contra a pena capital baseados em valores africanos?" + - input_choice_list: + A: um zigoto. + B: "um embri\xE3o." + C: uma viabilidade. + D: um blastocisto. + input_correct_responses: + - A + input_question: "Um \xF3vulo fertilizado tamb\xE9m \xE9 conhecido como" + - input_choice_list: + A: estar na classe alta + B: a capacidade de aprender e resolver problemas + C: "sendo mais bonita que a maioria da popula\xE7\xE3o" + D: "sendo mais curto que a m\xE9dia" + input_correct_responses: + - B + input_question: "Qual das alternativas a seguir \xE9 um bem intr\xEDnseco, de\ + \ acordo com Singer?" + - input_choice_list: + A: "A SCNT n\xE3o \xE9 um tipo diferente de reprodu\xE7\xE3o porque n\xE3o existem\ + \ diferen\xE7as moralmente relevantes entre ela e outros meios de reprodu\xE7\ + \xE3o permitidos." + B: "Como existe um baixo risco de danos para a gemelaridade natural, haver\xE1\ + \ um baixo risco de danos para o TNCS." + C: ambos a e B + D: nem A nem B + input_correct_responses: + - A + input_question: "Pence compara seis casos diferentes de reprodu\xE7\xE3o, desde\ + \ gemelaridade natural at\xE9 SCNT. Que conclus\xE3o ele tira dessa compara\xE7\ + \xE3o?" + - input_choice_list: + A: "a estupidez da maioria dos exemplos filos\xF3ficos tradicionais." + B: "uma falta geral de intelig\xEAncia entre pessoas com diplomas avan\xE7ados." + C: o erro da tortura. + D: "a disponibilidade por parte de muitas pessoas inteligentes para verem a\ + \ tortura como a melhor solu\xE7\xE3o para lidar com o terrorismo." + input_correct_responses: + - D + input_question: "Baron admite que as vers\xF5es hipot\xE9ticas da bomba-rel\xF3\ + gio que ela discute s\xE3o "incrivelmente est\xFApidas", mas ela afirma\ + \ que isso \xE9 na verdade uma evid\xEAncia de" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_moral_disputes +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_moral_disputes +task_alias: moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f31115bf90669d20568c24f3e492c59ab2b8b39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_moral_scenarios.yaml @@ -0,0 +1,68 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Errado errado + B: "Errado, n\xE3o errado" + C: "N\xE3o est\xE1 errado, errado" + D: "N\xE3o est\xE1 errado, n\xE3o est\xE1 errado" + input_correct_responses: + - C + input_question: "Para qual destes dois cen\xE1rios o personagem principal (que\ + \ usa eu/me/meu) faz algo claramente moralmente errado, de acordo com os padr\xF5\ + es morais comuns nos EUA em 2020? Cen\xE1rio 1 | Eu realmente amei meu filho\ + \ rec\xE9m-nascido. Cen\xE1rio 2 | A novata tem vergonha de morar em um trailer,\ + \ ent\xE3o convido a turma para ir \xE0 casa dela." + - input_choice_list: + A: Errado errado + B: "Errado, n\xE3o errado" + C: "N\xE3o est\xE1 errado, errado" + D: "N\xE3o est\xE1 errado, n\xE3o est\xE1 errado" + input_correct_responses: + - C + input_question: "Para qual destes dois cen\xE1rios o personagem principal (que\ + \ usa eu/me/meu) faz algo claramente moralmente errado, de acordo com os padr\xF5\ + es morais comuns nos EUA em 2020? Cen\xE1rio 1 | Pedi um cigarro ao cara do\ + \ lado de fora da loja. Cen\xE1rio 2 | Quando o seguran\xE7a deixou o posto,\ + \ embolsei uma moeda rara." + - input_choice_list: + A: Errado errado + B: "Errado, n\xE3o errado" + C: "N\xE3o est\xE1 errado, errado" + D: "N\xE3o est\xE1 errado, n\xE3o est\xE1 errado" + input_correct_responses: + - D + input_question: "Para qual destes dois cen\xE1rios o personagem principal (que\ + \ usa eu/me/meu) faz algo claramente moralmente errado, de acordo com os padr\xF5\ + es morais comuns nos EUA em 2020? Cen\xE1rio 1 | Comprei uma caminhonete velha\ + \ para trabalhar. Cen\xE1rio 2 | Olhei para a linda cachoeira e joguei uma pedra\ + \ nela." + - input_choice_list: + A: Errado errado + B: "Errado, n\xE3o errado" + C: "N\xE3o est\xE1 errado, errado" + D: "N\xE3o est\xE1 errado, n\xE3o est\xE1 errado" + input_correct_responses: + - B + input_question: "Para qual destes dois cen\xE1rios o personagem principal (que\ + \ usa eu/me/meu) faz algo claramente moralmente errado, de acordo com os padr\xF5\ + es morais comuns nos EUA em 2020? Cen\xE1rio 1 | Durante a demoli\xE7\xE3o da\ + \ casa, empurrei-o contra a parede. Cen\xE1rio 2 | Quando escolho uma bab\xE1\ + \ para minhas sobrinhas g\xEAmeas, n\xE3o me importo com sua origem racial." + - input_choice_list: + A: Errado errado + B: "Errado, n\xE3o errado" + C: "N\xE3o est\xE1 errado, errado" + D: "N\xE3o est\xE1 errado, n\xE3o est\xE1 errado" + input_correct_responses: + - A + input_question: "Para qual destes dois cen\xE1rios o personagem principal (que\ + \ usa eu/me/meu) faz algo claramente moralmente errado, de acordo com os padr\xF5\ + es morais comuns nos EUA em 2020? Cen\xE1rio 1 | Peguei a carteira dele depois\ + \ que ele me assediou no bar. Cen\xE1rio 2 | Fiz amor com meu filho rec\xE9\ + m-nascido." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_moral_scenarios +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_moral_scenarios +task_alias: moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40528784b758cb9b982b14e4e534782aec0ec29f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_nutrition.yaml @@ -0,0 +1,68 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Os bebedores de ch\xE1 t\xEAm menor risco de desenvolver diabetes." + B: "Os bebedores de ch\xE1 t\xEAm maior risco de desenvolver diabetes." + C: "Com base nas informa\xE7\xF5es fornecidas, n\xE3o podemos dizer se a diferen\xE7\ + a observada no risco de doen\xE7a \xE9 resultado do acaso." + D: "A raz\xE3o de risco est\xE1 pr\xF3xima do valor um, portanto n\xE3o h\xE1\ + \ diferen\xE7a no risco de doen\xE7a entre os dois grupos." + input_correct_responses: + - C + input_question: "Num estudo de coorte, a taxa de risco de desenvolver diabetes\ + \ foi de 0,86 quando se comparou os consumidores de ch\xE1 (os expostos) com\ + \ aqueles que n\xE3o beberam ch\xE1 (os n\xE3o expostos). Qual afirma\xE7\xE3\ + o est\xE1 correta (de acordo com o conhecimento em 2020)?" + - input_choice_list: + A: "Consumidores com fenilceton\xFAria devem evitar o consumo do ado\xE7ante\ + \ aspartame" + B: "Consumidores com fenilceton\xFAria devem evitar o consumo do ado\xE7ante\ + \ sacarina" + C: "Consumidores com fenilceton\xFAria devem evitar o consumo do ado\xE7ante\ + \ sucralose" + D: "Consumidores com fenilceton\xFAria devem evitar o consumo do ado\xE7ante\ + \ acessulfame K" + input_correct_responses: + - A + input_question: "Qual das seguintes afirma\xE7\xF5es est\xE1 correta (de acordo\ + \ com o conhecimento em 2020)?" + - input_choice_list: + A: "O \xE1cido propi\xF4nico, formado durante a fermenta\xE7\xE3o das fibras\ + \ do c\xF3lon, inibe a s\xEDntese de \xE1cidos graxos no f\xEDgado" + B: "O \xE1cido but\xEDrico, formado durante a fermenta\xE7\xE3o das fibras do\ + \ c\xF3lon, estimula o "silenciamento" do gene supressor de tumor\ + \ SLC5A8" + C: "Nenhuma dessas op\xE7\xF5es est\xE1 correta" + D: "O \xE1cido but\xEDrico, formado durante a fermenta\xE7\xE3o das fibras do\ + \ c\xF3lon, estimula as defesas antioxidantes no c\xF3lon" + input_correct_responses: + - D + input_question: "Qual das alternativas a seguir \xE9 a explica\xE7\xE3o mais plaus\xED\ + vel para o efeito protetor da fibra alimentar contra o c\xE2ncer de c\xF3lon,\ + \ a partir de 2020?" + - input_choice_list: + A: "50% dos adultos consomem iodo em n\xEDveis abaixo do RNI" + B: "Os produtos l\xE1cteos s\xE3o uma fonte pobre de iodo" + C: "O teor de iodo do leite org\xE2nico \xE9 geralmente inferior ao n\xEDvel\ + \ do leite n\xE3o org\xE2nico" + D: "Os valores de refer\xEAncia diet\xE9ticos do Reino Unido recomendam um aumento\ + \ na ingest\xE3o de iodo durante a gravidez" + input_correct_responses: + - C + input_question: "Qual das seguintes afirma\xE7\xF5es sobre o iodo est\xE1 correta\ + \ em 2020?" + - input_choice_list: + A: Acarbose + B: Metformina + C: Sulfonilureias + D: Insulina + input_correct_responses: + - B + input_question: "Qual \xE9 o medicamento de primeira linha para pacientes com\ + \ diabetes tipo 2 e obesidade, a partir de 2020?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_nutrition +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_nutrition +task_alias: nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8448147d68f119281b07abf70e442823d2a4fcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_philosophy.yaml @@ -0,0 +1,53 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "uma teoria \xE9tica sobre como devemos nos comportar." + B: "uma generaliza\xE7\xE3o sobre a maneira como as pessoas tendem a se comportar." + C: "uma afirma\xE7\xE3o sobre a natureza humana e as maneiras como as pessoas\ + \ s\xE3o capazes de se comportar." + D: nenhuma das acima. + input_correct_responses: + - C + input_question: "O ego\xEDsmo psicol\xF3gico \xE9:" + - input_choice_list: + A: prazer. + B: felicidade. + C: bom. + D: virtude. + input_correct_responses: + - C + input_question: "De acordo com o \u201Cutilitarismo ideal\u201D de Moore, a a\xE7\ + \xE3o correta \xE9 aquela que provoca a maior quantidade de:" + - input_choice_list: + A: escolhas livres + B: ditames da alma + C: "leis naturais necess\xE1rias" + D: vontade indeterminada + input_correct_responses: + - C + input_question: Segundo d'Holbach, as pessoas sempre agem de acordo com _____. + - input_choice_list: + A: otimista + B: satisfeito + C: nominalmente religioso + D: pessimista + input_correct_responses: + - D + input_question: "Antes da convers\xE3o crist\xE3 de Tolstoi, qual era a sua perspectiva\ + \ sobre o sentido da vida?" + - input_choice_list: + A: "metaf\xEDsica" + B: epistemologia + C: "f\xEDsica qu\xE2ntica" + D: axiologia + input_correct_responses: + - A + input_question: "O estudo da realidade no sentido mais amplo, uma investiga\xE7\ + \xE3o sobre a natureza elementar do universo e das coisas nele contidas, \xE9\ + \ conhecido como _____." +include: _continuation_template_yaml +process_docs: !function utils.process_docs_philosophy +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_philosophy +task_alias: philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb35c49d59c152d6f2cd1073802591aecf1c488e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_prehistory.yaml @@ -0,0 +1,63 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "satisfazer os poderosos sacerdotes astr\xF4nomos maias." + B: "mostrar sua generosidade para com as pessoas comuns, j\xE1 que lhes era\ + \ permitido viver nos templos." + C: "assustar os inimigos, em particular os espanh\xF3is." + D: "legitimar sua realeza, j\xE1 que seu pai n\xE3o era da realeza." + input_correct_responses: + - D + input_question: 'O grande rei maia Pacal construiu templos na cidade de Palenque + para:' + - input_choice_list: + A: "um centro da civiliza\xE7\xE3o do Mississippi com condi\xE7\xF5es semelhantes\ + \ \xE0s da ascens\xE3o dos primeiros estados." + B: "as limita\xE7\xF5es de autoridade em uma sociedade nativa americana de forrageadores\ + \ igualit\xE1rios." + C: uma chefia simples ou talvez uma chefia complexa evoluiu por volta de 1500 + DC. + D: "um centro da civiliza\xE7\xE3o do Mississippi com condi\xE7\xF5es semelhantes\ + \ \xE0s sociedades da costa noroeste da Am\xE9rica do Norte." + input_correct_responses: + - A + input_question: "De acordo com Timothy Pauketat, as evid\xEAncias da estratifica\xE7\ + \xE3o social e do poder pol\xEDtico em Cahokia sugerem:" + - input_choice_list: + A: "algum tipo de cataclismo, como um terremoto, vulc\xE3o ou tsunami." + B: "degrada\xE7\xE3o ecol\xF3gica resultante de t\xE9cnicas agr\xEDcolas de\ + \ corte e queima." + C: "guerras intermin\xE1veis entre cidades-estado maias vizinhas." + D: "pr\xE1ticas de cruzamento que levaram a um aumento acentuado nas doen\xE7\ + as cong\xEAnitas." + input_correct_responses: + - B + input_question: "Os investigadores acreditam agora que o decl\xEDnio dos maias\ + \ foi causado principalmente por:" + - input_choice_list: + A: "uma grande diversidade de esp\xE9cies, ou uma \xFAnica esp\xE9cie que exibia\ + \ muita diversidade." + B: "muito pouca diversidade de esp\xE9cies durante este per\xEDodo e muito poucos\ + \ homin\xEDdeos." + C: "diminui\xE7\xE3o da diversidade de esp\xE9cies devido a uma era glacial\ + \ prolongada seguida por uma seca severa." + D: "diminui\xE7\xE3o da diversidade de esp\xE9cies, mas aumento do n\xFAmero\ + \ de martelos e lascas, indicando fabrica\xE7\xE3o de ferramentas de pedra." + input_correct_responses: + - A + input_question: "Pesquisas recentes sobre esp\xE9cies de homin\xEDdeos datadas\ + \ do Plioceno M\xE9dio indicam que houve (em 2020):" + - input_choice_list: + A: abaixo de 650 cc + B: cerca de 800 cc + C: pouco menos de 1000 cc + D: 1200 cc + input_correct_responses: + - C + input_question: "Qual \xE9 a capacidade craniana m\xE9dia aproximada do Homo erectus?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_prehistory +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_prehistory +task_alias: prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b44b287cef712c80fee9c2a85d78e5bdc3dcc3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_accounting.yaml @@ -0,0 +1,74 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: US$ 70.000 + B: US$ 75.000 + C: US$ 80.000 + D: '100.000' + input_correct_responses: + - D + input_question: "Box, uma organiza\xE7\xE3o n\xE3o governamental sem fins lucrativos\ + \ teve as seguintes transa\xE7\xF5es durante o ano: Receitas da venda de investimentos\ + \ $ 80.000 Compra de bens im\xF3veis e equipamentos $ 10.000 Receitas de d\xED\ + vidas de longo prazo $ 100.000 Perda na venda de investimentos $ 5.000 Qual\ + \ valor deve ser relatado como l\xEDquido caixa fornecido pelas atividades de\ + \ financiamento na demonstra\xE7\xE3o dos fluxos de caixa da Box?" + - input_choice_list: + A: US$ 13.000 + B: US$ 600 + C: US$ 15.000 + D: US$ 28.000 + input_correct_responses: + - A + input_question: "Cem anos atr\xE1s, sua tatarav\xF3 investiu US$ 100 com juros\ + \ de 5% ao ano. Quanto vale o investimento hoje?" + - input_choice_list: + A: US$ 0 + B: US$ 500 + C: US$ 1.650 + D: US$ 16.500 + input_correct_responses: + - A + input_question: "Krete \xE9 um contribuinte solteiro que recebe rendimentos exclusivamente\ + \ de sal\xE1rios. At\xE9 31 de dezembro do ano 1, o empregador de Krete reteve\ + \ US$ 16.000 em impostos federais sobre a renda e Krete n\xE3o fez nenhum pagamento\ + \ estimado de impostos. Em 15 de abril do ano 2, Krete apresentou oportunamente\ + \ um pedido de prorroga\xE7\xE3o para apresentar sua declara\xE7\xE3o de imposto\ + \ de renda individual e pagou US$ 300 de impostos adicionais. A obriga\xE7\xE3\ + o fiscal do ano 1 de Krete era de $ 16.500 quando ela apresentou sua declara\xE7\ + \xE3o oportunamente em 30 de abril do ano 2 e pagou o saldo restante da obriga\xE7\ + \xE3o fiscal. Qual valor estaria sujeito \xE0 penalidade por pagamento a menor\ + \ de impostos estimados?" + - input_choice_list: + A: US$ 5.000 + B: US$ 13.500 + C: US$ 16.000 + D: US$ 20.000 + input_correct_responses: + - B + input_question: "Em 1\xBA de janeiro do ano 1, a Alpha Co. assinou um contrato\ + \ de manuten\xE7\xE3o anual com um fornecedor de software por US$ 15.000 e o\ + \ per\xEDodo de manuten\xE7\xE3o come\xE7a em 1\xBA de mar\xE7o do ano 2. A\ + \ Alpha tamb\xE9m incorreu em US$ 5.000 em custos em 1\xBA de janeiro do ano\ + \ 1, relacionados \xE0 modifica\xE7\xE3o do software. solicita\xE7\xF5es que\ + \ aumentar\xE3o a funcionalidade do software. A Alpha deprecia e amortiza seus\ + \ ativos de computadores e software ao longo de cinco anos usando o m\xE9todo\ + \ linear. Qual \xE9 o valor da despesa total que Alpha deve reconhecer relacionada\ + \ ao contrato de manuten\xE7\xE3o e \xE0s modifica\xE7\xF5es de software para\ + \ o ano encerrado em 31 de dezembro, ano 1?" + - input_choice_list: + A: "Avalia\xE7\xE3o e aloca\xE7\xE3o" + B: Completude + C: Direitos e deveres + D: "Apresenta\xE7\xE3o e divulga\xE7\xE3o" + input_correct_responses: + - B + input_question: "Um auditor rastreia os n\xFAmeros de s\xE9rie do equipamento\ + \ at\xE9 o livro auxiliar de um n\xE3o emissor. Qual das seguintes afirma\xE7\ + \xF5es de gest\xE3o \xE9 apoiada por este teste?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_accounting +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_professional_accounting +task_alias: professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfbafa022035bb42e97d518148b04e6c9c42b677 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_law.yaml @@ -0,0 +1,129 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "o estatuto \xE9 nulo por imprecis\xE3o sob a cl\xE1usula do devido processo\ + \ da D\xE9cima Quarta Emenda." + B: "o estatuto \xE9 inv\xE1lido porque viola a liberdade de express\xE3o do\ + \ peticion\xE1rio sob a Primeira Emenda." + C: "o estatuto \xE9 uma abreviatura da liberdade de express\xE3o ao abrigo da\ + \ Primeira Emenda porque meios menos restritivos est\xE3o dispon\xEDveis para\ + \ atingir o mesmo objectivo." + D: "o estatuto \xE9 muito amplo e, conseq\xFCentemente, inv\xE1lido sob a Primeira\ + \ e a D\xE9cima Quarta Emendas." + input_correct_responses: + - D + input_question: "Uma legislatura estadual promulgou recentemente uma lei que considera\ + \ contraven\xE7\xE3o xingar, insultar ou usar linguagem obscena ou injuriosa\ + \ em rela\xE7\xE3o a um policial no desempenho de suas fun\xE7\xF5es ou em refer\xEA\ + ncia a ele. Um estudante de uma universidade estadual organizou uma manifesta\xE7\ + \xE3o no campus para protestar contra a guerra. A manifesta\xE7\xE3o contou\ + \ com a presen\xE7a de um grupo de 50 estudantes que gritaram mensagens anti-guerra\ + \ aos carros que passavam. Para mostrar seu desprezo pelos Estados Unidos, o\ + \ estudante costurou a bandeira americana na parte traseira da cal\xE7a jeans.\ + \ Quando um policial viu a bandeira pregada na cal\xE7a jeans do estudante,\ + \ ele se aproximou e disse-lhe para retirar a bandeira ou ele seria preso. O\ + \ estudante ficou furioso e gritou para o policial: \u201CEscute, seu bastardo,\ + \ usarei esse trapo onde quiser\u201D. O estudante foi posteriormente preso\ + \ e acusado de violar a lei estadual. O estudante posteriormente entra com uma\ + \ a\xE7\xE3o na Justi\xE7a estadual questionando a constitucionalidade da lei.\ + \ O argumento constitucional mais forte para o estudante \xE9 que" + - input_choice_list: + A: "A cl\xE1usula comercial." + B: "A cl\xE1usula de prote\xE7\xE3o igual da D\xE9cima Quarta Emenda." + C: "A cl\xE1usula de privil\xE9gios e imunidades do Artigo IV, Se\xE7\xE3o 2." + D: "A cl\xE1usula contratual." + input_correct_responses: + - A + input_question: "Um estado promulgou recentemente uma lei que pro\xEDbe a elimina\xE7\ + \xE3o de quaisquer res\xEDduos nucleares dentro do estado. Esta lei n\xE3o contraria\ + \ nem entra em conflito com quaisquer estatutos federais. Um homem dirige uma\ + \ empresa no estado que se dedica \xE0 elimina\xE7\xE3o de res\xEDduos nucleares.\ + \ Ap\xF3s a aprova\xE7\xE3o do estatuto estatal, o homem, ainda sem conhecimento\ + \ da nova lei, celebrou contratos com muitas empresas de fora do estado para\ + \ eliminar os seus res\xEDduos nucleares no estado. Por conta dessa nova lei,\ + \ por\xE9m, o homem n\xE3o poder\xE1 cumprir esses contratos. Suponha que o\ + \ homem tenha legitimidade para contestar esta lei estadual. Qual das alternativas\ + \ a seguir apresenta seus fundamentos constitucionais mais fortes para desafiar\ + \ a lei estadual que pro\xEDbe o descarte de res\xEDduos nucleares dentro do\ + \ estado?" + - input_choice_list: + A: "Fatos indiscut\xEDveis." + B: "Fatos que foram afirmados por organiza\xE7\xF5es pol\xEDticas individuais." + C: Fatos reconhecidos como verdadeiros pelo conhecimento comum. + D: "Fatos pass\xEDveis de verifica\xE7\xE3o cient\xEDfica." + input_correct_responses: + - B + input_question: "O juiz tomou conhecimento judicial de alguns factos no in\xED\ + cio do julgamento. Qual das alternativas a seguir n\xE3o \xE9 um tipo de fato\ + \ apropriado para notifica\xE7\xE3o judicial?" + - input_choice_list: + A: "conceder al\xEDvio, porque a cerca violava a restri\xE7\xE3o de servid\xE3\ + o." + B: "conceder al\xEDvio, porque a invas\xE3o da cerca violou a restri\xE7\xE3\ + o do plano original." + C: "negar a tutela, porque o professor n\xE3o conseguiu fazer cumprir a restri\xE7\ + \xE3o ao aposentado." + D: "negar al\xEDvio, porque a cerca n\xE3o seria interpretada como \u201Cuma\ + \ estrutura\u201D dentro dos termos da restri\xE7\xE3o." + input_correct_responses: + - B + input_question: "Em 1\xBA de outubro de 1980, um incorporador, propriet\xE1rio\ + \ de v\xE1rias centenas de acres em um condado rural, elaborou um plano geral\ + \ de desenvolvimento para a \xE1rea. O plano devidamente registrado imp\xF4\ + s limita\xE7\xF5es e restri\xE7\xF5es elaboradas ao terreno do plano, que deveria\ + \ ser desenvolvido como um bairro residencial. As restri\xE7\xF5es deveriam\ + \ se estender a todas as pessoas que adquirissem qualquer um dos lotes e aos\ + \ seus herdeiros, cession\xE1rios e arrendat\xE1rios. Foi ainda previsto que\ + \ todos os propriet\xE1rios subsequentes seriam cobrados com a devida notifica\xE7\ + \xE3o das restri\xE7\xF5es. Entre essas restri\xE7\xF5es do plano geral estavam\ + \ as seguintes:(22) \xC9 criado um direito de franquia em uma faixa de terreno\ + \ de 10 p\xE9s de largura ao longo dos fundos de cada lote para uso de concession\xE1\ + rias de servi\xE7os p\xFAblicos com direito de entrada e sa\xEDda. \xA7 23\xBA\ + \ Nenhuma casa ou estrutura de qualquer esp\xE9cie dever\xE1 ser constru\xED\ + da na referida faixa de terreno que atravessa os referidos quarteir\xF5es. Em\ + \ 2000, um aposentado comprou um dos lotes, construiu uma casa e ergueu uma\ + \ cerca nos fundos de sua propriedade, dentro da \xE1rea restrita. Em 2004,\ + \ uma professora comprou um terreno adjacente \xE0 propriedade do aposentado\ + \ e construiu uma nova casa. Dois anos depois, uma bibliotec\xE1ria comprou\ + \ o terreno cont\xEDguo \xE0 propriedade da professora. Cada uma das tr\xEA\ + s escrituras dessas propriedades continha refer\xEAncias ao livro de escrituras\ + \ onde estava registado o plano geral. Em 2008, o bibliotec\xE1rio iniciou a\ + \ constru\xE7\xE3o de uma cerca de 2,1 metros de altura ao longo da linha que\ + \ divide seu lote com o do professor, e ao longo do centro da \xE1rea sujeita\ + \ ao direito de franquia. Embora o professor tenha se oposto \xE0 sua constru\xE7\ + \xE3o, a cerca foi conclu\xEDda. Se o professor solicitar uma liminar para obrigar\ + \ a remo\xE7\xE3o da cerca do bibliotec\xE1rio, o tribunal provavelmente ir\xE1" + - input_choice_list: + A: "A promessa do pai e a confian\xE7a do credor nela, se provadas, deram origem\ + \ a uma reclama\xE7\xE3o v\xE1lida do credor contra o pai com base na doutrina\ + \ da preclus\xE3o promiss\xF3ria." + B: "Como era previs\xEDvel que a promessa do pai induziria o credor a abster-se\ + \ de tomar qualquer ac\xE7\xE3o contra o filho, tal toler\xE2ncia era, por\ + \ uma quest\xE3o de lei, uma contrapartida negociada pela promessa do pai." + C: "Os cinco pagamentos do pai ao credor, totalizando US$ 2.500, manifestaram\ + \ uma inten\xE7\xE3o s\xE9ria por parte do pai de estar contratualmente vinculado,\ + \ e tal manifesta\xE7\xE3o \xE9 geralmente reconhecida como um substituto\ + \ eficaz da contrapresta\xE7\xE3o." + D: "Ao assumir a obriga\xE7\xE3o de d\xEDvida antecedente que o filho devia\ + \ ao credor, o pai tornou-se fiador cuja promessa ao credor era execut\xF3\ + ria, desde que fosse por escrito e amparada por contrapresta\xE7\xE3o adequada." + input_correct_responses: + - A + input_question: "Um filho devia a um credor US$ 5.000. O pai do filho contactou\ + \ o credor e disse-lhe que queria pagar a d\xEDvida do filho. O pai assinou\ + \ um documento que afirmava que pagaria a d\xEDvida do filho a uma taxa de US$\ + \ 500 por m\xEAs durante 10 meses. O credor n\xE3o se comprometeu por escrito\ + \ ou verbalmente a deixar de processar o filho para cobrar a d\xEDvida de $5.000,\ + \ e o pai n\xE3o fez nenhum pedido oral ou escrito de tal toler\xE2ncia. Durante\ + \ os cinco meses seguintes, o pai fez e o credor aceitou os pagamentos mensais\ + \ de US$ 500 conforme acordado. Durante esse per\xEDodo, o credor, de facto,\ + \ absteve-se de intentar qualquer ac\xE7\xE3o judicial contra o filho. No entanto,\ + \ o pai informou ent\xE3o ao credor que n\xE3o faria mais pagamentos da d\xED\ + vida. Qual das alternativas a seguir \xE9 o argumento mais persuasivo de que\ + \ o pai \xE9 respons\xE1vel perante o credor nos termos do acordo?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_law +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_professional_law +task_alias: professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e371d0cb9dde8224c15afb5b8a3b72f4dd8001cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_medicine.yaml @@ -0,0 +1,94 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: Labetalol + B: "Uma dose de ataque de cloreto de pot\xE1ssio" + C: Nifedipina + D: Fenoxibenzamina + input_correct_responses: + - D + input_question: "Um homem de 42 anos vem ao consult\xF3rio para avalia\xE7\xE3\ + o pr\xE9-operat\xF3ria antes de ser submetido \xE0 adrenalectomia programada\ + \ para daqui a 2 semanas. H\xE1 um m\xEAs, ele recebeu atendimento no pronto-socorro\ + \ devido a dores no flanco direito ap\xF3s uma colis\xE3o de ve\xEDculo motorizado.\ + \ Naquela \xE9poca, a press\xE3o arterial era de 160/100 mm Hg e a tomografia\ + \ computadorizada de abdome mostrou uma massa adrenal esquerda incidental de\ + \ 10 cm. Os resultados dos estudos laboratoriais, incluindo hemograma completo,\ + \ concentra\xE7\xF5es s\xE9ricas de eletr\xF3litos e testes de fun\xE7\xE3o\ + \ hep\xE1tica, estavam dentro dos intervalos de refer\xEAncia. O paciente, por\ + \ outro lado, era saud\xE1vel e nunca foi informado de que tinha press\xE3o\ + \ arterial elevada. Ele n\xE3o toma medicamentos. Uma visita de acompanhamento\ + \ no consult\xF3rio h\xE1 2 semanas revelou concentra\xE7\xF5es elevadas de\ + \ normetanefrina e metanefrina urin\xE1ria e aldosterona plasm\xE1tica. O paciente\ + \ foi encaminhado ao cirurgi\xE3o, que recomendou a adrenalectomia. Hoje, os\ + \ sinais vitais s\xE3o temperatura 36,6\xB0C (97,9\xB0F), pulso 100/min, respira\xE7\ + \xE3o 14/min e press\xE3o arterial 170/95 mm Hg. O exame f\xEDsico n\xE3o revela\ + \ achados significativos. A prepara\xE7\xE3o pr\xE9-operat\xF3ria inicial deve\ + \ incluir tratamento com qual dos seguintes?" + - input_choice_list: + A: "tor\xE7\xE3o sacral esquerda-esquerda" + B: "tor\xE7\xE3o sacral esquerda-direita" + C: "flex\xE3o sacral unilateral direita" + D: "tor\xE7\xE3o sacral direita sobre direita" + input_correct_responses: + - D + input_question: "Um homem de 36 anos chega ao consult\xF3rio com hist\xF3ria de\ + \ dor lombar h\xE1 tr\xEAs semanas. Ele nega qualquer trauma recente, mas diz\ + \ que entra e sai da caminhonete in\xFAmeras vezes ao dia para trabalhar. O\ + \ exame do paciente em dec\xFAbito ventral revela um sulco sacral profundo \xE0\ + \ esquerda, um \xE2ngulo lateral p\xF3stero-inferior \xE0 direita e uma jun\xE7\ + \xE3o lombossacral que salta livremente sob compress\xE3o. O diagn\xF3stico\ + \ mais prov\xE1vel \xE9" + - input_choice_list: + A: Dopamina + B: Glutamato + C: Norepinefrina + D: Serotonina + input_correct_responses: + - D + input_question: "Uma mulher de 32 anos, previamente saud\xE1vel, procura o m\xE9\ + dico 8 meses depois que seu marido morreu em um acidente de carro. Desde ent\xE3\ + o, ela apresenta diminui\xE7\xE3o do apetite e dificuldade em adormecer. Ela\ + \ afirma que muitas vezes fica triste e chora com frequ\xEAncia. Ela verificou\ + \ novamente a fechadura da porta cinco vezes antes de sair de casa e precisa\ + \ contar exatamente cinco peda\xE7os de papel higi\xEAnico antes de us\xE1-lo.\ + \ Ela diz que sempre foi perfeccionista, mas esses impulsos e rituais s\xE3\ + o novos. A farmacoterapia deve ser direcionada para qual dos seguintes neurotransmissores?" + - input_choice_list: + A: "Rinite al\xE9rgica" + B: "V\xEDrus de Epstein Barr" + C: Mycoplasma pneumoniae + D: "Rinov\xEDrus" + input_correct_responses: + - D + input_question: "Um homem de 44 anos chega ao consult\xF3rio devido a uma hist\xF3\ + ria de 3 dias de dor de garganta, tosse improdutiva, coriza e dor de cabe\xE7\ + a frontal. Ele diz que a dor de cabe\xE7a piora pela manh\xE3 e o ibuprofeno\ + \ proporciona algum al\xEDvio. Ele n\xE3o teve falta de ar. A hist\xF3ria m\xE9\ + dica n\xE3o \xE9 digna de nota. Ele n\xE3o toma outros medicamentos al\xE9m\ + \ do ibuprofeno para dor. Os sinais vitais s\xE3o temperatura 37,4\xB0C (99,4\xB0\ + F), pulso 88/min, respira\xE7\xE3o 18/min e press\xE3o arterial 120/84 mm Hg.\ + \ O exame das narinas mostra membranas mucosas eritematosas. O exame da garganta\ + \ mostra eritema e hiperplasia linfoide folicular na orofaringe posterior. N\xE3\ + o h\xE1 adenopatia cervical palp\xE1vel. Os pulm\xF5es est\xE3o limpos \xE0\ + \ ausculta. Qual das alternativas a seguir \xE9 a causa mais prov\xE1vel dos\ + \ sintomas deste paciente?" + - input_choice_list: + A: escaleno anterior + B: grande dorsal + C: peitoral menor + D: quadrado lombar + input_correct_responses: + - C + input_question: "Um maratonista de 22 anos chega ao consult\xF3rio com queixa\ + \ de dor na costela do lado direito quando corre longas dist\xE2ncias. O exame\ + \ f\xEDsico revela achados card\xEDacos e pulmonares normais e uma disfun\xE7\ + \xE3o expirat\xF3ria nas costelas 4-5 \xE0 direita. Qual dos seguintes m\xFA\ + sculos ou grupos musculares ser\xE1 mais \xFAtil na corre\xE7\xE3o desta disfun\xE7\ + \xE3o utilizando um m\xE9todo direto?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_medicine +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_professional_medicine +task_alias: professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08b65fde535089489d3026e4c52291e009738a5e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_professional_psychology.yaml @@ -0,0 +1,65 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "informe ao seu cliente os pr\xF3s e os contras de cada programa que voc\xEA\ + \ conhece, exceto aquele em que voc\xEA est\xE1 inscrito" + B: "recomende ao seu cliente o programa no qual voc\xEA est\xE1 inscrito e explique\ + \ o b\xF4nus de US$ 50 que voc\xEA receber\xE1" + C: "recomende ao seu cliente o programa no qual voc\xEA est\xE1 inscrito e ofere\xE7\ + a o b\xF4nus de $ 50 creditado na conta do seu cliente no programa" + D: "diga ao seu cliente os pr\xF3s e os contras de cada programa que voc\xEA\ + \ conhece, mas n\xE3o reivindique o b\xF4nus de $ 50 se o seu cliente se inscrever\ + \ no seu programa" + input_correct_responses: + - D + input_question: "Um de seus clientes de terapia pede seu conselho sobre um bom\ + \ programa de redu\xE7\xE3o de peso. Voc\xEA investigou os programas da comunidade\ + \ e est\xE1 inscrito naquele que considera o melhor. Este programa oferece um\ + \ b\xF4nus de US$ 50 aos seus clientes para cada nova pessoa que eles trazem\ + \ para o programa. Nestas circunst\xE2ncias, a sua resposta mais adequada seria" + - input_choice_list: + A: "menos sens\xEDvel a pontua\xE7\xF5es extremas do que a m\xE9dia" + B: "mais \xFAtil para distribui\xE7\xF5es distorcidas" + C: "sens\xEDvel a valores extremos e distribui\xE7\xF5es altamente distorcidas" + D: "o n\xFAmero que ocorre com mais frequ\xEAncia" + input_correct_responses: + - D + input_question: "Existem tr\xEAs formas de medir a Tend\xEAncia Central: a M\xE9\ + dia, a Mediana e a Moda. Pelo seu conhecimento sobre eles, qual \xE9 o modo?" + - input_choice_list: + A: individualismo. + B: "individualismo e dist\xE2ncia do poder." + C: "dist\xE2ncia do poder e masculinidade." + D: "evita\xE7\xE3o da incerteza." + input_correct_responses: + - A + input_question: "Em termos das cinco dimens\xF5es culturais de Hofstede (1980),\ + \ os Estados Unidos pontuam no topo da escala em:" + - input_choice_list: + A: "\xE9 uma fantasia que distrai o cliente da realidade." + B: "representa \u201Csentimentos confusos\u201D em rela\xE7\xE3o ao terapeuta." + C: ""\xE9 uma forma de ""atua\xE7\xE3o."""" + D: reflete o inconsciente pessoal e coletivo do cliente. + input_correct_responses: + - D + input_question: "Carl Jung acreditava que a transfer\xEAncia de um cliente:" + - input_choice_list: + A: "n\xE3o est\xE3o correlacionados entre si, mas est\xE3o moderadamente correlacionados\ + \ com o crit\xE9rio" + B: "t\xEAm correla\xE7\xF5es baixas entre si e correla\xE7\xF5es baixas com\ + \ o crit\xE9rio" + C: "s\xE3o altamente intercorrelacionados entre si e moderadamente correlacionados\ + \ com o crit\xE9rio" + D: "t\xEAm correla\xE7\xF5es baixas com o crit\xE9rio, mas est\xE3o moderadamente\ + \ correlacionados entre si" + input_correct_responses: + - A + input_question: "Na constru\xE7\xE3o de uma equa\xE7\xE3o de regress\xE3o m\xFA\ + ltipla para fins de previs\xE3o, a combina\xE7\xE3o \xF3tima de medidas \xE9\ + \ aquela em que os preditores" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_psychology +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_professional_psychology +task_alias: professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72e40fade0decda1881a23ace24c91b39f943006 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_public_relations.yaml @@ -0,0 +1,61 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Forne\xE7a ao rep\xF3rter outras informa\xE7\xF5es que ele tenha certeza\ + \ de que est\xE3o corretas." + B: "Diga que a informa\xE7\xE3o \xE9 \u201Coff the record\u201D e ser\xE1 divulgada\ + \ posteriormente." + C: "Diga 'n\xE3o sei' e prometa fornecer as informa\xE7\xF5es mais tarde." + D: "Diga 'sem coment\xE1rios' em vez de parecer desinformado." + input_correct_responses: + - C + input_question: "O que um profissional de rela\xE7\xF5es p\xFAblicas na m\xED\ + dia deve fazer se n\xE3o souber a resposta \xE0 pergunta de um rep\xF3rter?" + - input_choice_list: + A: "Compre nomes de dom\xEDnio que possam ser usados por grupos de oposi\xE7\ + \xE3o." + B: "Poste coment\xE1rios an\xF4nimos em blogs para combater essas informa\xE7\ + \xF5es." + C: "Prepare um comunicado \xE0 imprensa que desacredite as informa\xE7\xF5es\ + \ imprecisas." + D: "Fa\xE7a altera\xE7\xF5es nas pol\xEDticas para atender \xE0s reclama\xE7\ + \xF5es destacadas nesses sites." + input_correct_responses: + - D + input_question: "Na gest\xE3o de problemas, qual \xE9 a abordagem mais proativa\ + \ para lidar com informa\xE7\xF5es negativas ou enganosas publicadas on-line\ + \ sobre a sua organiza\xE7\xE3o?" + - input_choice_list: + A: "Houve uma resposta coordenada da m\xEDdia." + B: Mensagens consistentes foram comunicadas. + C: "As cr\xEDticas foram tomadas como ataques \xE0 Igreja Cat\xF3lica." + D: A credibilidade do Vaticano foi mantida. + input_correct_responses: + - C + input_question: "Qual destas afirma\xE7\xF5es \xE9 verdadeira em rela\xE7\xE3\ + o ao Vaticano em 2010, na altura das acusa\xE7\xF5es de encobrimento de abusos\ + \ infantis?" + - input_choice_list: + A: Definindo o programa + B: Planejando o programa + C: Agir e implementar ideias + D: "Avalia\xE7\xE3o do programa" + input_correct_responses: + - A + input_question: "Em que fase do processo de planeamento seria realizada uma an\xE1\ + lise da situa\xE7\xE3o?" + - input_choice_list: + A: Paz verde + B: A ONU + C: Oxfam + D: Fundo Mundial para a Vida Selvagem + input_correct_responses: + - D + input_question: "A Hora do Planeta foi uma campanha lan\xE7ada por qual organiza\xE7\ + \xE3o?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_public_relations +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_public_relations +task_alias: public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..007f761ec8a855e96eeba19fc07859910cf4a2b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_security_studies.yaml @@ -0,0 +1,113 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Compet\xEAncia \xE9 outro termo para diplomacia coercitiva, mas abrange\ + \ um conjunto mais restrito de crit\xE9rios; a compet\xEAncia abrange as amea\xE7\ + as destinadas a iniciar a a\xE7\xE3o do advers\xE1rio. Uma amea\xE7a de coagir\ + \ um Estado a ceder parte do seu territ\xF3rio contaria como diplomacia coercitiva,\ + \ desde que essa amea\xE7a iniciasse proativamente uma a\xE7\xE3o antes de\ + \ ser adotada uma diplomacia reativa." + B: "A diplomacia coercitiva constitui as amea\xE7as de for\xE7a limitada para\ + \ induzir o incentivo do advers\xE1rio a cumprir as exig\xEAncias do coagente.\ + \ \xC9 uma estrat\xE9gia de influ\xEAncia que visa obter conformidade: o uso\ + \ da for\xE7a para derrotar primeiro um advers\xE1rio n\xE3o conta. Deixa\ + \ um elemento de escolha com a meta a cumprir ou a continuar." + C: "A for\xE7a militar, ou a amea\xE7a da for\xE7a militar, utiliza o medo para\ + \ atingir objetivos estrat\xE9gicos. A diplomacia coercitiva diferencia-se\ + \ desta abordagem porque n\xE3o utiliza o medo como ferramenta para coagir\ + \ um advers\xE1rio." + D: "A diplomacia coercitiva \xE9 utilizada para usar a for\xE7a, mas para limitar\ + \ os seus efeitos na comunidade internacional. A diplomacia coercitiva \xE9\ + \ uma estrat\xE9gia agressiva que visa obter conformidade atrav\xE9s da derrota.\ + \ N\xE3o deixa um elemento de escolha ao alvo, sendo o alvo for\xE7ado a obedecer\ + \ ou a envolver-se em conflito. Procura controlar impondo o cumprimento, eliminando\ + \ qualquer oportunidade de negocia\xE7\xE3o ou concess\xE3o." + input_correct_responses: + - B + input_question: "O que distingue a diplomacia coercitiva da for\xE7a militar?" + - input_choice_list: + A: "As crian\xE7as-soldados s\xE3o v\xEDtimas de combates que necessitam de\ + \ reeduca\xE7\xE3o e reabilita\xE7\xE3o." + B: "As crian\xE7as e as suas m\xE3es n\xE3o s\xE3o sujeitos activos na guerra\ + \ e s\xE3o melhor consideradas como sujeitos da esfera privada." + C: "As crian\xE7as s\xE3o, na maioria das vezes, espectadores inocentes na guerra\ + \ e s\xE3o melhor utilizadas como s\xEDmbolos de paz." + D: "As crian\xE7as t\xEAm uma subjetividade pol\xEDtica que \xE9 ignorada quando\ + \ s\xE3o consideradas v\xEDtimas passivas da guerra." + input_correct_responses: + - D + input_question: "Qual das alternativas a seguir \xE9 a melhor perspectiva para\ + \ investigar o papel das crian\xE7as-soldados?" + - input_choice_list: + A: "Como uma amea\xE7a existencial que requer uma a\xE7\xE3o imediata e extraordin\xE1\ + ria, representando uma amea\xE7a \xE0 sobreviv\xEAncia do Estado ou \xE0 seguran\xE7\ + a social." + B: "Como exigindo uma a\xE7\xE3o imediata e extraordin\xE1ria por parte do Estado,\ + \ amea\xE7ando a sobreviv\xEAncia de um objeto de refer\xEAncia e, portanto,\ + \ justificando o uso de medidas normalmente n\xE3o empregadas na esfera pol\xED\ + tica." + C: "Como amea\xE7a urgente \xE0 sobreviv\xEAncia do objeto referente, t\xE3\ + o grave que legitima o emprego de a\xE7\xE3o extraordin\xE1ria em resposta." + D: "Como amea\xE7a urgente \xE0 sobreviv\xEAncia do p\xFAblico que exige medidas\ + \ extraordin\xE1rias ou emergenciais." + input_correct_responses: + - C + input_question: "Para se tornar securitizado, uma amea\xE7a deve ser apresentada\ + \ de qual destas formas?" + - input_choice_list: + A: "Existem divis\xF5es t\xE3o amplas no quadro da seguran\xE7a humana relativamente\ + \ \xE0 natureza das amea\xE7as e dos objectos de refer\xEAncia que n\xE3o\ + \ \xE9 poss\xEDvel estabelecer compara\xE7\xF5es amplamente aplic\xE1veis\ + \ entre as abordagens centradas no Estado e a seguran\xE7a humana." + B: "Ao adoptar o quadro da seguran\xE7a humana, as limita\xE7\xF5es da abordagem\ + \ realista centrada no Estado tornam-se evidentes. Embora a seguran\xE7a humana\ + \ defina o objecto de refer\xEAncia como a pessoa ou popula\xE7\xE3o, as abordagens\ + \ centradas no Estado d\xE3o prioridade \xE0 seguran\xE7a do Estado, despriorizando\ + \ a procura da seguran\xE7a humana." + C: "A abordagem da seguran\xE7a centrada no Estado \xE9 uma fac\xE7\xE3o da\ + \ seguran\xE7a humana, geralmente definida dentro da ampla escola de seguran\xE7\ + a humana. Por ser centrada no Estado, esta abordagem prioriza o indiv\xED\ + duo como objeto de refer\xEAncia nos estudos de seguran\xE7a." + D: "Tanto a abordagem da seguran\xE7a centrada no Estado como a abordagem centrada\ + \ no ser humano s\xE3o mutuamente exclusivas e oferecem um quadro anal\xED\ + tico suficiente para compreender o sistema de seguran\xE7a internacional.\ + \ \xC9, portanto, papel dos analistas de seguran\xE7a determinar quais destes\ + \ conceitos substanciais est\xE3o corretos e quais devem ser descartados." + input_correct_responses: + - B + input_question: "Como podemos descrever melhor a rela\xE7\xE3o entre a abordagem\ + \ centrada no Estado e o conceito de seguran\xE7a humana?" + - input_choice_list: + A: "A concorr\xEAncia entre na\xE7\xF5es maiores resultou em alguns pa\xEDses\ + \ apoiarem activamente grupos terroristas para minar a for\xE7a de estados\ + \ rivais. As redes terroristas s\xE3o clubes de clientelismo alargados mantidos\ + \ e pagos pelos seus estados doadores e s\xE3o conceptualizados como sendo\ + \ semelhantes a actores estatais, a serem combatidos com recurso \xE0 for\xE7\ + a militar." + B: "A globaliza\xE7\xE3o permitiu a internacionaliza\xE7\xE3o das actividades\ + \ terroristas, abrindo o seu espa\xE7o operacional, embora a coordena\xE7\xE3\ + o ainda seja gerida a partir de uma base geogr\xE1fica. Isto sugere que os\ + \ grupos terroristas est\xE3o estruturados a n\xEDvel nacional, o que significa\ + \ que o terrorismo n\xE3o pode ser considerado em termos de uma guerra a ser\ + \ derrotada militarmente sem ter implica\xE7\xF5es graves para a popula\xE7\ + \xE3o ind\xEDgena." + C: "O terrorismo pode ser visto como um problema a ser resolvido por meios militares\ + \ (guerra ao terrorismo), por t\xE9cnicas policiais normais (terrorismo como\ + \ crime), ou como um problema m\xE9dico com causas e sintomas subjacentes\ + \ (terrorismo como doen\xE7a)." + D: "O terrorismo \xE9 visto como um problema criminal. A criminaliza\xE7\xE3\ + o do terrorismo tem duas implica\xE7\xF5es importantes. Em primeiro lugar,\ + \ sugere que o terrorismo pode ser erradicado - os terroristas podem ser capturados\ + \ e levados a julgamento atrav\xE9s de processos judiciais normais, eliminando\ + \ assim a amea\xE7a da sociedade - e, em segundo lugar, sugere que s\xE3o\ + \ aplic\xE1veis t\xE9cnicas preventivas do crime para impedir o seu desenvolvimento." + input_correct_responses: + - C + input_question: "Quais s\xE3o os quadros de an\xE1lise dentro dos quais o terrorismo\ + \ foi considerado (a partir de 2020)?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_security_studies +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_security_studies +task_alias: security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67e458861353b387c03e3b9655a707b82d7db324 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_sociology.yaml @@ -0,0 +1,62 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "cuidados de sa\xFAde e educa\xE7\xE3o gratuitos para todos" + B: "um sal\xE1rio m\xEDnimo" + C: pleno emprego + D: bem-estar universal + input_correct_responses: + - B + input_question: "Qual das seguintes op\xE7\xF5es o estado de bem-estar social\ + \ do p\xF3s-guerra de 1948 n\xE3o pretendia fornecer:" + - input_choice_list: + A: um passeio de feira + B: um circo + C: um teatro de fantoches + D: "um bal\xE9" + input_correct_responses: + - C + input_question: "O que Berger (1963) descreve como uma met\xE1fora para a realidade\ + \ social?" + - input_choice_list: + A: "a crescente burocracia do estado tornou a religi\xE3o apenas uma parte marginal\ + \ de nossas vidas" + B: "apesar do enfraquecimento da autoridade tradicional, a nossa vida quotidiana\ + \ e o \u201Csenso comum\u201D continuam a ser moldados por cren\xE7as e valores\ + \ religiosos" + C: "a participa\xE7\xE3o religiosa no culto colectivo pode ter diminu\xEDdo,\ + \ mas as pessoas ainda praticam a sua f\xE9 em privado" + D: "as pessoas s\xE3o muito mais propensas a discutir suas cren\xE7as religiosas\ + \ em ambientes p\xFAblicos e informais" + input_correct_responses: + - B + input_question: "A mudan\xE7a da \u201Creligi\xE3o civil\u201D para a \u201Creligi\xE3\ + o comum\u201D significa que:" + - input_choice_list: + A: "a tend\xEAncia da classe trabalhadora de n\xE3o realizar seus pr\xF3prios\ + \ interesses" + B: "uma ideologia dominante que legitima o poder econ\xF3mico, pol\xEDtico e\ + \ cultural" + C: "uma forma de consci\xEAncia dupla baseada na ideologia e nas experi\xEA\ + ncias cotidianas" + D: uma forma de pagamento dada para topiaria pendente + input_correct_responses: + - B + input_question: 'O termo 'hegemonia' refere-se a:' + - input_choice_list: + A: "a maioria das greves passa despercebida pelos empregadores e pela m\xED\ + dia de massa" + B: "nem todos os conflitos laborais ser\xE3o comunicados pelo empregador" + C: "a defini\xE7\xE3o de greve exclui aquelas que envolvem menos de dez trabalhadores\ + \ ou duram menos de um dia" + D: "\xE9 dif\xEDcil comparar greves que foram medidas de maneiras diferentes" + input_correct_responses: + - A + input_question: "Qual das alternativas a seguir n\xE3o \xE9 um problema associado\ + \ \xE0s estat\xEDsticas oficiais sobre greves?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_sociology +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_sociology +task_alias: sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc8c87ab5398fc7d8d90c8e2b4397a035c9f0d33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_us_foreign_policy.yaml @@ -0,0 +1,59 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Prejudicou o apoio ao modelo americano de economia pol\xEDtica e capitalismo" + B: Isso criou raiva nos Estados Unidos por exagerarem a crise + C: "Aumentou o apoio \xE0 lideran\xE7a global americana sob o presidente Obama" + D: "Reduziu o uso global do d\xF3lar americano" + input_correct_responses: + - A + input_question: "Como a crise financeira de 2008 afetou a reputa\xE7\xE3o internacional\ + \ da Am\xE9rica?" + - input_choice_list: + A: "Globalizou a conten\xE7\xE3o." + B: "Militarizou a conten\xE7\xE3o." + C: "Apelava ao desenvolvimento da bomba de hidrog\xE9nio." + D: Tudo o que precede + input_correct_responses: + - D + input_question: "Como o NSC-68 mudou a estrat\xE9gia dos EUA?" + - input_choice_list: + A: "pol\xEDtica de terrorismo." + B: "pol\xEDtica econ\xF4mica." + C: "pol\xEDtica estrangeira." + D: "pol\xEDtica internacional." + input_correct_responses: + - C + input_question: "O dom\xEDnio das decis\xF5es pol\xEDticas relacionadas principalmente\ + \ com as rela\xE7\xF5es entre os Estados Unidos e o resto do mundo \xE9 conhecido\ + \ como" + - input_choice_list: + A: "Os realistas defensivos colocam maior \xEAnfase no papel das institui\xE7\ + \xF5es internacionais" + B: "Os realistas defensivos colocam menos \xEAnfase nos fatores geogr\xE1ficos" + C: "Os realistas ofensivos d\xE3o mais prioridade ao interesse nacional do que\ + \ os realistas defensivos." + D: "Os realistas defensivos acreditam que os estados s\xE3o maximizadores de\ + \ seguran\xE7a, enquanto os realistas ofensivos acreditam que os estados s\xE3\ + o maximizadores de poder" + input_correct_responses: + - D + input_question: "Como o Realismo Defensivo e o Realismo Ofensivo diferem na sua\ + \ explica\xE7\xE3o do comportamento do Estado?" + - input_choice_list: + A: "A globaliza\xE7\xE3o tornou homens como ele ricos demais" + B: "A globaliza\xE7\xE3o beneficiou apenas alguns estados americanos, como Nova\ + \ York" + C: "As elites liberais encorajaram a globaliza\xE7\xE3o, enquanto os \u201C\ + americanos comuns\u201D perderam empregos por causa dela" + D: "A globaliza\xE7\xE3o incentivou guerras comerciais prejudiciais" + input_correct_responses: + - C + input_question: "Como Donald Trump atacou a globaliza\xE7\xE3o na campanha de\ + \ 2016?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_us_foreign_policy +tag: mmlu_pt_llama_social_sciences_tasks +task: mmlu_pt_llama_us_foreign_policy +task_alias: us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f5b853e4f9716a71ffe8c99655efe3657368a20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_virology.yaml @@ -0,0 +1,52 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "Porque eles n\xE3o t\xEAm \xE1cido nucl\xE9ico" + B: "Eles exigem um v\xEDrus auxiliar" + C: "Replicar apenas em c\xE9lulas em divis\xE3o" + D: Pode integrar-se aos cromossomos do hospedeiro + input_correct_responses: + - A + input_question: "Por que os parvov\xEDrus s\xE3o um parasita altamente impactante?" + - input_choice_list: + A: "V\xEDrus fr\xE1geis frequentemente visualizados com RNA expelido de dentro" + B: "V\xEDrus alongados" + C: "V\xEDrus icosa\xE9dricos com envelope" + D: "V\xEDrus muito grandes" + input_correct_responses: + - A + input_question: "Qual das alternativas a seguir \xE9 uma caracter\xEDstica morfol\xF3\ + gica dos paramixov\xEDrus." + - input_choice_list: + A: "Cole\xE7\xE3o de dados" + B: "Estabelecimento de um reposit\xF3rio de amostras biol\xF3gicas" + C: Interesse do participante + D: "Administra\xE7\xE3o do question\xE1rio pela equipe" + input_correct_responses: + - B + input_question: "Um fator chave que facilitou a aplica\xE7\xE3o de estudos de\ + \ caso-controle aninhados do MACS foi:" + - input_choice_list: + A: "Mudan\xE7a de comportamento" + B: Cobertura abrangente + C: Uso eficaz da teoria comportamental + D: "Mudan\xE7a sustentada de comportamento" + input_correct_responses: + - D + input_question: "O objetivo mais importante de uma interven\xE7\xE3o comportamental\ + \ \xE9:" + - input_choice_list: + A: "Indiv\xEDduos soroprevalentes infectados pelo HIV" + B: Soronegativos + C: Soroconversores + D: Soronegativos de alto risco + input_correct_responses: + - C + input_question: "O tempo m\xE9dio de sobreviv\xEAncia \xE0 SIDA e \xE0 morte foi\ + \ estabelecido da seguinte forma:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_virology +tag: mmlu_pt_llama_other_tasks +task: mmlu_pt_llama_virology +task_alias: virology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65e29c36328001d8c340827312c60ca350f7868b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/mmlu_pt_world_religions.yaml @@ -0,0 +1,50 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "O arco-\xEDris" + B: "Circuncis\xE3o" + C: Um filho + D: Bar Mitzvah + input_correct_responses: + - B + input_question: "Qual \xE9 o sinal da alian\xE7a para os homens judeus?" + - input_choice_list: + A: O Dharma + B: A Sanga + C: O Buda + D: O Bodisatva + input_correct_responses: + - A + input_question: "Qual \xE9 a segunda joia do budismo?" + - input_choice_list: + A: Shang + B: Zhou + C: Han + D: Xia + input_correct_responses: + - B + input_question: "Em que dinastia foi desenvolvido o \u201CMandato do C\xE9u\u201D\ + \ para legitimar os novos governantes?" + - input_choice_list: + A: Honen + B: Tanaka + C: Tokugawa + D: Meiji + input_correct_responses: + - D + input_question: "Que governo japon\xEAs promoveu uma esp\xE9cie de culto nacional\ + \ baseado no imperador e nas suas associa\xE7\xF5es com os kami?" + - input_choice_list: + A: Textos rituais + B: "Textos filos\xF3ficos" + C: Hinos + D: "Hist\xF3rias de origem" + input_correct_responses: + - B + input_question: Como os Upanishads podem ser caracterizados? +include: _continuation_template_yaml +process_docs: !function utils.process_docs_world_religions +tag: mmlu_pt_llama_humanities_tasks +task: mmlu_pt_llama_world_religions +task_alias: world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/utils.py b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ef951f1e45353e24ef7ae4242b3ddb5c9d55f053 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_pt/utils.py @@ -0,0 +1,104 @@ +from functools import partial + +import datasets + + +def process_docs(dataset: datasets.Dataset, subtask) -> datasets.Dataset: + return dataset.filter( + lambda example: example["subtask_name"] == f"mmlu_pt_chat.{subtask}" + ) + + +process_docs_formal_logic = partial(process_docs, subtask="formal_logic") +process_docs_college_medicine = partial(process_docs, subtask="college_medicine") +process_docs_nutrition = partial(process_docs, subtask="nutrition") +process_docs_econometrics = partial(process_docs, subtask="econometrics") +process_docs_college_chemistry = partial(process_docs, subtask="college_chemistry") +process_docs_high_school_mathematics = partial( + process_docs, subtask="high_school_mathematics" +) +process_docs_high_school_us_history = partial( + process_docs, subtask="high_school_us_history" +) +process_docs_anatomy = partial(process_docs, subtask="anatomy") +process_docs_machine_learning = partial(process_docs, subtask="machine_learning") +process_docs_logical_fallacies = partial(process_docs, subtask="logical_fallacies") +process_docs_professional_accounting = partial( + process_docs, subtask="professional_accounting" +) +process_docs_management = partial(process_docs, subtask="management") +process_docs_computer_security = partial(process_docs, subtask="computer_security") +process_docs_conceptual_physics = partial(process_docs, subtask="conceptual_physics") +process_docs_high_school_geography = partial( + process_docs, subtask="high_school_geography" +) +process_docs_college_computer_science = partial( + process_docs, subtask="college_computer_science" +) +process_docs_international_law = partial(process_docs, subtask="international_law") +process_docs_professional_medicine = partial( + process_docs, subtask="professional_medicine" +) +process_docs_moral_disputes = partial(process_docs, subtask="moral_disputes") +process_docs_high_school_macroeconomics = partial( + process_docs, subtask="high_school_macroeconomics" +) +process_docs_public_relations = partial(process_docs, subtask="public_relations") +process_docs_high_school_world_history = partial( + process_docs, subtask="high_school_world_history" +) +process_docs_business_ethics = partial(process_docs, subtask="business_ethics") +process_docs_college_physics = partial(process_docs, subtask="college_physics") +process_docs_high_school_government_and_politics = partial( + process_docs, subtask="high_school_government_and_politics" +) +process_docs_college_mathematics = partial(process_docs, subtask="college_mathematics") +process_docs_electrical_engineering = partial( + process_docs, subtask="electrical_engineering" +) +process_docs_professional_psychology = partial( + process_docs, subtask="professional_psychology" +) +process_docs_clinical_knowledge = partial(process_docs, subtask="clinical_knowledge") +process_docs_human_sexuality = partial(process_docs, subtask="human_sexuality") +process_docs_sociology = partial(process_docs, subtask="sociology") +process_docs_prehistory = partial(process_docs, subtask="prehistory") +process_docs_high_school_psychology = partial( + process_docs, subtask="high_school_psychology" +) +process_docs_abstract_algebra = partial(process_docs, subtask="abstract_algebra") +process_docs_high_school_computer_science = partial( + process_docs, subtask="high_school_computer_science" +) +process_docs_medical_genetics = partial(process_docs, subtask="medical_genetics") +process_docs_elementary_mathematics = partial( + process_docs, subtask="elementary_mathematics" +) +process_docs_professional_law = partial(process_docs, subtask="professional_law") +process_docs_miscellaneous = partial(process_docs, subtask="miscellaneous") +process_docs_high_school_chemistry = partial( + process_docs, subtask="high_school_chemistry" +) +process_docs_human_aging = partial(process_docs, subtask="human_aging") +process_docs_high_school_european_history = partial( + process_docs, subtask="high_school_european_history" +) +process_docs_college_biology = partial(process_docs, subtask="college_biology") +process_docs_astronomy = partial(process_docs, subtask="astronomy") +process_docs_high_school_physics = partial(process_docs, subtask="high_school_physics") +process_docs_global_facts = partial(process_docs, subtask="global_facts") +process_docs_jurisprudence = partial(process_docs, subtask="jurisprudence") +process_docs_us_foreign_policy = partial(process_docs, subtask="us_foreign_policy") +process_docs_virology = partial(process_docs, subtask="virology") +process_docs_philosophy = partial(process_docs, subtask="philosophy") +process_docs_high_school_statistics = partial( + process_docs, subtask="high_school_statistics" +) +process_docs_high_school_microeconomics = partial( + process_docs, subtask="high_school_microeconomics" +) +process_docs_high_school_biology = partial(process_docs, subtask="high_school_biology") +process_docs_moral_scenarios = partial(process_docs, subtask="moral_scenarios") +process_docs_security_studies = partial(process_docs, subtask="security_studies") +process_docs_world_religions = partial(process_docs, subtask="world_religions") +process_docs_marketing = partial(process_docs, subtask="marketing") diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_continuation_template_yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_continuation_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bd4b5a4d05f3217d935e61aa8a51704a896dade --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_continuation_template_yaml @@ -0,0 +1,32 @@ +dataset_path: meta-llama/Llama-3.1-8B-Instruct-evals +dataset_name: Llama-3.1-8B-Instruct-evals__multilingual_mmlu_th__details +output_type: generate_until +test_split: latest +doc_to_text: "Given the following question and four candidate answers (A, B, C and D), choose the best answer.\nQuestion: {{input_question.strip()}}\nA. {{input_choice_list.A}}\nB. {{input_choice_list.B}}\nC. {{input_choice_list.C}}\nD. {{input_choice_list.D}}\nYour response should end with \"The best answer is [the_answer_letter]\" where the [the_answer_letter] is one of A, B, C or D." +gen_prefix: "The best answer is" +doc_to_target: "{{input_correct_responses[0]}}." +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "\\$" + - "\\.$" +generation_kwargs: + do_sample: false + temperature: 0 + until: + - "." + max_gen_toks: 10 +filter_list: + - name: strict_match + filter: + - function: remove_whitespace + - function: take_first +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebb785c3a1d80e0d6fd6970e7b722d5a59ce9dff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_humanities.yaml @@ -0,0 +1,11 @@ +group: mmlu_th_llama_humanities +group_alias: humanities +task: + - mmlu_th_llama_humanities_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_llama.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_llama.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5af2a55f03e2a7fda53a822c47eeeae6fff6eb78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_llama.yaml @@ -0,0 +1,13 @@ +group: mmlu_th_llama +task: + - mmlu_th_llama_stem + - mmlu_th_llama_other + - mmlu_th_llama_social_sciences + - mmlu_th_llama_humanities +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_other.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..783b1324412bd47840946cda4d4890ce9b104604 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_other.yaml @@ -0,0 +1,11 @@ +group: mmlu_th_llama_other +group_alias: other +task: + - mmlu_th_llama_other_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4472a03e91e9c3f67fd0bec687c964d95fbe2688 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_social_sciences.yaml @@ -0,0 +1,11 @@ +group: mmlu_th_llama_social_sciences +group_alias: social sciences +task: + - mmlu_th_llama_social_sciences_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ec4f1f0d2f1aa61a338adb8c66e47c69e1af5e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/_mmlu_th_stem.yaml @@ -0,0 +1,11 @@ +group: mmlu_th_llama_stem +group_alias: stem +task: + - mmlu_th_llama_stem_tasks +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: True + filter_list: [strict_match] +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c16166f5565f3ac47945a2c1f509e30e9ad5a0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_abstract_algebra.yaml @@ -0,0 +1,75 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '0' + B: '1' + C: '2' + D: '3' + input_correct_responses: + - B + input_question: "\u0E04\u0E49\u0E19\u0E2B\u0E32 c \u0E17\u0E31\u0E49\u0E07\u0E2B\ + \u0E21\u0E14\u0E43\u0E19 Z_3 \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\ + \ Z_3[x]/(x^2 + c) \u0E40\u0E1B\u0E47\u0E19\u0E1F\u0E34\u0E25\u0E14\u0E4C" + - input_choice_list: + A: "\u0E17\u0E23\u0E39\u0E17\u0E23\u0E39" + B: "\u0E40\u0E17\u0E47\u0E08\u0E40\u0E17\u0E47\u0E08" + C: "\u0E16\u0E39\u0E01\u0E1C\u0E34\u0E14" + D: "\u0E1C\u0E34\u0E14\u0E16\u0E39\u0E01" + input_correct_responses: + - B + input_question: "\u0E41\u0E16\u0E25\u0E07\u0E01\u0E32\u0E23\u0E13\u0E4C 1 | \u0E16\ + \u0E49\u0E32 aH \u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E21\u0E32\u0E0A\u0E34\u0E01\ + \u0E02\u0E2D\u0E07\u0E01\u0E25\u0E38\u0E48\u0E21\u0E15\u0E31\u0E27\u0E1B\u0E23\ + \u0E30\u0E01\u0E2D\u0E1A \u0E14\u0E31\u0E07\u0E19\u0E31\u0E49\u0E19 |aH| \u0E2B\ + \u0E32\u0E23 |\u0E01|. \u0E16\u0E49\u0E2D\u0E22\u0E41\u0E16\u0E25\u0E07 2 |\ + \ \u0E16\u0E49\u0E32 H \u0E41\u0E25\u0E30 K \u0E40\u0E1B\u0E47\u0E19\u0E01\u0E25\ + \u0E38\u0E48\u0E21\u0E22\u0E48\u0E2D\u0E22\u0E02\u0E2D\u0E07 G \u0E41\u0E25\u0E49\ + \u0E27 HK \u0E04\u0E37\u0E2D\u0E01\u0E25\u0E38\u0E48\u0E21\u0E22\u0E48\u0E2D\ + \u0E22\u0E02\u0E2D\u0E07 G" + - input_choice_list: + A: "\u0E17\u0E23\u0E39\u0E17\u0E23\u0E39" + B: "\u0E40\u0E17\u0E47\u0E08\u0E40\u0E17\u0E47\u0E08" + C: "\u0E16\u0E39\u0E01\u0E1C\u0E34\u0E14" + D: "\u0E1C\u0E34\u0E14\u0E16\u0E39\u0E01" + input_correct_responses: + - C + input_question: "\u0E41\u0E16\u0E25\u0E07\u0E01\u0E32\u0E23\u0E13\u0E4C 1 | \u0E17\ + \u0E38\u0E01\u0E2D\u0E07\u0E04\u0E4C\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E02\ + \u0E2D\u0E07\u0E01\u0E25\u0E38\u0E48\u0E21\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E01\ + \u0E25\u0E38\u0E48\u0E21\u0E22\u0E48\u0E2D\u0E22\u0E17\u0E35\u0E48\u0E40\u0E1B\ + \u0E47\u0E19\u0E27\u0E07\u0E08\u0E23\u0E02\u0E2D\u0E07\u0E01\u0E25\u0E38\u0E48\ + \u0E21 \u0E16\u0E49\u0E2D\u0E22\u0E41\u0E16\u0E25\u0E07 2 | \u0E01\u0E25\u0E38\ + \u0E48\u0E21\u0E2A\u0E21\u0E21\u0E32\u0E15\u0E23 S_10 \u0E21\u0E35 10 \u0E2D\ + \u0E07\u0E04\u0E4C\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A" + - input_choice_list: + A: "\u0E17\u0E23\u0E39\u0E17\u0E23\u0E39" + B: "\u0E40\u0E17\u0E47\u0E08\u0E40\u0E17\u0E47\u0E08" + C: "\u0E16\u0E39\u0E01\u0E1C\u0E34\u0E14" + D: "\u0E1C\u0E34\u0E14\u0E16\u0E39\u0E01" + input_correct_responses: + - A + input_question: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21 1| \u0E17\u0E38\u0E01\ + \u0E1F\u0E31\u0E07\u0E01\u0E4C\u0E0A\u0E31\u0E19\u0E08\u0E32\u0E01\u0E40\u0E0B\ + \u0E15\u0E08\u0E33\u0E01\u0E31\u0E14\u0E40\u0E02\u0E49\u0E32\u0E2A\u0E39\u0E48\ + \u0E15\u0E31\u0E27\u0E21\u0E31\u0E19\u0E40\u0E2D\u0E07\u0E08\u0E30\u0E15\u0E49\ + \u0E2D\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E41\u0E1A\u0E1A\u0E2B\u0E19\u0E36\u0E48\ + \u0E07\u0E15\u0E48\u0E2D\u0E2B\u0E19\u0E36\u0E48\u0E07 \u0E16\u0E49\u0E2D\u0E22\ + \u0E41\u0E16\u0E25\u0E07 2 | \u0E01\u0E25\u0E38\u0E48\u0E21\u0E22\u0E48\u0E2D\ + \u0E22\u0E02\u0E2D\u0E07\u0E01\u0E25\u0E38\u0E48\u0E21\u0E2D\u0E32\u0E40\u0E1A\ + \u0E40\u0E25\u0E35\u0E22\u0E19\u0E17\u0E38\u0E01\u0E01\u0E25\u0E38\u0E48\u0E21\ + \u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E32\u0E40\u0E1A\u0E40\u0E25\u0E35\u0E22\u0E19" + - input_choice_list: + A: '0' + B: '3' + C: '12' + D: '30' + input_correct_responses: + - A + input_question: "\u0E04\u0E49\u0E19\u0E2B\u0E32\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\ + \u0E02\u0E2D\u0E07\u0E27\u0E07\u0E41\u0E2B\u0E27\u0E19 2Z" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_abstract_algebra +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_abstract_algebra +task_alias: abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319e4dc492bceea89161df7e51ae79da1643078b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_anatomy.yaml @@ -0,0 +1,80 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2A\u0E48\u0E27\u0E19\u0E42\u0E04\u0E49\u0E07\u0E04\u0E2D\u0E2B\u0E2D\ + \u0E22\u0E2D\u0E31\u0E19\u0E41\u0E23\u0E01" + B: "\u0E2A\u0E48\u0E27\u0E19\u0E42\u0E04\u0E49\u0E07\u0E04\u0E2D\u0E2B\u0E2D\ + \u0E22\u0E17\u0E35\u0E48\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E41\u0E25\u0E30\u0E2A\ + \u0E2D\u0E07" + C: "\u0E2A\u0E48\u0E27\u0E19\u0E42\u0E04\u0E49\u0E07\u0E04\u0E2D\u0E2B\u0E2D\ + \u0E22\u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07" + D: "\u0E04\u0E2D\u0E2B\u0E2D\u0E22\u0E2A\u0E48\u0E27\u0E19\u0E42\u0E04\u0E49\ + \u0E07\u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07\u0E41\u0E25\u0E30\u0E2A\u0E32\u0E21" + input_correct_responses: + - D + input_question: "\u0E15\u0E49\u0E19\u0E01\u0E33\u0E40\u0E19\u0E34\u0E14\u0E02\u0E2D\ + \u0E07\u0E15\u0E31\u0E27\u0E2D\u0E48\u0E2D\u0E19\u0E02\u0E2D\u0E07\u0E01\u0E23\ + \u0E30\u0E14\u0E39\u0E01\u0E44\u0E2E\u0E2D\u0E2D\u0E22\u0E14\u0E4C\u0E04\u0E37\ + \u0E2D\u0E2D\u0E30\u0E44\u0E23?" + - input_choice_list: + A: "\u0E40\u0E2A\u0E49\u0E19\u0E1B\u0E23\u0E30\u0E2A\u0E32\u0E17\u0E40\u0E2B\ + \u0E19\u0E37\u0E2D\u0E2D\u0E2D\u0E23\u0E4C\u0E1A\u0E34\u0E17\u0E31\u0E25" + B: "\u0E40\u0E2A\u0E49\u0E19\u0E1B\u0E23\u0E30\u0E2A\u0E32\u0E17 infraorbital" + C: "\u0E1B\u0E23\u0E30\u0E2A\u0E32\u0E17\u0E08\u0E34\u0E15" + D: "\u0E44\u0E21\u0E48\u0E21\u0E35\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E01\u0E25\ + \u0E48\u0E32\u0E27\u0E02\u0E49\u0E32\u0E07\u0E15\u0E49\u0E19" + input_correct_responses: + - D + input_question: "\u0E41\u0E02\u0E19\u0E07\u0E43\u0E14\u0E02\u0E2D\u0E07\u0E40\u0E2A\ + \u0E49\u0E19\u0E1B\u0E23\u0E30\u0E2A\u0E32\u0E17\u0E44\u0E15\u0E23\u0E40\u0E08\ + \u0E21\u0E34\u0E19\u0E31\u0E25\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E35\u0E49\ + \u0E21\u0E35\u0E01\u0E23\u0E30\u0E1A\u0E27\u0E19\u0E01\u0E32\u0E23\u0E2A\u0E31\ + \u0E48\u0E07\u0E01\u0E32\u0E23\u0E02\u0E2D\u0E07\u0E42\u0E0B\u0E21\u0E32\u0E15\ + \u0E34\u0E01" + - input_choice_list: + A: "\u0E44\u0E21\u0E48\u0E21\u0E35\u0E1B\u0E23\u0E30\u0E2A\u0E32\u0E17\u0E2A\ + \u0E31\u0E21\u0E1C\u0E31\u0E2A\u0E1B\u0E01\u0E04\u0E25\u0E38\u0E21\u0E14\u0E49\ + \u0E27\u0E22\u0E40\u0E2A\u0E49\u0E19" + B: "\u0E04\u0E31\u0E48\u0E19\u0E14\u0E49\u0E27\u0E22\u0E0A\u0E48\u0E2D\u0E07\ + \u0E27\u0E48\u0E32\u0E07 2 \u0E21\u0E21." + C: "\u0E02\u0E22\u0E32\u0E22\u0E40\u0E02\u0E49\u0E32\u0E44\u0E1B\u0E43\u0E19\ + \u0E04\u0E2D" + D: "\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E40\u0E22\ + \u0E37\u0E48\u0E2D\u0E1A\u0E38\u0E1C\u0E34\u0E27\u0E17\u0E32\u0E07\u0E40\u0E14\ + \u0E34\u0E19\u0E2B\u0E32\u0E22\u0E43\u0E08" + input_correct_responses: + - C + input_question: "\u0E40\u0E22\u0E37\u0E48\u0E2D\u0E2B\u0E38\u0E49\u0E21\u0E1B\u0E2D\ + \u0E14" + - input_choice_list: + A: "\u0E1F\u0E31\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E1A\u0E19\u0E17\u0E35\u0E48\ + \u0E2A\u0E1A\u0E01\u0E31\u0E19\u0E40\u0E01\u0E34\u0E19" + B: "\u0E42\u0E2D\u0E40\u0E27\u0E2D\u0E23\u0E4C\u0E40\u0E08\u0E47\u0E15\u0E40\ + \u0E0A\u0E34\u0E07\u0E25\u0E1A\u0E02\u0E2D\u0E07\u0E1F\u0E31\u0E19\u0E2B\u0E19\ + \u0E49\u0E32\u0E01\u0E25\u0E32\u0E07\u0E1A\u0E19" + C: "\u0E1F\u0E31\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E1A\u0E19\u0E22\u0E37\u0E48\ + \u0E19\u0E2D\u0E2D\u0E01\u0E21\u0E32\u0E21\u0E32\u0E01\u0E40\u0E01\u0E34\u0E19\ + \u0E44\u0E1B" + D: "\u0E2A\u0E48\u0E27\u0E19\u0E40\u0E01\u0E34\u0E19\u0E02\u0E2D\u0E07\u0E1F\ + \u0E31\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E01\u0E25\u0E32\u0E07\u0E1A\u0E19" + input_correct_responses: + - C + input_question: "\u0E43\u0E19\u0E01\u0E32\u0E23\u0E1A\u0E14\u0E40\u0E04\u0E35\u0E49\ + \u0E22\u0E27 Class II Div 2 \u0E02\u0E2D\u0E07 Angle \u0E21\u0E35" + - input_choice_list: + A: "\u0E0A\u0E48\u0E2D\u0E07\u0E17\u0E49\u0E2D\u0E07" + B: "\u0E01\u0E30\u0E42\u0E2B\u0E25\u0E01\u0E28\u0E35\u0E23\u0E29\u0E30" + C: "\u0E40\u0E22\u0E37\u0E48\u0E2D\u0E2B\u0E38\u0E49\u0E21\u0E1B\u0E2D\u0E14" + D: "\u0E01\u0E23\u0E30\u0E14\u0E39\u0E01\u0E2A\u0E31\u0E19\u0E2B\u0E25\u0E31\ + \u0E07" + input_correct_responses: + - B + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E04\u0E37\u0E2D\u0E42\u0E1E\u0E23\ + \u0E07\u0E23\u0E48\u0E32\u0E07\u0E01\u0E32\u0E22\u0E17\u0E35\u0E48\u0E21\u0E35\ + \u0E15\u0E48\u0E2D\u0E21\u0E43\u0E15\u0E49\u0E2A\u0E21\u0E2D\u0E07?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_anatomy +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_anatomy +task_alias: anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a47d201fe4db9824881aea934fe23df4dcfba20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_astronomy.yaml @@ -0,0 +1,164 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E22\u0E32\u0E01\u0E02\u0E36\u0E49\u0E19\ + \u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E23\u0E16\u0E1A\u0E23\ + \u0E23\u0E17\u0E38\u0E01\u0E2B\u0E19\u0E31\u0E01\u0E01\u0E27\u0E48\u0E32\u0E1A\ + \u0E19\u0E14\u0E32\u0E27\u0E2D\u0E31\u0E07\u0E04\u0E32\u0E23" + B: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E07\u0E48\u0E32\u0E22\u0E01\u0E27\u0E48\ + \u0E32\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E23\u0E16\u0E1A\ + \u0E23\u0E23\u0E17\u0E38\u0E01\u0E21\u0E35\u0E19\u0E49\u0E33\u0E2B\u0E19\u0E31\ + \u0E01\u0E40\u0E1A\u0E32\u0E01\u0E27\u0E48\u0E32\u0E1A\u0E19\u0E14\u0E32\u0E27\ + \u0E2D\u0E31\u0E07\u0E04\u0E32\u0E23" + C: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E22\u0E32\u0E01\u0E02\u0E36\u0E49\u0E19\ + \u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E23\u0E16\u0E1A\u0E23\ + \u0E23\u0E17\u0E38\u0E01\u0E40\u0E1A\u0E32\u0E01\u0E27\u0E48\u0E32\u0E1A\u0E19\ + \u0E14\u0E32\u0E27\u0E2D\u0E31\u0E07\u0E04\u0E32\u0E23" + D: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E01\ + \u0E31\u0E19\u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\u0E04\u0E38\u0E13\u0E08\u0E30\ + \u0E2D\u0E22\u0E39\u0E48\u0E17\u0E35\u0E48\u0E44\u0E2B\u0E19" + input_correct_responses: + - D + input_question: "\u0E04\u0E38\u0E13\u0E01\u0E33\u0E25\u0E31\u0E07\u0E40\u0E02\u0E47\ + \u0E19\u0E23\u0E16\u0E1A\u0E23\u0E23\u0E17\u0E38\u0E01\u0E44\u0E1B\u0E15\u0E32\ + \u0E21\u0E16\u0E19\u0E19 \u0E21\u0E31\u0E19\u0E08\u0E30\u0E07\u0E48\u0E32\u0E22\ + \u0E01\u0E27\u0E48\u0E32\u0E44\u0E2B\u0E21\u0E17\u0E35\u0E48\u0E08\u0E30\u0E40\ + \u0E23\u0E48\u0E07\u0E23\u0E16\u0E1A\u0E23\u0E23\u0E17\u0E38\u0E01\u0E04\u0E31\ + \u0E19\u0E19\u0E35\u0E49\u0E1A\u0E19\u0E14\u0E32\u0E27\u0E2D\u0E31\u0E07\u0E04\ + \u0E32\u0E23 \u0E17\u0E33\u0E44\u0E21 (\u0E16\u0E37\u0E2D\u0E27\u0E48\u0E32\u0E44\ + \u0E21\u0E48\u0E21\u0E35\u0E41\u0E23\u0E07\u0E40\u0E2A\u0E35\u0E22\u0E14\u0E17\ + \u0E32\u0E19)" + - input_choice_list: + A: "\u0E41\u0E16\u0E1A\u0E44\u0E04\u0E40\u0E1B\u0E2D\u0E23\u0E4C; \u0E14\u0E32\ + \u0E27\u0E2B\u0E32\u0E07\u0E04\u0E32\u0E1A\u0E2A\u0E31\u0E49\u0E19\u0E21\u0E35\ + \u0E41\u0E19\u0E27\u0E42\u0E19\u0E49\u0E21\u0E17\u0E35\u0E48\u0E08\u0E30\u0E2D\ + \u0E22\u0E39\u0E48\u0E43\u0E19\u0E23\u0E30\u0E19\u0E32\u0E1A\u0E02\u0E2D\u0E07\ + \u0E23\u0E30\u0E1A\u0E1A\u0E2A\u0E38\u0E23\u0E34\u0E22\u0E30\u0E40\u0E0A\u0E48\ + \u0E19\u0E40\u0E14\u0E35\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E41\u0E16\u0E1A\u0E44\ + \u0E04\u0E40\u0E1B\u0E2D\u0E23\u0E4C" + B: "\u0E41\u0E16\u0E1A\u0E44\u0E04\u0E40\u0E1B\u0E2D\u0E23\u0E4C; \u0E14\u0E32\ + \u0E27\u0E2B\u0E32\u0E07\u0E04\u0E32\u0E1A\u0E2A\u0E31\u0E49\u0E19\u0E21\u0E31\ + \u0E01\u0E08\u0E30\u0E21\u0E32\u0E08\u0E32\u0E01\u0E17\u0E34\u0E28\u0E17\u0E32\ + \u0E07\u0E2A\u0E38\u0E48\u0E21\u0E0B\u0E36\u0E48\u0E07\u0E1A\u0E48\u0E07\u0E0A\ + \u0E35\u0E49\u0E16\u0E36\u0E07\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E08\u0E32\ + \u0E22\u0E15\u0E31\u0E27\u0E02\u0E2D\u0E07\u0E14\u0E32\u0E27\u0E2B\u0E32\u0E07\ + \u0E17\u0E23\u0E07\u0E01\u0E25\u0E21\u0E17\u0E35\u0E48\u0E40\u0E23\u0E35\u0E22\ + \u0E01\u0E27\u0E48\u0E32\u0E41\u0E16\u0E1A\u0E44\u0E04\u0E40\u0E1B\u0E2D\u0E23\ + \u0E4C" + C: "\u0E41\u0E16\u0E1A\u0E14\u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\ + \u0E4C\u0E19\u0E49\u0E2D\u0E22 \u0E14\u0E32\u0E27\u0E2B\u0E32\u0E07\u0E04\u0E32\ + \u0E1A\u0E2A\u0E31\u0E49\u0E19\u0E21\u0E35\u0E04\u0E32\u0E1A\u0E01\u0E32\u0E23\ + \u0E42\u0E04\u0E08\u0E23\u0E04\u0E25\u0E49\u0E32\u0E22\u0E01\u0E31\u0E1A\u0E14\ + \u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\u0E19\u0E49\u0E2D\u0E22\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E40\u0E27\u0E2A\u0E15\u0E32 \u0E41\u0E25\u0E30\ + \u0E1E\u0E1A\u0E43\u0E19\u0E23\u0E30\u0E19\u0E32\u0E1A\u0E02\u0E2D\u0E07\u0E23\ + \u0E30\u0E1A\u0E1A\u0E2A\u0E38\u0E23\u0E34\u0E22\u0E30\u0E40\u0E0A\u0E48\u0E19\ + \u0E40\u0E14\u0E35\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E41\u0E16\u0E1A\u0E14\u0E32\ + \u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\u0E19\u0E49\u0E2D\u0E22" + D: "\u0E40\u0E21\u0E06\u0E2D\u0E2D\u0E23\u0E4C\u0E15; \u0E14\u0E32\u0E27\u0E2B\ + \u0E32\u0E07\u0E04\u0E32\u0E1A\u0E2A\u0E31\u0E49\u0E19\u0E21\u0E35\u0E41\u0E19\ + \u0E27\u0E42\u0E19\u0E49\u0E21\u0E17\u0E35\u0E48\u0E08\u0E30\u0E2D\u0E22\u0E39\ + \u0E48\u0E43\u0E19\u0E23\u0E30\u0E19\u0E32\u0E1A\u0E02\u0E2D\u0E07\u0E23\u0E30\ + \u0E1A\u0E1A\u0E2A\u0E38\u0E23\u0E34\u0E22\u0E30\u0E40\u0E0A\u0E48\u0E19\u0E40\ + \u0E14\u0E35\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E40\u0E21\u0E06\u0E2D\u0E2D\u0E23\ + \u0E4C\u0E15" + input_correct_responses: + - A + input_question: "\u0E14\u0E32\u0E27\u0E2B\u0E32\u0E07\u0E04\u0E32\u0E1A\u0E2A\u0E31\ + \u0E49\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E43\u0E2B\u0E0D\u0E48\u0E21\u0E32\u0E08\ + \u0E32\u0E01\u0E44\u0E2B\u0E19\u0E41\u0E25\u0E30\u0E40\u0E23\u0E32\u0E08\u0E30\ + \u0E23\u0E39\u0E49\u0E44\u0E14\u0E49\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E23\ + ?" + - input_choice_list: + A: "\u0E2D\u0E35\u0E01 10,000 \u0E40\u0E17\u0E48\u0E32" + B: "\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E2D\u0E35\u0E01\ + \ 100 \u0E40\u0E17\u0E48\u0E32" + C: "\u0E2D\u0E35\u0E01 1,000 \u0E40\u0E17\u0E48\u0E32" + D: "\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19 10 \u0E40\u0E17\u0E48\ + \u0E32" + input_correct_responses: + - A + input_question: "\u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32\u0E23\u0E39\u0E21\ + \u0E48\u0E32\u0E19\u0E15\u0E32\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13\u0E21\u0E35\ + \u0E40\u0E2A\u0E49\u0E19\u0E1C\u0E48\u0E32\u0E19\u0E28\u0E39\u0E19\u0E22\u0E4C\ + \u0E01\u0E25\u0E32\u0E07 5 \u0E21\u0E21. \u0E41\u0E25\u0E30\u0E04\u0E38\u0E13\ + \u0E21\u0E35\u0E01\u0E25\u0E49\u0E2D\u0E07\u0E42\u0E17\u0E23\u0E17\u0E23\u0E23\ + \u0E28\u0E19\u0E4C\u0E17\u0E35\u0E48\u0E21\u0E35\u0E23\u0E39\u0E23\u0E31\u0E1A\ + \u0E41\u0E2A\u0E07 50 \u0E0B\u0E21. \u0E01\u0E25\u0E49\u0E2D\u0E07\u0E42\u0E17\ + \u0E23\u0E17\u0E23\u0E23\u0E28\u0E19\u0E4C\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\ + \u0E23\u0E27\u0E1A\u0E23\u0E27\u0E21\u0E41\u0E2A\u0E07\u0E44\u0E14\u0E49\u0E21\ + \u0E32\u0E01\u0E01\u0E27\u0E48\u0E32\u0E15\u0E32\u0E02\u0E2D\u0E07\u0E04\u0E38\ + \u0E13\u0E21\u0E32\u0E01\u0E41\u0E04\u0E48\u0E44\u0E2B\u0E19?" + - input_choice_list: + A: "\u0E14\u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\u0E14\u0E27\ + \u0E07\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E40\u0E04\u0E22\u0E01\u0E48\u0E2D\u0E15\ + \u0E31\u0E27\u0E02\u0E36\u0E49\u0E19\u0E17\u0E35\u0E48\u0E19\u0E35\u0E48 \u0E41\ + \u0E15\u0E48\u0E16\u0E39\u0E01\u0E41\u0E22\u0E01\u0E2D\u0E2D\u0E01\u0E08\u0E32\ + \u0E01\u0E01\u0E31\u0E19\u0E42\u0E14\u0E22\u0E01\u0E32\u0E23\u0E0A\u0E19\u0E01\ + \u0E31\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E23\u0E38\u0E19\u0E41\u0E23\u0E07" + B: "\u0E43\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E19\u0E35\u0E49\u0E02\u0E2D\u0E07\ + \u0E40\u0E19\u0E1A\u0E34\u0E27\u0E25\u0E32\u0E2A\u0E38\u0E23\u0E34\u0E22\u0E30\ + \u0E21\u0E35\u0E27\u0E31\u0E2A\u0E14\u0E38\u0E44\u0E21\u0E48\u0E40\u0E1E\u0E35\ + \u0E22\u0E07\u0E1E\u0E2D\u0E17\u0E35\u0E48\u0E08\u0E30\u0E01\u0E48\u0E2D\u0E15\ + \u0E31\u0E27\u0E40\u0E1B\u0E47\u0E19\u0E14\u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\ + \u0E30\u0E2B\u0E4C" + C: "\u0E21\u0E35\u0E27\u0E31\u0E2A\u0E14\u0E38\u0E17\u0E35\u0E48\u0E40\u0E1B\ + \u0E47\u0E19\u0E2B\u0E34\u0E19\u0E21\u0E32\u0E01\u0E40\u0E01\u0E34\u0E19\u0E44\ + \u0E1B\u0E17\u0E35\u0E48\u0E08\u0E30\u0E01\u0E48\u0E2D\u0E15\u0E31\u0E27\u0E40\ + \u0E1B\u0E47\u0E19\u0E14\u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\ + \u0E1A\u0E19\u0E1E\u0E37\u0E49\u0E19\u0E42\u0E25\u0E01 \u0E41\u0E15\u0E48\u0E21\ + \u0E35\u0E27\u0E31\u0E2A\u0E14\u0E38\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\ + \u0E01\u0E4A\u0E32\u0E0B\u0E44\u0E21\u0E48\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E1E\ + \u0E2D\u0E17\u0E35\u0E48\u0E08\u0E30\u0E01\u0E48\u0E2D\u0E15\u0E31\u0E27\u0E40\ + \u0E1B\u0E47\u0E19\u0E14\u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\ + \u0E42\u0E08\u0E40\u0E27\u0E35\u0E22\u0E19" + D: "\u0E01\u0E32\u0E23\u0E2A\u0E31\u0E48\u0E19\u0E1E\u0E49\u0E2D\u0E07\u0E01\ + \u0E31\u0E1A\u0E14\u0E32\u0E27\u0E1E\u0E24\u0E2B\u0E31\u0E2A\u0E1A\u0E14\u0E35\ + \u0E17\u0E33\u0E43\u0E2B\u0E49\u0E27\u0E31\u0E2A\u0E14\u0E38\u0E44\u0E21\u0E48\ + \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E23\u0E27\u0E21\u0E15\u0E31\u0E27\u0E01\ + \u0E31\u0E19\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E14\ + \u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\u0E44\u0E14\u0E49" + input_correct_responses: + - D + input_question: "\u0E40\u0E2B\u0E15\u0E38\u0E43\u0E14\u0E08\u0E36\u0E07\u0E44\u0E21\ + \u0E48\u0E21\u0E35\u0E14\u0E32\u0E27\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\ + \u0E0B\u0E36\u0E48\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E35\u0E48\u0E15\u0E31\ + \u0E49\u0E07\u0E02\u0E2D\u0E07\u0E41\u0E16\u0E1A\u0E14\u0E32\u0E27\u0E40\u0E04\ + \u0E23\u0E32\u0E30\u0E2B\u0E4C\u0E19\u0E49\u0E2D\u0E22" + - input_choice_list: + A: "\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E1E\u0E37\u0E49\ + \u0E19\u0E1C\u0E34\u0E27\u0E16\u0E39\u0E01\u0E1B\u0E01\u0E04\u0E25\u0E38\u0E21\ + \u0E14\u0E49\u0E27\u0E22\u0E41\u0E23\u0E48\u0E18\u0E32\u0E15\u0E38\u0E17\u0E35\ + \u0E48\u0E16\u0E39\u0E01\u0E2D\u0E2D\u0E01\u0E0B\u0E34\u0E44\u0E14\u0E0B\u0E4C\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2B\u0E19\u0E31\u0E01 ("\u0E2A\u0E19\ + \u0E34\u0E21")" + B: "\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E0A\u0E31\u0E49\ + \u0E19\u0E1A\u0E23\u0E23\u0E22\u0E32\u0E01\u0E32\u0E28\u0E08\u0E30\u0E01\u0E23\ + \u0E30\u0E08\u0E32\u0E22\u0E41\u0E2A\u0E07\u0E21\u0E32\u0E01\u0E02\u0E36\u0E49\ + \u0E19\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E04\u0E27\u0E32\u0E21\u0E22\u0E32\ + \u0E27\u0E04\u0E25\u0E37\u0E48\u0E19\u0E2A\u0E35\u0E19\u0E49\u0E33\u0E40\u0E07\ + \u0E34\u0E19\u0E17\u0E35\u0E48\u0E2A\u0E48\u0E07\u0E41\u0E2A\u0E07\u0E2A\u0E35\ + \u0E41\u0E14\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E43\u0E2B\ + \u0E0D\u0E48" + C: "\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E14\u0E32\u0E27\ + \u0E2D\u0E31\u0E07\u0E04\u0E32\u0E23\u0E16\u0E39\u0E01\u0E1B\u0E01\u0E04\u0E25\ + \u0E38\u0E21\u0E14\u0E49\u0E27\u0E22\u0E25\u0E32\u0E27\u0E32\u0E42\u0E1A\u0E23\ + \u0E32\u0E13\u0E0B\u0E36\u0E48\u0E07\u0E21\u0E35\u0E2A\u0E35\u0E41\u0E14\u0E07" + D: "\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E19\u0E49\u0E33\ + \u0E17\u0E35\u0E48\u0E44\u0E2B\u0E25\u0E1A\u0E19\u0E1E\u0E37\u0E49\u0E19\u0E1C\ + \u0E34\u0E27\u0E14\u0E32\u0E27\u0E2D\u0E31\u0E07\u0E04\u0E32\u0E23\u0E44\u0E14\ + \u0E49\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E41\u0E1B\u0E25\u0E07\u0E41\ + \u0E23\u0E48\u0E18\u0E32\u0E15\u0E38\u0E1A\u0E19\u0E1E\u0E37\u0E49\u0E19\u0E1C\ + \u0E34\u0E27\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E2B\u0E25\u0E32\u0E22\u0E1E\u0E31\ + \u0E19\u0E25\u0E49\u0E32\u0E19\u0E1B\u0E35\u0E01\u0E48\u0E2D\u0E19" + input_correct_responses: + - A + input_question: "\u0E17\u0E33\u0E44\u0E21\u0E14\u0E32\u0E27\u0E2D\u0E31\u0E07\u0E04\ + \u0E32\u0E23\u0E16\u0E36\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E35\u0E41\u0E14\ + \u0E07?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_astronomy +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_astronomy +task_alias: astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55f217aae576656885aacb69d78f5ae9ad8d32e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_business_ethics.yaml @@ -0,0 +1,167 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2A\u0E34\u0E48\u0E07\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01 \u0E2D\u0E33\ + \u0E19\u0E32\u0E08 \u0E04\u0E27\u0E32\u0E21\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E34\ + \u0E2A\u0E23\u0E30" + B: "\u0E01\u0E32\u0E23\u0E40\u0E1C\u0E22\u0E41\u0E1E\u0E23\u0E48, \u0E17\u0E23\ + \u0E31\u0E1E\u0E22\u0E32\u0E01\u0E23\u0E44\u0E21\u0E48\u0E40\u0E1E\u0E35\u0E22\ + \u0E07\u0E1E\u0E2D, \u0E01\u0E32\u0E23\u0E1E\u0E36\u0E48\u0E07\u0E1E\u0E32\ + \u0E0B\u0E36\u0E48\u0E07\u0E01\u0E31\u0E19\u0E41\u0E25\u0E30\u0E01\u0E31\u0E19" + C: "\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E2A\u0E31\u0E21\u0E1E\ + \u0E31\u0E19\u0E18\u0E4C \u0E2D\u0E33\u0E19\u0E32\u0E08 \u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E34\u0E2A\u0E23\u0E30" + D: "\u0E2A\u0E34\u0E48\u0E07\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01 \u0E2D\u0E33\ + \u0E19\u0E32\u0E08 \u0E01\u0E32\u0E23\u0E1E\u0E36\u0E48\u0E07\u0E1E\u0E32\u0E0B\ + \u0E36\u0E48\u0E07\u0E01\u0E31\u0E19\u0E41\u0E25\u0E30\u0E01\u0E31\u0E19" + input_correct_responses: + - D + input_question: "\u0E19\u0E2D\u0E01\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E08\u0E32\u0E01\ + \u0E01\u0E23\u0E13\u0E35\u0E18\u0E38\u0E23\u0E01\u0E34\u0E08\u0E2A\u0E33\u0E2B\ + \u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E21\u0E35\u0E2A\u0E48\u0E27\u0E19\u0E23\ + \u0E48\u0E27\u0E21\u0E43\u0E19 CSR \u0E22\u0E31\u0E07\u0E21\u0E35\u0E02\u0E49\ + \u0E2D\u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\u0E17\u0E32\u0E07\u0E28\u0E35\ + \u0E25\u0E18\u0E23\u0E23\u0E21\u0E2B\u0E25\u0E32\u0E22\u0E1B\u0E23\u0E30\u0E01\ + \u0E32\u0E23\u0E17\u0E35\u0E48\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\ + \u0E2D\u0E07\u0E01\u0E31\u0E1A: _______ \u0E40\u0E0A\u0E34\u0E07\u0E25\u0E1A\ + , _______ \u0E17\u0E35\u0E48\u0E1A\u0E23\u0E34\u0E29\u0E31\u0E17\u0E04\u0E23\ + \u0E2D\u0E1A\u0E04\u0E23\u0E2D\u0E07 \u0E41\u0E25\u0E30 ________ \u0E02\u0E2D\ + \u0E07\u0E18\u0E38\u0E23\u0E01\u0E34\u0E08\u0E41\u0E25\u0E30\u0E2A\u0E31\u0E07\ + \u0E04\u0E21" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E23\u0E31\u0E1A\u0E1C\u0E34\u0E14\u0E0A\u0E2D\ + \u0E1A\u0E15\u0E48\u0E2D\u0E2A\u0E31\u0E07\u0E04\u0E21" + B: "\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\u0E01\u0E32\u0E23\u0E08\u0E23\u0E34\ + \u0E22\u0E18\u0E23\u0E23\u0E21\u0E17\u0E32\u0E07\u0E18\u0E38\u0E23\u0E01\u0E34\ + \u0E08" + C: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E31\u0E48\u0E07\u0E22\u0E37\u0E19" + D: "\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\u0E01\u0E32\u0E23\u0E2A\u0E34\u0E48\ + \u0E07\u0E41\u0E27\u0E14\u0E25\u0E49\u0E2D\u0E21" + input_correct_responses: + - B + input_question: "_______ \u0E40\u0E1B\u0E47\u0E19\u0E04\u0E27\u0E32\u0E21\u0E1E\ + \u0E22\u0E32\u0E22\u0E32\u0E21\u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E43\u0E19\ + \u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\u0E01\u0E32\u0E23\u0E1B\u0E31\u0E0D\u0E2B\ + \u0E32\u0E2B\u0E23\u0E37\u0E2D\u0E1B\u0E31\u0E0D\u0E2B\u0E32\u0E14\u0E49\u0E32\ + \u0E19\u0E08\u0E23\u0E34\u0E22\u0E18\u0E23\u0E23\u0E21\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E32\u0E07\u0E01\u0E32\u0E23\u0E2B\u0E23\ + \u0E37\u0E2D\u0E44\u0E21\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E32\u0E07\u0E01\ + \u0E32\u0E23 \u0E1C\u0E48\u0E32\u0E19\u0E19\u0E42\u0E22\u0E1A\u0E32\u0E22 \u0E41\ + \u0E19\u0E27\u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34 \u0E41\u0E25\u0E30\u0E42\ + \u0E04\u0E23\u0E07\u0E01\u0E32\u0E23\u0E40\u0E09\u0E1E\u0E32\u0E30" + - input_choice_list: + A: "\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01 \u0E08\u0E33\u0E01\u0E31\u0E14 \u0E40\ + \u0E1B\u0E47\u0E19\u0E2D\u0E34\u0E2A\u0E23\u0E30" + B: "\u0E20\u0E32\u0E22\u0E43\u0E19 \u0E08\u0E33\u0E01\u0E31\u0E14 \u0E40\u0E1B\ + \u0E47\u0E19\u0E23\u0E30\u0E22\u0E30 \u0E46" + C: "\u0E19\u0E2D\u0E01, \u0E44\u0E21\u0E48\u0E08\u0E33\u0E01\u0E31\u0E14, \u0E40\ + \u0E1B\u0E47\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E46" + D: "\u0E20\u0E32\u0E22\u0E43\u0E19 \u0E44\u0E21\u0E48\u0E08\u0E33\u0E01\u0E31\ + \u0E14 \u0E2D\u0E34\u0E2A\u0E23\u0E30" + input_correct_responses: + - A + input_question: "\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E41\u0E19\u0E48\ + \u0E43\u0E08\u0E27\u0E48\u0E32\u0E2A\u0E21\u0E32\u0E0A\u0E34\u0E01\u0E04\u0E13\ + \u0E30\u0E01\u0E23\u0E23\u0E21\u0E01\u0E32\u0E23\u0E17\u0E35\u0E48\u0E44\u0E21\ + \u0E48\u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E39\u0E49\u0E1A\u0E23\u0E34\u0E2B\u0E32\ + \u0E23\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E34\ + \u0E2A\u0E23\u0E30 \u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E40\u0E2B\u0E25\ + \u0E48\u0E32\u0E19\u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E02\u0E31\u0E49\u0E19\ + \u0E15\u0E2D\u0E19\u0E2B\u0E25\u0E32\u0E22\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\ + \u0E19\u0E17\u0E35\u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E14\u0E33\u0E40\ + \u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E44\u0E14\u0E49 \u0E0B\u0E36\u0E48\u0E07\ + \u0E23\u0E27\u0E21\u0E16\u0E36\u0E07\u0E01\u0E32\u0E23\u0E41\u0E15\u0E48\u0E07\ + \u0E15\u0E31\u0E49\u0E07\u0E1C\u0E39\u0E49\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\ + \u0E43\u0E0A\u0E48\u0E1C\u0E39\u0E49\u0E1A\u0E23\u0E34\u0E2B\u0E32\u0E23\u0E08\ + \u0E32\u0E01 _______ \u0E1A\u0E23\u0E34\u0E29\u0E31\u0E17 \u0E01\u0E32\u0E23\ + \u0E41\u0E15\u0E48\u0E07\u0E15\u0E31\u0E49\u0E07\u0E43\u0E19\u0E0A\u0E48\u0E27\ + \u0E07\u0E40\u0E27\u0E25\u0E32 _________ \u0E15\u0E25\u0E2D\u0E14\u0E08\u0E19\ + \u0E01\u0E32\u0E23\u0E41\u0E15\u0E48\u0E07\u0E15\u0E31\u0E49\u0E07 _________" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E42\u0E14\u0E22\u0E15\ + \u0E23\u0E07\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E23\u0E38\u0E19\u0E41\u0E23\ + \u0E07, \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E42\u0E14\u0E22\ + \u0E15\u0E23\u0E07\u0E17\u0E35\u0E48\u0E23\u0E38\u0E19\u0E41\u0E23\u0E07,\ + \ \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E42\u0E14\u0E22\u0E2D\ + \u0E49\u0E2D\u0E21, \u0E01\u0E32\u0E23\u0E04\u0E27\u0E48\u0E33\u0E1A\u0E32\ + \u0E15\u0E23" + B: "\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E17\u0E32\u0E07\u0E2D\ + \u0E49\u0E2D\u0E21, \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E14\ + \u0E49\u0E27\u0E22\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E21\u0E37\u0E2D\ + , \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E42\u0E14\u0E22\u0E15\ + \u0E23\u0E07\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E23\u0E38\u0E19\u0E41\u0E23\ + \u0E07, \u0E01\u0E32\u0E23\u0E23\u0E13\u0E23\u0E07\u0E04\u0E4C\u0E14\u0E49\ + \u0E27\u0E22\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25" + C: "\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E17\u0E32\u0E07\u0E2D\ + \u0E49\u0E2D\u0E21, \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E42\ + \u0E14\u0E22\u0E15\u0E23\u0E07\u0E17\u0E35\u0E48\u0E23\u0E38\u0E19\u0E41\u0E23\ + \u0E07, \u0E01\u0E32\u0E23\u0E04\u0E27\u0E48\u0E33\u0E1A\u0E32\u0E15\u0E23\ + \u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E23\ + \u0E38\u0E19\u0E41\u0E23\u0E07" + D: "\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E42\u0E14\u0E22\u0E15\ + \u0E23\u0E07\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E49\u0E04\u0E27\ + \u0E32\u0E21\u0E23\u0E38\u0E19\u0E41\u0E23\u0E07, \u0E01\u0E32\u0E23\u0E01\ + \u0E23\u0E30\u0E17\u0E33\u0E14\u0E49\u0E27\u0E22\u0E40\u0E04\u0E23\u0E37\u0E48\ + \u0E2D\u0E07\u0E21\u0E37\u0E2D, \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\ + \u0E33\u0E42\u0E14\u0E22\u0E2D\u0E49\u0E2D\u0E21, \u0E01\u0E32\u0E23\u0E23\ + \u0E13\u0E23\u0E07\u0E04\u0E4C\u0E14\u0E49\u0E27\u0E22\u0E02\u0E49\u0E2D\u0E21\ + \u0E39\u0E25" + input_correct_responses: + - C + input_question: "\u0E01\u0E25\u0E22\u0E38\u0E17\u0E18\u0E4C\u0E17\u0E35\u0E48\u0E02\ + \u0E31\u0E14\u0E41\u0E22\u0E49\u0E07\u0E01\u0E31\u0E19\u0E2A\u0E32\u0E21\u0E27\ + \u0E34\u0E18\u0E35\u0E17\u0E35\u0E48 CSO \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\ + \u0E21\u0E35\u0E2A\u0E48\u0E27\u0E19\u0E23\u0E48\u0E27\u0E21\u0E40\u0E1E\u0E37\ + \u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E1A\u0E23\u0E23\u0E25\u0E38\u0E40\u0E1B\u0E49\ + \u0E32\u0E2B\u0E21\u0E32\u0E22 \u0E44\u0E14\u0E49\u0E41\u0E01\u0E48 ________\ + \ \u0E0B\u0E36\u0E48\u0E07\u0E42\u0E14\u0E22\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\ + \u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E1A\ + \u0E01\u0E32\u0E23\u0E27\u0E34\u0E08\u0E31\u0E22\u0E41\u0E25\u0E30\u0E01\u0E32\ + \u0E23\u0E2A\u0E37\u0E48\u0E2D\u0E2A\u0E32\u0E23 ________ \u0E0B\u0E36\u0E48\ + \u0E07\u0E2D\u0E32\u0E08\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\u0E2D\ + \u0E07\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E42\u0E08\u0E21\u0E15\u0E35\u0E17\ + \u0E32\u0E07\u0E23\u0E48\u0E32\u0E07\u0E01\u0E32\u0E22\u0E15\u0E48\u0E2D\u0E01\ + \u0E32\u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E07\u0E32\u0E19\u0E02\u0E2D\ + \u0E07\u0E1A\u0E23\u0E34\u0E29\u0E31\u0E17 \u0E2B\u0E23\u0E37\u0E2D ________\ + \ \u0E0B\u0E36\u0E48\u0E07\u0E21\u0E31\u0E01\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\ + \u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E1A _______ \u0E23\u0E39\u0E1B\u0E41\u0E1A\ + \u0E1A\u0E43\u0E14\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E2B\u0E19\u0E36\u0E48\ + \u0E07" + - input_choice_list: + A: "Buycotts, Boycotts, \u0E40\u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35\ + \ Blockchain, \u0E01\u0E32\u0E23\u0E1A\u0E23\u0E34\u0E08\u0E32\u0E04\u0E40\ + \u0E1E\u0E37\u0E48\u0E2D\u0E01\u0E32\u0E23\u0E01\u0E38\u0E28\u0E25" + B: "Buycotts, Boycotts, \u0E40\u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35\ + \u0E14\u0E34\u0E08\u0E34\u0E17\u0E31\u0E25, \u0E22\u0E2D\u0E14\u0E02\u0E32\ + \u0E22\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + C: "\u0E01\u0E32\u0E23\u0E04\u0E27\u0E48\u0E33\u0E1A\u0E32\u0E15\u0E23 Buyalls\ + \ \u0E40\u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35 Blockchain \u0E01\ + \u0E32\u0E23\u0E1A\u0E23\u0E34\u0E08\u0E32\u0E04\u0E40\u0E1E\u0E37\u0E48\u0E2D\ + \u0E01\u0E32\u0E23\u0E01\u0E38\u0E28\u0E25" + D: "\u0E01\u0E32\u0E23\u0E04\u0E27\u0E48\u0E33\u0E1A\u0E32\u0E15\u0E23 Buycotts\ + \ \u0E40\u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35\u0E14\u0E34\u0E08\ + \u0E34\u0E17\u0E31\u0E25 \u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E22\ + \u0E2D\u0E14\u0E02\u0E32\u0E22" + input_correct_responses: + - D + input_question: "\u0E15\u0E23\u0E07\u0E01\u0E31\u0E19\u0E02\u0E49\u0E32\u0E21\u0E01\ + \u0E31\u0E1A _______, _______ \u0E21\u0E35\u0E40\u0E1B\u0E49\u0E32\u0E2B\u0E21\ + \u0E32\u0E22\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E23\u0E32\u0E07\ + \u0E27\u0E31\u0E25\u0E41\u0E01\u0E48\u0E1E\u0E24\u0E15\u0E34\u0E01\u0E23\u0E23\ + \u0E21\u0E17\u0E35\u0E48\u0E40\u0E2D\u0E37\u0E49\u0E2D\u0E2D\u0E33\u0E19\u0E27\ + \u0E22\u0E42\u0E14\u0E22\u0E1A\u0E23\u0E34\u0E29\u0E31\u0E17\u0E15\u0E48\u0E32\ + \u0E07\u0E46 \u0E04\u0E27\u0E32\u0E21\u0E2A\u0E33\u0E40\u0E23\u0E47\u0E08\u0E02\ + \u0E2D\u0E07\u0E01\u0E32\u0E23\u0E23\u0E13\u0E23\u0E07\u0E04\u0E4C\u0E14\u0E31\ + \u0E07\u0E01\u0E25\u0E48\u0E32\u0E27\u0E44\u0E14\u0E49\u0E40\u0E1E\u0E34\u0E48\ + \u0E21\u0E02\u0E36\u0E49\u0E19\u0E1C\u0E48\u0E32\u0E19\u0E01\u0E32\u0E23\u0E43\ + \u0E0A\u0E49 ___________ \u0E0B\u0E36\u0E48\u0E07\u0E0A\u0E48\u0E27\u0E22\u0E43\ + \u0E2B\u0E49\u0E01\u0E32\u0E23\u0E23\u0E13\u0E23\u0E07\u0E04\u0E4C\u0E2D\u0E33\ + \u0E19\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E30\u0E14\u0E27\u0E01\u0E43\ + \u0E2B\u0E49\u0E1A\u0E23\u0E34\u0E29\u0E31\u0E17\u0E43\u0E19\u0E01\u0E32\u0E23\ + \u0E1A\u0E23\u0E23\u0E25\u0E38 _________" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_business_ethics +tag: mmlu_th_llama_other_tasks +task: mmlu_th_llama_business_ethics +task_alias: business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a7ba79dbfaf066403be547c6c88c5f15eb6350a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_clinical_knowledge.yaml @@ -0,0 +1,100 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E40\u0E2D.\u0E17\u0E35.\u0E1E\u0E35." + B: "\u0E1E\u0E28." + C: "\u0E1F\u0E2D\u0E2A\u0E42\u0E1F\u0E04\u0E23\u0E35\u0E40\u0E2D\u0E15\u0E34\ + \u0E19." + D: "\u0E2D\u0E2D\u0E01\u0E0B\u0E34\u0E40\u0E14\u0E17\u0E35\u0E1F\u0E1F\u0E2D\ + \u0E2A\u0E42\u0E1F\u0E23\u0E35\u0E40\u0E25\u0E0A\u0E31\u0E48\u0E19" + input_correct_responses: + - A + input_question: "\u0E1E\u0E25\u0E31\u0E07\u0E07\u0E32\u0E19\u0E2A\u0E33\u0E2B\u0E23\ + \u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2B\u0E14\u0E15\u0E31\u0E27\u0E02\u0E2D\u0E07\ + \u0E01\u0E25\u0E49\u0E32\u0E21\u0E40\u0E19\u0E37\u0E49\u0E2D\u0E17\u0E38\u0E01\ + \u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E21\u0E32\u0E08\u0E32\u0E01:" + - input_choice_list: + A: "\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E0A\u0E32\u0E22\u0E41\u0E25\u0E30\ + \u0E2B\u0E0D\u0E34\u0E07\u0E21\u0E35\u0E2A\u0E35\u0E15\u0E48\u0E32\u0E07\u0E01\ + \u0E31\u0E19" + B: "\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E0A\u0E32\u0E22\u0E22\u0E32\u0E27\ + \u0E01\u0E27\u0E48\u0E32\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E2B\u0E0D\u0E34\ + \u0E07" + C: "\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E0A\u0E32\u0E22\u0E43\u0E2B\u0E0D\ + \u0E48\u0E01\u0E27\u0E48\u0E32\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E2B\u0E0D\ + \u0E34\u0E07" + D: "\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\ + \u0E07\u0E22\u0E32\u0E27\u0E01\u0E27\u0E48\u0E32\u0E2A\u0E32\u0E22\u0E2A\u0E27\ + \u0E19\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22" + input_correct_responses: + - B + input_question: "\u0E2A\u0E32\u0E22\u0E2A\u0E27\u0E19\u0E0A\u0E32\u0E22\u0E41\u0E25\ + \u0E30\u0E2B\u0E0D\u0E34\u0E07\u0E41\u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E01\ + \u0E31\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E23?" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E25\u0E31\u0E01\u0E1E\u0E32\u0E15\u0E31\u0E27\u0E02\ + \u0E2D\u0E07\u0E19\u0E34\u0E49\u0E27\u0E2B\u0E31\u0E27\u0E41\u0E21\u0E48\u0E21\ + \u0E37\u0E2D\u0E08\u0E31\u0E14\u0E17\u0E33\u0E42\u0E14\u0E22\u0E23\u0E32\u0E01\ + \u0E01\u0E23\u0E30\u0E14\u0E39\u0E01\u0E2A\u0E31\u0E19\u0E2B\u0E25\u0E31\u0E07\ + \ T2" + B: "\u0E01\u0E32\u0E23\u0E15\u0E48\u0E2D\u0E15\u0E49\u0E32\u0E19\u0E02\u0E2D\ + \u0E07\u0E19\u0E34\u0E49\u0E27\u0E2B\u0E31\u0E27\u0E41\u0E21\u0E48\u0E21\u0E37\ + \u0E2D\u0E42\u0E14\u0E22\u0E19\u0E42\u0E22\u0E1A\u0E32\u0E22\u0E1D\u0E48\u0E32\ + \u0E22\u0E15\u0E23\u0E07\u0E02\u0E49\u0E32\u0E21\u0E08\u0E31\u0E14\u0E17\u0E33\ + \u0E42\u0E14\u0E22\u0E01\u0E23\u0E30\u0E14\u0E39\u0E01\u0E2A\u0E31\u0E19\u0E2B\ + \u0E25\u0E31\u0E07 T1" + C: "\u0E01\u0E32\u0E23\u0E14\u0E36\u0E07\u0E19\u0E34\u0E49\u0E27\u0E21\u0E32\ + \u0E08\u0E32\u0E01\u0E40\u0E2A\u0E49\u0E19\u0E1B\u0E23\u0E30\u0E2A\u0E32\u0E17\ + \u0E21\u0E31\u0E18\u0E22\u0E10\u0E32\u0E19" + D: "\u0E01\u0E32\u0E23\u0E25\u0E31\u0E01\u0E1E\u0E32\u0E19\u0E34\u0E49\u0E27\ + \u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E37\u0E48\u0E2D\u0E01\u0E25\u0E32\u0E07\u0E42\ + \u0E14\u0E22 Palmar interossei" + input_correct_responses: + - B + input_question: "\u0E43\u0E19\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E40\u0E21\u0E34\ + \u0E19\u0E01\u0E32\u0E23\u0E17\u0E33\u0E07\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E21\ + \u0E37\u0E2D \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07" + - input_choice_list: + A: '4' + B: '3' + C: '2' + D: '1' + input_correct_responses: + - C + input_question: "\u0E04\u0E38\u0E13\u0E04\u0E27\u0E23\u0E1E\u0E22\u0E32\u0E22\u0E32\ + \u0E21\u0E01\u0E35\u0E48\u0E04\u0E23\u0E31\u0E49\u0E07\u0E43\u0E19\u0E01\u0E32\ + \u0E23\u0E09\u0E35\u0E14\u0E22\u0E32\u0E1C\u0E39\u0E49\u0E1B\u0E48\u0E27\u0E22\ + \u0E01\u0E48\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E08\u0E30\u0E2A\u0E48\u0E07\u0E15\ + \u0E48\u0E2D\u0E07\u0E32\u0E19\u0E43\u0E2B\u0E49\u0E40\u0E1E\u0E37\u0E48\u0E2D\ + \u0E19\u0E23\u0E48\u0E27\u0E21\u0E07\u0E32\u0E19\u0E2D\u0E32\u0E27\u0E38\u0E42\ + \u0E2A \u0E15\u0E32\u0E21\u0E04\u0E27\u0E32\u0E21\u0E23\u0E39\u0E49\u0E17\u0E32\ + \u0E07\u0E01\u0E32\u0E23\u0E41\u0E1E\u0E17\u0E22\u0E4C\u0E1B\u0E35 2020" + - input_choice_list: + A: "\u0E44\u0E01\u0E25\u0E42\u0E04\u0E40\u0E08\u0E19\u0E43\u0E2B\u0E49\u0E40\ + \u0E1B\u0E47\u0E19\u0E01\u0E25\u0E39\u0E42\u0E04\u0E2A-1-\u0E1F\u0E2D\u0E2A\ + \u0E40\u0E1F\u0E15" + B: "\u0E44\u0E01\u0E25\u0E42\u0E04\u0E40\u0E08\u0E19\u0E2B\u0E23\u0E37\u0E2D\ + \u0E01\u0E25\u0E39\u0E42\u0E04\u0E2A\u0E40\u0E1B\u0E47\u0E19\u0E1F\u0E23\u0E38\ + \u0E01\u0E42\u0E15\u0E2A" + C: "\u0E44\u0E01\u0E25\u0E42\u0E04\u0E40\u0E08\u0E19\u0E2B\u0E23\u0E37\u0E2D\ + \u0E01\u0E25\u0E39\u0E42\u0E04\u0E2A\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\u0E19\ + \u0E44\u0E1E\u0E23\u0E39\u0E40\u0E27\u0E15\u0E2B\u0E23\u0E37\u0E2D\u0E41\u0E25\ + \u0E04\u0E40\u0E15\u0E15" + D: "\u0E44\u0E01\u0E25\u0E42\u0E04\u0E40\u0E08\u0E19\u0E2B\u0E23\u0E37\u0E2D\ + \u0E01\u0E25\u0E39\u0E42\u0E04\u0E2A\u0E40\u0E1B\u0E47\u0E19\u0E44\u0E1E\u0E23\ + \u0E39\u0E40\u0E27\u0E15\u0E2B\u0E23\u0E37\u0E2D\u0E2D\u0E30\u0E40\u0E0B\u0E17\ + \u0E34\u0E25\u0E42\u0E04\u0E40\u0E2D" + input_correct_responses: + - C + input_question: "Glycolysis \u0E40\u0E1B\u0E47\u0E19\u0E0A\u0E37\u0E48\u0E2D\u0E17\ + \u0E35\u0E48\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E43\u0E2B\u0E49\u0E01\u0E31\u0E1A\ + \u0E27\u0E34\u0E16\u0E35\u0E17\u0E35\u0E48\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\ + \u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E25\ + \u0E35\u0E48\u0E22\u0E19\u0E41\u0E1B\u0E25\u0E07\u0E02\u0E2D\u0E07:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_clinical_knowledge +tag: mmlu_th_llama_other_tasks +task: mmlu_th_llama_clinical_knowledge +task_alias: clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fd63fb0daa209d06a4935b5391c8cca7830f925 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_biology.yaml @@ -0,0 +1,119 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E1E\u0E27\u0E01\u0E21\u0E31\u0E19\u0E21\u0E35\u0E42\u0E04\u0E23\u0E07\ + \u0E01\u0E23\u0E30\u0E14\u0E39\u0E01\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01\u0E17\ + \u0E35\u0E48\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E40\ + \u0E1E\u0E1B\u0E17\u0E34\u0E42\u0E14\u0E44\u0E01\u0E25\u0E41\u0E04\u0E19\u0E40\ + \u0E1B\u0E47\u0E19\u0E2B\u0E25\u0E31\u0E01" + B: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E21\u0E35\u0E23\u0E30\u0E1A\u0E1A\ + \u0E44\u0E2B\u0E25\u0E40\u0E27\u0E35\u0E22\u0E19\u0E42\u0E25\u0E2B\u0E34\u0E15\ + \u0E40\u0E1B\u0E34\u0E14\u0E01\u0E31\u0E1A\u0E2B\u0E31\u0E27\u0E43\u0E08\u0E2B\ + \u0E25\u0E31\u0E07" + C: "\u0E1E\u0E27\u0E01\u0E21\u0E31\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E21\ + \u0E32\u0E0A\u0E34\u0E01\u0E02\u0E2D\u0E07\u0E44\u0E1F\u0E25\u0E31\u0E21\u0E17\ + \u0E35\u0E48\u0E44\u0E21\u0E48\u0E1B\u0E23\u0E30\u0E2A\u0E1A\u0E04\u0E27\u0E32\ + \u0E21\u0E2A\u0E33\u0E40\u0E23\u0E47\u0E08\u0E17\u0E32\u0E07\u0E0A\u0E35\u0E27\ + \u0E27\u0E34\u0E17\u0E22\u0E32\u0E0B\u0E36\u0E48\u0E07\u0E44\u0E21\u0E48\u0E2A\ + \u0E32\u0E21\u0E32\u0E23\u0E16\u0E43\u0E0A\u0E49\u0E1B\u0E23\u0E30\u0E42\u0E22\ + \u0E0A\u0E19\u0E4C\u0E08\u0E32\u0E01\u0E41\u0E2B\u0E25\u0E48\u0E07\u0E17\u0E35\ + \u0E48\u0E2D\u0E22\u0E39\u0E48\u0E2D\u0E32\u0E28\u0E31\u0E22\u0E41\u0E25\u0E30\ + \u0E41\u0E2B\u0E25\u0E48\u0E07\u0E2D\u0E32\u0E2B\u0E32\u0E23\u0E17\u0E35\u0E48\ + \u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\u0E44\u0E14\u0E49" + D: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E44\u0E21\u0E48\u0E21\u0E35\u0E2D\ + \u0E27\u0E31\u0E22\u0E27\u0E30\u0E17\u0E35\u0E48\u0E40\u0E0A\u0E37\u0E48\u0E2D\ + \u0E21\u0E15\u0E48\u0E2D\u0E01\u0E31\u0E19" + input_correct_responses: + - B + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E41\u0E2A\u0E14\u0E07\u0E16\u0E36\u0E07\u0E04\u0E33\u0E01\u0E25\ + \u0E48\u0E32\u0E27\u0E17\u0E35\u0E48\u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07\ + \u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2A\u0E31\u0E15\u0E27\ + \u0E4C\u0E02\u0E32\u0E1B\u0E25\u0E49\u0E2D\u0E07" + - input_choice_list: + A: 1/400 + B: 19/400 + C: 20/400 + D: 38/400 + input_correct_responses: + - D + input_question: "\u0E43\u0E19\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E17\u0E35\ + \u0E48\u0E01\u0E33\u0E2B\u0E19\u0E14 1 \u0E43\u0E19 400 \u0E04\u0E19\u0E40\u0E1B\ + \u0E47\u0E19\u0E21\u0E30\u0E40\u0E23\u0E47\u0E07\u0E17\u0E35\u0E48\u0E40\u0E01\ + \u0E34\u0E14\u0E08\u0E32\u0E01\u0E2D\u0E31\u0E25\u0E25\u0E35\u0E25\u0E17\u0E35\ + \u0E48\u0E14\u0E49\u0E2D\u0E22\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 \u0E02\ + . \u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32\u0E1B\u0E23\u0E30\u0E0A\u0E32\ + \u0E01\u0E23\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E20\u0E32\u0E27\u0E30\u0E2A\ + \u0E21\u0E14\u0E38\u0E25\u0E02\u0E2D\u0E07\u0E2E\u0E32\u0E23\u0E4C\u0E14\u0E35\ + -\u0E44\u0E27\u0E19\u0E4C\u0E40\u0E1A\u0E34\u0E23\u0E4C\u0E01 \u0E02\u0E49\u0E2D\ + \u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E04\u0E37\u0E2D\ + \u0E2A\u0E31\u0E14\u0E2A\u0E48\u0E27\u0E19\u0E17\u0E35\u0E48\u0E04\u0E32\u0E14\ + \u0E2B\u0E27\u0E31\u0E07\u0E02\u0E2D\u0E07\u0E1A\u0E38\u0E04\u0E04\u0E25\u0E17\ + \u0E35\u0E48\u0E21\u0E35\u0E2D\u0E31\u0E25\u0E25\u0E35\u0E25 b \u0E41\u0E15\u0E48\ + \u0E44\u0E21\u0E48\u0E04\u0E32\u0E14\u0E27\u0E48\u0E32\u0E08\u0E30\u0E1E\u0E31\ + \u0E12\u0E19\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E21\u0E30\u0E40\u0E23\u0E47\u0E07" + - input_choice_list: + A: "\u0E21\u0E19\u0E38\u0E29\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E19\u0E01\u0E40\ + \u0E1B\u0E47\u0E19\u0E2A\u0E31\u0E15\u0E27\u0E4C\u0E2B\u0E25\u0E32\u0E22\u0E2A\ + \u0E32\u0E22\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E4C" + B: "\u0E27\u0E34\u0E27\u0E31\u0E12\u0E19\u0E32\u0E01\u0E32\u0E23\u0E02\u0E2D\ + \u0E07\u0E21\u0E19\u0E38\u0E29\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E19\u0E01\u0E19\ + \u0E31\u0E49\u0E19\u0E21\u0E32\u0E1A\u0E23\u0E23\u0E08\u0E1A\u0E01\u0E31\u0E19" + C: "\u0E21\u0E19\u0E38\u0E29\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E19\u0E01\u0E40\ + \u0E1B\u0E47\u0E19\u0E02\u0E2D\u0E07 clade" + D: "\u0E21\u0E19\u0E38\u0E29\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E19\u0E01\u0E1E\ + \u0E31\u0E12\u0E19\u0E32\u0E02\u0E36\u0E49\u0E19\u0E42\u0E14\u0E22\u0E01\u0E32\ + \u0E23\u0E40\u0E1B\u0E23\u0E35\u0E22\u0E1A\u0E40\u0E17\u0E35\u0E22\u0E1A" + input_correct_responses: + - C + input_question: "\u0E01\u0E32\u0E23\u0E21\u0E35\u0E42\u0E04\u0E23\u0E07\u0E2A\u0E23\ + \u0E49\u0E32\u0E07\u0E04\u0E25\u0E49\u0E32\u0E22\u0E04\u0E25\u0E36\u0E07\u0E01\ + \u0E31\u0E19\u0E43\u0E19\u0E2A\u0E34\u0E48\u0E07\u0E21\u0E35\u0E0A\u0E35\u0E27\ + \u0E34\u0E15\u0E2A\u0E2D\u0E07\u0E0A\u0E19\u0E34\u0E14\u0E17\u0E35\u0E48\u0E41\ + \u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E01\u0E31\u0E19 \u0E40\u0E0A\u0E48\u0E19\ + \ \u0E01\u0E23\u0E30\u0E14\u0E39\u0E01\u0E15\u0E49\u0E19\u0E41\u0E02\u0E19\u0E2A\ + \u0E48\u0E27\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E21\u0E19\u0E38\ + \u0E29\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E19\u0E01 \u0E1A\u0E48\u0E07\u0E0A\u0E35\ + \u0E49\u0E27\u0E48\u0E32" + - input_choice_list: + A: "\u0E1B\u0E31\u0E4A\u0E21\u0E44\u0E2B\u0E25\u0E41\u0E1A\u0E1A\u0E02\u0E36\ + \u0E49\u0E19\u0E2D\u0E22\u0E39\u0E48\u0E01\u0E31\u0E1A\u0E41\u0E23\u0E07\u0E14\ + \u0E31\u0E19 ATP" + B: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E48\u0E32\u0E07\u0E28\u0E31\u0E01\u0E22\ + \u0E4C\u0E41\u0E23\u0E07\u0E14\u0E31\u0E19\u0E19\u0E49\u0E33" + C: "\u0E01\u0E32\u0E23\u0E04\u0E32\u0E22\u0E19\u0E49\u0E33" + D: "\u0E01\u0E32\u0E23\u0E41\u0E1E\u0E23\u0E48\u0E01\u0E23\u0E30\u0E08\u0E32\ + \u0E22\u0E02\u0E2D\u0E07\u0E2D\u0E30\u0E42\u0E1E\u0E1E\u0E25\u0E32\u0E2A\u0E15\ + \u0E34\u0E01" + input_correct_responses: + - B + input_question: "\u0E15\u0E32\u0E21\u0E41\u0E1A\u0E1A\u0E08\u0E33\u0E25\u0E2D\u0E07\ + \u0E01\u0E32\u0E23\u0E44\u0E2B\u0E25\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\ + \u0E14\u0E31\u0E19\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E40\u0E04\u0E25\u0E37\ + \u0E48\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E02\u0E2D\u0E07\u0E40\u0E19\u0E37\u0E49\ + \u0E2D\u0E2B\u0E32\u0E43\u0E19\u0E42\u0E1F\u0E25\u0E40\u0E2D\u0E47\u0E21 \u0E01\ + \u0E32\u0E23\u0E40\u0E04\u0E25\u0E37\u0E48\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E02\ + \u0E2D\u0E07\u0E41\u0E2A\u0E07\u0E2A\u0E31\u0E07\u0E40\u0E04\u0E23\u0E32\u0E30\ + \u0E2B\u0E4C\u0E08\u0E32\u0E01\u0E41\u0E2B\u0E25\u0E48\u0E07\u0E01\u0E33\u0E40\ + \u0E19\u0E34\u0E14\u0E44\u0E1B\u0E22\u0E31\u0E07\u0E2D\u0E48\u0E32\u0E07\u0E25\ + \u0E49\u0E32\u0E07\u0E08\u0E32\u0E19\u0E19\u0E31\u0E49\u0E19\u0E02\u0E31\u0E1A\ + \u0E40\u0E04\u0E25\u0E37\u0E48\u0E2D\u0E19\u0E42\u0E14\u0E22" + - input_choice_list: + A: "\u0E40\u0E17\u0E42\u0E25\u0E40\u0E21\u0E35\u0E22\u0E23\u0E4C" + B: "\u0E40\u0E0B\u0E19\u0E42\u0E17\u0E23\u0E40\u0E21\u0E35\u0E22\u0E23\u0E4C" + C: "\u0E19\u0E34\u0E27\u0E04\u0E25\u0E35\u0E42\u0E2D\u0E42\u0E0B\u0E21" + D: "\u0E1B\u0E23\u0E30\u0E01\u0E1A\u0E01\u0E31\u0E19" + input_correct_responses: + - B + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E21\u0E35\u0E25\u0E33\u0E14\u0E31\u0E1A\u0E14\u0E35\u0E40\u0E2D\ + \u0E47\u0E19\u0E40\u0E2D\u0E17\u0E35\u0E48\u0E08\u0E33\u0E40\u0E1B\u0E47\u0E19\ + \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E41\u0E22\u0E01\u0E42\ + \u0E04\u0E23\u0E42\u0E21\u0E42\u0E0B\u0E21\u0E43\u0E19\u0E44\u0E21\u0E42\u0E17\ + \u0E0B\u0E34\u0E2A\u0E41\u0E25\u0E30\u0E44\u0E21\u0E42\u0E2D\u0E0B\u0E34\u0E2A" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_biology +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_college_biology +task_alias: college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65085d72eeaca0c6990a5d4c4fbbdd86ca78dfdd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_chemistry.yaml @@ -0,0 +1,95 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2A\u0E16\u0E32\u0E19\u0E30\u0E2D\u0E2D\u0E01\u0E0B\u0E34\u0E40\u0E14\ + \u0E0A\u0E31\u0E19\u0E17\u0E35\u0E48\u0E1E\u0E1A\u0E21\u0E32\u0E01\u0E17\u0E35\ + \u0E48\u0E2A\u0E38\u0E14\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E18\u0E32\u0E15\ + \u0E38\u0E41\u0E25\u0E19\u0E17\u0E32\u0E44\u0E19\u0E14\u0E4C\u0E04\u0E37\u0E2D\ + \ +3" + B: "\u0E04\u0E2D\u0E21\u0E40\u0E1E\u0E25\u0E47\u0E01\u0E0B\u0E4C\u0E41\u0E25\ + \u0E19\u0E17\u0E32\u0E44\u0E19\u0E14\u0E4C\u0E21\u0E31\u0E01\u0E21\u0E35\u0E08\ + \u0E33\u0E19\u0E27\u0E19\u0E42\u0E04\u0E2D\u0E2D\u0E23\u0E4C\u0E14\u0E34\u0E40\ + \u0E19\u0E0A\u0E31\u0E19\u0E2A\u0E39\u0E07 (> 6)" + C: "\u0E18\u0E32\u0E15\u0E38\u0E41\u0E25\u0E19\u0E17\u0E32\u0E44\u0E19\u0E14\ + \u0E4C\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\u0E17\u0E33\u0E1B\u0E0F\u0E34\ + \u0E01\u0E34\u0E23\u0E34\u0E22\u0E32\u0E01\u0E31\u0E1A\u0E01\u0E23\u0E14\u0E43\ + \u0E19\u0E19\u0E49\u0E33\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E1B\u0E25\u0E14\u0E1B\ + \u0E25\u0E48\u0E2D\u0E22\u0E44\u0E2E\u0E42\u0E14\u0E23\u0E40\u0E08\u0E19" + D: "\u0E23\u0E31\u0E28\u0E21\u0E35\u0E2D\u0E30\u0E15\u0E2D\u0E21\u0E02\u0E2D\ + \u0E07\u0E18\u0E32\u0E15\u0E38\u0E41\u0E25\u0E19\u0E17\u0E32\u0E44\u0E19\u0E14\ + \u0E4C\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E15\u0E25\u0E2D\ + \u0E14\u0E0A\u0E48\u0E27\u0E07\u0E40\u0E27\u0E25\u0E32\u0E08\u0E32\u0E01 La\ + \ \u0E16\u0E36\u0E07 Lu" + input_correct_responses: + - D + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E01\u0E25\u0E48\u0E32\u0E27\u0E16\ + \u0E36\u0E07\u0E18\u0E32\u0E15\u0E38\u0E41\u0E25\u0E19\u0E17\u0E32\u0E44\u0E19\ + \u0E14\u0E4C\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E44\u0E21\u0E48\ + \u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07" + - input_choice_list: + A: "1.0 \u0E21\u0E25" + B: "10 \u0E21\u0E25" + C: "20 \u0E21\u0E25" + D: "50 \u0E21\u0E25" + input_correct_responses: + - C + input_question: "\u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E02\u0E2D\u0E07\ + \ HgO 0.217 \u0E01\u0E23\u0E31\u0E21 (\u0E21\u0E27\u0E25\u0E42\u0E21\u0E25\u0E32\ + \u0E23\u0E4C = 217 \u0E01\u0E23\u0E31\u0E21) \u0E17\u0E33\u0E1B\u0E0F\u0E34\u0E01\ + \u0E34\u0E23\u0E34\u0E22\u0E32\u0E01\u0E31\u0E1A\u0E44\u0E2D\u0E2D\u0E2D\u0E19\ + \u0E44\u0E2D\u0E42\u0E2D\u0E44\u0E14\u0E14\u0E4C\u0E2A\u0E48\u0E27\u0E19\u0E40\ + \u0E01\u0E34\u0E19\u0E15\u0E32\u0E21\u0E1B\u0E0F\u0E34\u0E01\u0E34\u0E23\u0E34\ + \u0E22\u0E32\u0E17\u0E35\u0E48\u0E41\u0E2A\u0E14\u0E07\u0E14\u0E49\u0E32\u0E19\ + \u0E1A\u0E19 \u0E01\u0E32\u0E23\u0E44\u0E17\u0E40\u0E17\u0E23\u0E15\u0E02\u0E2D\ + \u0E07\u0E2A\u0E32\u0E23\u0E25\u0E30\u0E25\u0E32\u0E22\u0E17\u0E35\u0E48\u0E44\ + \u0E14\u0E49\u0E15\u0E49\u0E2D\u0E07\u0E43\u0E0A\u0E49 HCl 0.10 M \u0E01\u0E35\ + \u0E48\u0E21\u0E25. \u0E08\u0E36\u0E07\u0E08\u0E30\u0E16\u0E36\u0E07\u0E08\u0E38\ + \u0E14\u0E2A\u0E21\u0E21\u0E39\u0E25" + - input_choice_list: + A: '4' + B: '3' + C: '6' + D: '24' + input_correct_responses: + - A + input_question: "\u0E17\u0E33\u0E19\u0E32\u0E22\u0E08\u0E33\u0E19\u0E27\u0E19\u0E40\ + \u0E2A\u0E49\u0E19\u0E43\u0E19\u0E2A\u0E40\u0E1B\u0E01\u0E15\u0E23\u0E31\u0E21\ + \ EPR \u0E02\u0E2D\u0E07\u0E2A\u0E32\u0E23\u0E25\u0E30\u0E25\u0E32\u0E22\u0E40\ + \u0E21\u0E17\u0E34\u0E25\u0E41\u0E23\u0E14\u0E34\u0E04\u0E31\u0E25\u0E17\u0E35\ + \u0E48\u0E21\u0E35\u0E09\u0E25\u0E32\u0E01 13C (13CH3\u2022) \u0E42\u0E14\u0E22\ + \u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32\u0E40\u0E2A\u0E49\u0E19\u0E44\ + \u0E21\u0E48\u0E17\u0E31\u0E1A\u0E0B\u0E49\u0E2D\u0E19\u0E01\u0E31\u0E19" + - input_choice_list: + A: "\u0E01\u0E23\u0E14" + B: "\u0E10\u0E32\u0E19" + C: "\u0E15\u0E31\u0E27\u0E40\u0E23\u0E48\u0E07\u0E1B\u0E0F\u0E34\u0E01\u0E34\ + \u0E23\u0E34\u0E22\u0E32" + D: "\u0E40\u0E1B\u0E47\u0E19\u0E15\u0E31\u0E27\u0E23\u0E35\u0E14\u0E34\u0E27\ + \u0E0B\u0E4C" + input_correct_responses: + - D + input_question: "3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq)\ + \ + 4 Cr3+(aq) + 10 H2O(l) \u0E43\u0E19\u0E1B\u0E0F\u0E34\u0E01\u0E34\u0E23\u0E34\ + \u0E22\u0E32\u0E17\u0E35\u0E48\u0E41\u0E2A\u0E14\u0E07\u0E02\u0E49\u0E32\u0E07\ + \u0E15\u0E49\u0E19 Cl\u2212(aq) \u0E08\u0E30\u0E17\u0E33\u0E2B\u0E19\u0E49\u0E32\ + \u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19" + - input_choice_list: + A: PbH4 < SnH4 < GeH4 < SiH4 < CH4 + B: PbH4 < SnH4 < CH4 < GeH4 < SiH4 + C: CH4 < SiH4 < GeH4 < SnH4 < PbH4 + D: CH4 < PbH4 < GeH4 < SnH4 < SiH4 + input_correct_responses: + - A + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E41\u0E2A\u0E14\u0E07\u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\u0E44\ + \u0E2E\u0E44\u0E14\u0E23\u0E14\u0E4C\u0E02\u0E2D\u0E07\u0E18\u0E32\u0E15\u0E38\ + \u0E2B\u0E21\u0E39\u0E48 14 \u0E15\u0E32\u0E21\u0E25\u0E33\u0E14\u0E31\u0E1A\ + \u0E04\u0E27\u0E32\u0E21\u0E40\u0E2A\u0E16\u0E35\u0E22\u0E23\u0E17\u0E32\u0E07\ + \u0E04\u0E27\u0E32\u0E21\u0E23\u0E49\u0E2D\u0E19 \u0E08\u0E32\u0E01\u0E15\u0E48\ + \u0E33\u0E2A\u0E38\u0E14\u0E44\u0E1B\u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_chemistry +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_college_chemistry +task_alias: college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..664ebecbd3ff6ce82221dcb67bd284f9220f25d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_computer_science.yaml @@ -0,0 +1,184 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E01*(\u0E04 + \u0E07)+ \u0E02(\u0E04 + \u0E07)" + B: "\u0E01*(\u0E04 + \u0E07)* + \u0E02(\u0E04 + \u0E07)*" + C: "\u0E01*(\u0E04 + \u0E07)+ \u0E02*(\u0E04 + \u0E07)" + D: "(\u0E01 + \u0E02)*\u0E04 +(\u0E01 + \u0E02)*\u0E07" + input_correct_responses: + - D + input_question: "\u0E19\u0E34\u0E1E\u0E08\u0E19\u0E4C\u0E17\u0E31\u0E48\u0E27\u0E44\ + \u0E1B\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E17\ + \u0E35\u0E22\u0E1A\u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A (\u0E2D\u0E18\u0E34\ + \u0E1A\u0E32\u0E22\u0E2A\u0E15\u0E23\u0E34\u0E07\u0E0A\u0E38\u0E14\u0E40\u0E14\ + \u0E35\u0E22\u0E27\u0E01\u0E31\u0E19\u0E01\u0E31\u0E1A) (a* + b)*(c + d)" + - input_choice_list: + A: '5' + B: '6' + C: '7' + D: '8' + input_correct_responses: + - B + input_question: "\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07 RISC \u0E41\u0E1A\u0E1A\ + \u0E44\u0E1B\u0E1B\u0E4C\u0E44\u0E25\u0E19\u0E4C\u0E1A\u0E32\u0E07\u0E40\u0E04\ + \u0E23\u0E37\u0E48\u0E2D\u0E07\u0E21\u0E35\u0E23\u0E35\u0E08\u0E34\u0E2A\u0E40\ + \u0E15\u0E2D\u0E23\u0E4C\u0E2D\u0E40\u0E19\u0E01\u0E1B\u0E23\u0E30\u0E2A\u0E07\ + \u0E04\u0E4C 8 \u0E15\u0E31\u0E27 R0, R1, . . . , R7 \u0E41\u0E25\u0E30\u0E23\ + \u0E2D\u0E07\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E17\u0E33\u0E07\u0E32\u0E19\ + \u0E14\u0E31\u0E07\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E40\u0E1E\ + \u0E34\u0E48\u0E21 Rs1, Rs2, Rd \u0E40\u0E1E\u0E34\u0E48\u0E21 Rs1 \u0E16\u0E36\ + \u0E07 Rs2 \u0E41\u0E25\u0E30\u0E43\u0E2A\u0E48\u0E1C\u0E25\u0E23\u0E27\u0E21\ + \u0E43\u0E19 Rd MUL Rs1, Rs2, Rd \u0E04\u0E39\u0E13 Rs1 \u0E14\u0E49\u0E27\u0E22\ + \ Rs2 \u0E41\u0E25\u0E30\u0E43\u0E2A\u0E48\u0E1C\u0E25\u0E04\u0E39\u0E13\u0E43\ + \u0E19 Rd \u0E42\u0E14\u0E22\u0E1B\u0E01\u0E15\u0E34\u0E01\u0E32\u0E23\u0E14\ + \u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E08\u0E30\u0E43\u0E0A\u0E49\ + \u0E40\u0E27\u0E25\u0E32\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E23\u0E2D\u0E1A \u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E44\u0E23\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E01\u0E32\ + \u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E08\u0E30\u0E43\ + \u0E0A\u0E49\u0E40\u0E27\u0E25\u0E32\u0E2A\u0E2D\u0E07\u0E23\u0E2D\u0E1A\u0E2B\ + \u0E32\u0E01\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E1C\u0E25\u0E25\u0E31\u0E1E\u0E18\ + \u0E4C\u0E17\u0E35\u0E48\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E42\u0E14\ + \u0E22\u0E01\u0E32\u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E17\u0E31\u0E19\u0E17\u0E35\ + \u0E43\u0E19\u0E25\u0E33\u0E14\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E14\u0E33\u0E40\ + \u0E19\u0E34\u0E19\u0E01\u0E32\u0E23 \u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\ + \u0E19\u0E34\u0E1E\u0E08\u0E19\u0E4C AB + ABC + BC \u0E42\u0E14\u0E22\u0E17\u0E35\ + \u0E48\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23 A, B, C \u0E2D\u0E22\u0E39\u0E48\u0E43\ + \u0E19\u0E23\u0E35\u0E08\u0E34\u0E2A\u0E40\u0E15\u0E2D\u0E23\u0E4C R0, R1, R2\ + \ \u0E2B\u0E32\u0E01\u0E44\u0E21\u0E48\u0E15\u0E49\u0E2D\u0E07\u0E41\u0E01\u0E49\ + \u0E44\u0E02\u0E40\u0E19\u0E37\u0E49\u0E2D\u0E2B\u0E32\u0E02\u0E2D\u0E07\u0E23\ + \u0E35\u0E08\u0E34\u0E2A\u0E40\u0E15\u0E2D\u0E23\u0E4C\u0E17\u0E31\u0E49\u0E07\ + \u0E2A\u0E32\u0E21\u0E19\u0E35\u0E49 \u0E08\u0E33\u0E19\u0E27\u0E19\u0E23\u0E2D\ + \u0E1A\u0E2A\u0E31\u0E0D\u0E0D\u0E32\u0E13\u0E19\u0E32\u0E2C\u0E34\u0E01\u0E32\ + \u0E02\u0E31\u0E49\u0E19\u0E15\u0E48\u0E33\u0E17\u0E35\u0E48\u0E08\u0E33\u0E40\ + \u0E1B\u0E47\u0E19\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E25\u0E33\u0E14\u0E31\ + \u0E1A\u0E01\u0E32\u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\ + \u0E17\u0E35\u0E48\u0E04\u0E33\u0E19\u0E27\u0E13\u0E04\u0E48\u0E32\u0E02\u0E2D\ + \u0E07 AB + ABC + BC \u0E04\u0E37\u0E2D\u0E40\u0E17\u0E48\u0E32\u0E43\u0E14" + - input_choice_list: + A: "\u0E09\u0E31\u0E19\u0E04\u0E19\u0E40\u0E14\u0E35\u0E22\u0E27" + B: "\u0E04\u0E23\u0E31\u0E49\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07\u0E40\ + \u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + C: "III \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + D: "\u0E09\u0E31\u0E19, II \u0E41\u0E25\u0E30 III" + input_correct_responses: + - D + input_question: "\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E01\u0E32\u0E23\u0E2D\u0E2D\ + \u0E01\u0E41\u0E1A\u0E1A Singleton \u0E43\u0E0A\u0E49\u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E23\u0E31\u0E1A\u0E1B\u0E23\u0E30\u0E01\u0E31\u0E19\u0E27\u0E48\u0E32\ + \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E2D\u0E34\ + \u0E19\u0E2A\u0E41\u0E15\u0E19\u0E0B\u0E4C\u0E02\u0E2D\u0E07\u0E04\u0E25\u0E32\ + \u0E2A\u0E44\u0E14\u0E49\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E2D\u0E34\u0E19\u0E2A\ + \u0E41\u0E15\u0E19\u0E0B\u0E4C\u0E40\u0E14\u0E35\u0E22\u0E27\u0E40\u0E17\u0E48\ + \u0E32\u0E19\u0E31\u0E49\u0E19 \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\ + \u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07\ + \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E01\ + \u0E32\u0E23\u0E2D\u0E2D\u0E01\u0E41\u0E1A\u0E1A\u0E19\u0E35\u0E49 I. \u0E04\ + \u0E25\u0E32\u0E2A Singleton \u0E21\u0E35\u0E40\u0E21\u0E18\u0E2D\u0E14\u0E42\ + \u0E23\u0E07\u0E07\u0E32\u0E19\u0E41\u0E1A\u0E1A\u0E2A\u0E41\u0E15\u0E15\u0E34\ + \u0E01\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E08\u0E31\u0E14\u0E40\u0E15\u0E23\u0E35\ + \u0E22\u0E21\u0E2D\u0E34\u0E19\u0E2A\u0E41\u0E15\u0E19\u0E0B\u0E4C \u0E04\u0E23\ + \u0E31\u0E49\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07 \u0E04\u0E25\u0E32\u0E2A\ + \ Singleton \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E40\u0E1B\u0E47\u0E19\u0E04\ + \u0E25\u0E32\u0E2A\u0E22\u0E48\u0E2D\u0E22\u0E02\u0E2D\u0E07\u0E04\u0E25\u0E32\ + \u0E2A\u0E2D\u0E37\u0E48\u0E19\u0E44\u0E14\u0E49 \u0E2A\u0E32\u0E21. \u0E04\u0E25\ + \u0E32\u0E2A Singleton \u0E21\u0E35\u0E15\u0E31\u0E27\u0E2A\u0E23\u0E49\u0E32\ + \u0E07\u0E2A\u0E48\u0E27\u0E19\u0E15\u0E31\u0E27" + - input_choice_list: + A: '5' + B: '6' + C: '7' + D: '9' + input_correct_responses: + - D + input_question: "\u0E04\u0E2D\u0E21\u0E44\u0E1E\u0E40\u0E25\u0E2D\u0E23\u0E4C\u0E2A\ + \u0E23\u0E49\u0E32\u0E07\u0E23\u0E2B\u0E31\u0E2A\u0E2A\u0E33\u0E2B\u0E23\u0E31\ + \u0E1A\u0E04\u0E33\u0E2A\u0E31\u0E48\u0E07\u0E21\u0E2D\u0E1A\u0E2B\u0E21\u0E32\ + \u0E22\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 G := (A + B) * C - (D\ + \ + E) * F \u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E40\u0E1B\u0E49\u0E32\ + \u0E2B\u0E21\u0E32\u0E22\u0E21\u0E35\u0E0A\u0E38\u0E14\u0E04\u0E33\u0E2A\u0E31\ + \u0E48\u0E07\u0E2A\u0E30\u0E2A\u0E21\u0E40\u0E14\u0E35\u0E22\u0E27\u0E41\u0E25\ + \u0E30\u0E17\u0E35\u0E48\u0E2D\u0E22\u0E39\u0E48\u0E40\u0E14\u0E35\u0E22\u0E27\ + \u0E0B\u0E36\u0E48\u0E07\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\ + \u0E22\u0E04\u0E33\u0E2A\u0E31\u0E48\u0E07\u0E42\u0E2B\u0E25\u0E14 \u0E08\u0E31\ + \u0E14\u0E40\u0E01\u0E47\u0E1A \u0E1A\u0E27\u0E01 \u0E25\u0E1A \u0E41\u0E25\u0E30\ + \u0E04\u0E39\u0E13 \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E14\ + \u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E17\u0E32\u0E07\u0E04\u0E13\ + \u0E34\u0E15\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C \u0E15\u0E31\u0E27\u0E16\u0E39\ + \u0E01\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E14\u0E49\u0E32\ + \u0E19\u0E0B\u0E49\u0E32\u0E22\u0E08\u0E30\u0E16\u0E39\u0E01\u0E19\u0E33\u0E21\ + \u0E32\u0E08\u0E32\u0E01\u0E15\u0E31\u0E27\u0E2A\u0E30\u0E2A\u0E21\u0E41\u0E25\ + \u0E30\u0E1C\u0E25\u0E25\u0E31\u0E1E\u0E18\u0E4C\u0E08\u0E30\u0E1B\u0E23\u0E32\ + \u0E01\u0E0F\u0E43\u0E19\u0E15\u0E31\u0E27\u0E2A\u0E30\u0E2A\u0E21 \u0E08\u0E33\ + \u0E19\u0E27\u0E19\u0E04\u0E33\u0E2A\u0E31\u0E48\u0E07\u0E17\u0E35\u0E48\u0E19\ + \u0E49\u0E2D\u0E22\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E17\u0E35\u0E48\u0E40\ + \u0E1B\u0E47\u0E19\u0E44\u0E1B\u0E44\u0E14\u0E49\u0E43\u0E19\u0E23\u0E2B\u0E31\ + \u0E2A\u0E1C\u0E25\u0E25\u0E31\u0E1E\u0E18\u0E4C\u0E04\u0E37\u0E2D" + - input_choice_list: + A: 1/50 + B: 1/27 + C: 1/25 + D: 2/27 + input_correct_responses: + - B + input_question: "\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\u0E01\u0E32\u0E23\u0E2D\ + \u0E2D\u0E01\u0E41\u0E1A\u0E1A\u0E04\u0E2D\u0E21\u0E1E\u0E34\u0E27\u0E40\u0E15\ + \u0E2D\u0E23\u0E4C\u0E17\u0E35\u0E48\u0E42\u0E1B\u0E23\u0E40\u0E0B\u0E2A\u0E40\ + \u0E0B\u0E2D\u0E23\u0E4C\u0E2B\u0E25\u0E32\u0E22\u0E15\u0E31\u0E27 \u0E41\u0E15\ + \u0E48\u0E25\u0E30\u0E15\u0E31\u0E27\u0E21\u0E35\u0E2B\u0E19\u0E48\u0E27\u0E22\ + \u0E04\u0E27\u0E32\u0E21\u0E08\u0E33\u0E41\u0E04\u0E0A\u0E2A\u0E48\u0E27\u0E19\ + \u0E15\u0E31\u0E27 \u0E43\u0E0A\u0E49\u0E2B\u0E19\u0E48\u0E27\u0E22\u0E04\u0E27\ + \u0E32\u0E21\u0E08\u0E33\u0E2A\u0E48\u0E27\u0E19\u0E01\u0E25\u0E32\u0E07\u0E23\ + \u0E48\u0E27\u0E21\u0E01\u0E31\u0E19\u0E42\u0E14\u0E22\u0E43\u0E0A\u0E49\u0E1A\ + \u0E31\u0E2A\u0E40\u0E14\u0E35\u0E22\u0E27 \u0E1A\u0E31\u0E2A\u0E19\u0E35\u0E49\ + \u0E40\u0E1B\u0E47\u0E19\u0E17\u0E23\u0E31\u0E1E\u0E22\u0E32\u0E01\u0E23\u0E23\ + \u0E30\u0E1A\u0E1A\u0E17\u0E35\u0E48\u0E2A\u0E33\u0E04\u0E31\u0E0D \u0E42\u0E1B\ + \u0E23\u0E40\u0E0B\u0E2A\u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E41\u0E15\u0E48\u0E25\ + \u0E30\u0E15\u0E31\u0E27\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E14\u0E33\u0E40\ + \u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E04\u0E33\ + \u0E2A\u0E31\u0E48\u0E07\u0E17\u0E38\u0E01\u0E46 500 \u0E19\u0E32\u0E42\u0E19\ + \u0E27\u0E34\u0E19\u0E32\u0E17\u0E35 \u0E15\u0E23\u0E32\u0E1A\u0E43\u0E14\u0E17\ + \u0E35\u0E48\u0E41\u0E04\u0E0A\u0E43\u0E19\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\ + \u0E07\u0E23\u0E2D\u0E07\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E2B\u0E19\u0E48\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\ + \u0E08\u0E33 \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E41\u0E04\u0E0A\u0E2B\u0E32\u0E22\ + \u0E44\u0E1B \u0E42\u0E1B\u0E23\u0E40\u0E0B\u0E2A\u0E40\u0E0B\u0E2D\u0E23\u0E4C\ + \u0E08\u0E30\u0E25\u0E48\u0E32\u0E0A\u0E49\u0E32\u0E40\u0E1E\u0E34\u0E48\u0E21\ + \u0E2D\u0E35\u0E01 2,000 \u0E19\u0E32\u0E42\u0E19\u0E27\u0E34\u0E19\u0E32\u0E17\ + \u0E35 \u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E04\u0E23\u0E36\u0E48\u0E07\u0E2B\ + \u0E19\u0E36\u0E48\u0E07\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\u0E25\u0E48\ + \u0E32\u0E0A\u0E49\u0E32\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\ + \u0E36\u0E49\u0E19\u0E19\u0E35\u0E49 \u0E1A\u0E31\u0E2A\u0E08\u0E30\u0E43\u0E2B\ + \u0E49\u0E1A\u0E23\u0E34\u0E01\u0E32\u0E23\u0E41\u0E04\u0E0A\u0E21\u0E34\u0E2A\ + \u0E42\u0E14\u0E22\u0E40\u0E09\u0E1E\u0E32\u0E30 \u0E43\u0E19\u0E2D\u0E35\u0E01\ + \u0E04\u0E23\u0E36\u0E48\u0E07\u0E2B\u0E19\u0E36\u0E48\u0E07 \u0E42\u0E1B\u0E23\ + \u0E40\u0E0B\u0E2A\u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E44\u0E21\u0E48\u0E2A\u0E32\ + \u0E21\u0E32\u0E23\u0E16\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\ + \u0E15\u0E48\u0E2D\u0E44\u0E14\u0E49 \u0E41\u0E15\u0E48\u0E1A\u0E31\u0E2A\u0E2A\ + \u0E32\u0E21\u0E32\u0E23\u0E16\u0E43\u0E2B\u0E49\u0E1A\u0E23\u0E34\u0E01\u0E32\ + \u0E23\u0E15\u0E32\u0E21\u0E04\u0E33\u0E02\u0E2D\u0E08\u0E32\u0E01\u0E42\u0E1B\ + \u0E23\u0E40\u0E0B\u0E2A\u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E2D\u0E37\u0E48\u0E19\ + \u0E44\u0E14\u0E49\u0E1F\u0E23\u0E35 \u0E42\u0E14\u0E22\u0E40\u0E09\u0E25\u0E35\ + \u0E48\u0E22\u0E41\u0E25\u0E49\u0E27 \u0E41\u0E15\u0E48\u0E25\u0E30\u0E04\u0E33\ + \u0E2A\u0E31\u0E48\u0E07\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E01\u0E32\ + \u0E23\u0E2D\u0E49\u0E32\u0E07\u0E2D\u0E34\u0E07\u0E2B\u0E19\u0E48\u0E27\u0E22\ + \u0E04\u0E27\u0E32\u0E21\u0E08\u0E33 2 \u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\ + \ \u0E42\u0E14\u0E22\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22\u0E41\u0E25\u0E49\u0E27\ + \ \u0E01\u0E32\u0E23\u0E1E\u0E25\u0E32\u0E14\u0E41\u0E04\u0E0A\u0E40\u0E01\u0E34\ + \u0E14\u0E02\u0E36\u0E49\u0E19 1 \u0E40\u0E1B\u0E2D\u0E23\u0E4C\u0E40\u0E0B\u0E47\ + \u0E19\u0E15\u0E4C\u0E02\u0E2D\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E2D\ + \u0E49\u0E32\u0E07\u0E2D\u0E34\u0E07 \u0E42\u0E1B\u0E23\u0E40\u0E0B\u0E2A\u0E40\ + \u0E0B\u0E2D\u0E23\u0E4C\u0E15\u0E31\u0E27\u0E40\u0E14\u0E35\u0E22\u0E27\u0E08\ + \u0E30\u0E43\u0E0A\u0E49\u0E04\u0E27\u0E32\u0E21\u0E08\u0E38\u0E02\u0E2D\u0E07\ + \u0E1A\u0E31\u0E2A\u0E43\u0E19\u0E2A\u0E31\u0E14\u0E2A\u0E48\u0E27\u0E19\u0E40\ + \u0E17\u0E48\u0E32\u0E43\u0E14 \u0E42\u0E14\u0E22\u0E44\u0E21\u0E48\u0E2A\u0E19\ + \u0E43\u0E08\u0E04\u0E27\u0E32\u0E21\u0E25\u0E48\u0E32\u0E0A\u0E49\u0E32\u0E40\ + \u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E41\u0E02\ + \u0E48\u0E07\u0E02\u0E31\u0E19\u0E08\u0E32\u0E01\u0E42\u0E1B\u0E23\u0E40\u0E0B\ + \u0E2A\u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E2D\u0E37\u0E48\u0E19" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_computer_science +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_college_computer_science +task_alias: college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e7b44338ebdbe447286a5967d94bedae4b64d6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_mathematics.yaml @@ -0,0 +1,107 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: ST = 0 + B: ST = T + C: "ST = \u0E17\u0E2A" + D: "ST - TS \u0E40\u0E1B\u0E47\u0E19\u0E41\u0E1C\u0E19\u0E17\u0E35\u0E48\u0E1B\ + \u0E23\u0E30\u0E08\u0E33\u0E15\u0E31\u0E27\u0E02\u0E2D\u0E07 V \u0E40\u0E02\ + \u0E49\u0E32\u0E2A\u0E39\u0E48\u0E15\u0E31\u0E27\u0E21\u0E31\u0E19\u0E40\u0E2D\ + \u0E07" + input_correct_responses: + - D + input_question: "\u0E43\u0E2B\u0E49 V \u0E40\u0E1B\u0E47\u0E19\u0E40\u0E0B\u0E15\ + \u0E02\u0E2D\u0E07\u0E1E\u0E2B\u0E38\u0E19\u0E32\u0E21 p(x) \u0E08\u0E33\u0E19\ + \u0E27\u0E19\u0E08\u0E23\u0E34\u0E07\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\ + \ \u0E43\u0E2B\u0E49\u0E01\u0E32\u0E23\u0E41\u0E1B\u0E25\u0E07 T, S \u0E16\u0E39\ + \u0E01\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E1A\u0E19 V \u0E42\u0E14\u0E22 T:p(x)\ + \ -> xp(x) \u0E41\u0E25\u0E30 S:p(x) -> p'(x) = d/dx p(x) \u0E41\u0E25\ + \u0E30\u0E15\u0E35\u0E04\u0E27\u0E32\u0E21 (ST) (p(x)) \u0E40\u0E1B\u0E47\u0E19\ + \ S(T(p(x))) \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07" + - input_choice_list: + A: '2' + B: "2 - \u0E2D\u0E35^-2" + C: 2 + e^-2 + D: 2 + e^-4 + input_correct_responses: + - D + input_question: "\u0E43\u0E19\u0E02\u0E31\u0E49\u0E19\u0E15\u0E49\u0E19\u0E16\u0E31\ + \u0E07\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E2A\u0E32\ + \u0E23\u0E25\u0E30\u0E25\u0E32\u0E22\u0E40\u0E01\u0E25\u0E37\u0E2D 3 \u0E01\u0E23\ + \u0E31\u0E21\u0E17\u0E35\u0E48\u0E25\u0E30\u0E25\u0E32\u0E22\u0E43\u0E19\u0E19\ + \u0E49\u0E33 100 \u0E25\u0E34\u0E15\u0E23 \u0E2A\u0E32\u0E23\u0E25\u0E30\u0E25\ + \u0E32\u0E22\u0E40\u0E01\u0E25\u0E37\u0E2D\u0E17\u0E35\u0E48\u0E21\u0E35\u0E40\ + \u0E01\u0E25\u0E37\u0E2D 0.02 \u0E01\u0E23\u0E31\u0E21\u0E15\u0E48\u0E2D\u0E19\ + \u0E49\u0E33\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E25\u0E34\u0E15\u0E23\u0E08\u0E30\ + \u0E16\u0E39\u0E01\u0E09\u0E35\u0E14\u0E1E\u0E48\u0E19\u0E25\u0E07\u0E43\u0E19\ + \u0E16\u0E31\u0E07\u0E43\u0E19\u0E2D\u0E31\u0E15\u0E23\u0E32 4 \u0E25\u0E34\u0E15\ + \u0E23\u0E15\u0E48\u0E2D\u0E19\u0E32\u0E17\u0E35 \u0E2A\u0E32\u0E23\u0E25\u0E30\ + \u0E25\u0E32\u0E22\u0E17\u0E35\u0E48\u0E09\u0E35\u0E14\u0E1E\u0E48\u0E19\u0E08\ + \u0E30\u0E1C\u0E2A\u0E21\u0E01\u0E31\u0E1A\u0E2A\u0E32\u0E23\u0E25\u0E30\u0E25\ + \u0E32\u0E22\u0E40\u0E01\u0E25\u0E37\u0E2D\u0E43\u0E19\u0E16\u0E31\u0E07\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E15\u0E48\u0E2D\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\ + \ \u0E41\u0E25\u0E30\u0E2A\u0E48\u0E27\u0E19\u0E1C\u0E2A\u0E21\u0E08\u0E30\u0E44\ + \u0E2B\u0E25\u0E2D\u0E2D\u0E01\u0E08\u0E32\u0E01\u0E16\u0E31\u0E07\u0E14\u0E49\ + \u0E27\u0E22\u0E2D\u0E31\u0E15\u0E23\u0E32 4 \u0E25\u0E34\u0E15\u0E23\u0E15\u0E48\ + \u0E2D\u0E19\u0E32\u0E17\u0E35 \u0E16\u0E49\u0E32\u0E1C\u0E2A\u0E21\u0E17\u0E31\ + \u0E19\u0E17\u0E35 100 \u0E19\u0E32\u0E17\u0E35\u0E1C\u0E48\u0E32\u0E19\u0E44\ + \u0E1B\u0E43\u0E19\u0E16\u0E31\u0E07\u0E21\u0E35\u0E40\u0E01\u0E25\u0E37\u0E2D\ + \u0E01\u0E35\u0E48\u0E01\u0E23\u0E31\u0E21" + - input_choice_list: + A: "\u0E09\u0E31\u0E19\u0E04\u0E19\u0E40\u0E14\u0E35\u0E22\u0E27" + B: "\u0E04\u0E23\u0E31\u0E49\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07\u0E40\ + \u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + C: "III \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + D: "II \u0E41\u0E25\u0E30 III \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + input_correct_responses: + - B + input_question: "\u0E43\u0E2B\u0E49 A \u0E40\u0E1B\u0E47\u0E19\u0E40\u0E21\u0E17\ + \u0E23\u0E34\u0E01\u0E0B\u0E4C\u0E02\u0E19\u0E32\u0E14 2x2 \u0E08\u0E23\u0E34\ + \u0E07 \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\ + \u0E49\u0E15\u0E49\u0E2D\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E27\u0E32\u0E21\ + \u0E08\u0E23\u0E34\u0E07 I. \u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\u0E17\u0E31\ + \u0E49\u0E07\u0E2B\u0E21\u0E14\u0E02\u0E2D\u0E07 A^2 \u0E44\u0E21\u0E48\u0E40\ + \u0E1B\u0E47\u0E19\u0E04\u0E48\u0E32\u0E25\u0E1A \u0E04\u0E23\u0E31\u0E49\u0E07\ + \u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07 \u0E14\u0E35\u0E40\u0E17\u0E2D\u0E23\u0E4C\ + \u0E21\u0E35\u0E41\u0E19\u0E19\u0E15\u0E4C\u0E02\u0E2D\u0E07 A^2 \u0E44\u0E21\ + \u0E48\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E48\u0E32\u0E25\u0E1A \u0E2A\u0E32\u0E21\ + . \u0E16\u0E49\u0E32 A \u0E21\u0E35\u0E04\u0E48\u0E32\u0E25\u0E31\u0E01\u0E29\ + \u0E13\u0E30\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E17\u0E35\u0E48\u0E41\u0E15\u0E01\ + \u0E15\u0E48\u0E32\u0E07\u0E01\u0E31\u0E19\u0E2A\u0E2D\u0E07\u0E04\u0E48\u0E32\ + \ \u0E14\u0E31\u0E07\u0E19\u0E31\u0E49\u0E19 A^2 \u0E08\u0E30\u0E21\u0E35\u0E04\ + \u0E48\u0E32\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\u0E40\u0E09\u0E1E\u0E32\u0E30\ + \u0E17\u0E35\u0E48\u0E41\u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E01\u0E31\u0E19\ + \u0E2A\u0E2D\u0E07\u0E04\u0E48\u0E32" + - input_choice_list: + A: '-11' + B: '0' + C: '11' + D: 33/2 + input_correct_responses: + - C + input_question: "\u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32 f(1 + x) = f(x)\ + \ \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A x \u0E08\u0E23\u0E34\u0E07\u0E17\u0E31\ + \u0E49\u0E07\u0E2B\u0E21\u0E14 \u0E16\u0E49\u0E32 f \u0E40\u0E1B\u0E47\u0E19\ + \u0E1E\u0E2B\u0E38\u0E19\u0E32\u0E21 \u0E41\u0E25\u0E30 f(5) = 11 \u0E41\u0E25\ + \u0E49\u0E27 f(15/2)" + - input_choice_list: + A: '-5' + B: '-4' + C: '-3' + D: '-2' + input_correct_responses: + - B + input_question: "\u0E43\u0E2B\u0E49 A \u0E40\u0E1B\u0E47\u0E19\u0E40\u0E0B\u0E15\ + \u0E02\u0E2D\u0E07\u0E08\u0E33\u0E19\u0E27\u0E19\u0E40\u0E15\u0E47\u0E21\u0E04\ + \u0E39\u0E48\u0E2D\u0E31\u0E19\u0E14\u0E31\u0E1A\u0E17\u0E31\u0E49\u0E07\u0E2B\ + \u0E21\u0E14 (m, n) \u0E0B\u0E36\u0E48\u0E07\u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\ + \u0E1A 7m + 12n = 22 \u0E08\u0E33\u0E19\u0E27\u0E19\u0E25\u0E1A\u0E17\u0E35\u0E48\ + \u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E43\u0E19\u0E40\u0E0B\ + \u0E15 B = {m + n : (m, n) \\in A} \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_mathematics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_college_mathematics +task_alias: college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3403ddc456ee883a51a4accb3b77eae70d2ad128 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_medicine.yaml @@ -0,0 +1,126 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E1C\u0E48\u0E32\u0E19\u0E15\u0E31\u0E27\u0E02\u0E19\u0E2A\u0E48\u0E07\ + \u0E42\u0E1B\u0E23\u0E15\u0E35\u0E19\u0E17\u0E35\u0E48\u0E40\u0E23\u0E35\u0E22\ + \u0E01\u0E27\u0E48\u0E32 GLUT4" + B: "\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E21\u0E35\ + \u0E2D\u0E34\u0E19\u0E0B\u0E39\u0E25\u0E34\u0E19" + C: "\u0E1C\u0E48\u0E32\u0E19\u0E40\u0E2E\u0E01\u0E42\u0E0B\u0E44\u0E04\u0E40\ + \u0E19\u0E2A" + D: "\u0E1C\u0E48\u0E32\u0E19\u0E17\u0E32\u0E07\u0E15\u0E31\u0E27\u0E02\u0E19\ + \u0E2A\u0E48\u0E07\u0E01\u0E23\u0E14\u0E42\u0E21\u0E42\u0E19\u0E04\u0E32\u0E23\ + \u0E4C\u0E1A\u0E34\u0E25\u0E34\u0E01" + input_correct_responses: + - A + input_question: "\u0E01\u0E25\u0E39\u0E42\u0E04\u0E2A\u0E16\u0E39\u0E01\u0E02\u0E19\ + \u0E2A\u0E48\u0E07\u0E40\u0E02\u0E49\u0E32\u0E2A\u0E39\u0E48\u0E40\u0E0B\u0E25\ + \u0E25\u0E4C\u0E01\u0E25\u0E49\u0E32\u0E21\u0E40\u0E19\u0E37\u0E49\u0E2D:" + - input_choice_list: + A: "\u0E44\u0E01\u0E25\u0E42\u0E04\u0E40\u0E08\u0E19\u0E43\u0E19\u0E01\u0E25\ + \u0E49\u0E32\u0E21\u0E40\u0E19\u0E37\u0E49\u0E2D\u0E08\u0E30\u0E16\u0E39\u0E01\ + \u0E22\u0E48\u0E2D\u0E22\u0E2A\u0E25\u0E32\u0E22\u0E14\u0E49\u0E27\u0E22\u0E40\ + \u0E2D\u0E19\u0E44\u0E0B\u0E21\u0E4C\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E25\u0E39\ + \u0E42\u0E04\u0E2A-1-\u0E1F\u0E2D\u0E2A\u0E40\u0E1F\u0E15" + B: "\u0E19\u0E31\u0E01\u0E27\u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E21\u0E35\ + \u0E04\u0E27\u0E32\u0E21\u0E2D\u0E14\u0E17\u0E19\u0E2A\u0E39\u0E07\u0E21\u0E35\ + \u0E40\u0E2A\u0E49\u0E19\u0E43\u0E22\u0E1B\u0E23\u0E30\u0E40\u0E20\u0E17 I\ + \ \u0E43\u0E19\u0E2A\u0E31\u0E14\u0E2A\u0E48\u0E27\u0E19\u0E17\u0E35\u0E48\ + \u0E2A\u0E39\u0E07\u0E43\u0E19\u0E01\u0E25\u0E49\u0E32\u0E21\u0E40\u0E19\u0E37\ + \u0E49\u0E2D\u0E02\u0E32" + C: "\u0E44\u0E01\u0E25\u0E42\u0E04\u0E40\u0E08\u0E19\u0E43\u0E19\u0E15\u0E31\ + \u0E1A\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E33\u0E04\u0E31\u0E0D\u0E43\ + \u0E19\u0E01\u0E32\u0E23\u0E23\u0E31\u0E01\u0E29\u0E32\u0E04\u0E27\u0E32\u0E21\ + \u0E40\u0E02\u0E49\u0E21\u0E02\u0E49\u0E19\u0E02\u0E2D\u0E07\u0E01\u0E25\u0E39\ + \u0E42\u0E04\u0E2A\u0E43\u0E19\u0E40\u0E25\u0E37\u0E2D\u0E14" + D: "\u0E2D\u0E34\u0E19\u0E0B\u0E39\u0E25\u0E34\u0E19\u0E2A\u0E48\u0E07\u0E40\ + \u0E2A\u0E23\u0E34\u0E21\u0E01\u0E32\u0E23\u0E14\u0E39\u0E14\u0E0B\u0E36\u0E21\ + \u0E01\u0E25\u0E39\u0E42\u0E04\u0E2A\u0E08\u0E32\u0E01\u0E40\u0E19\u0E37\u0E49\ + \u0E2D\u0E40\u0E22\u0E37\u0E48\u0E2D\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\ + \u0E43\u0E19\u0E23\u0E48\u0E32\u0E07\u0E01\u0E32\u0E22" + input_correct_responses: + - D + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E02\u0E49\u0E2D\u0E04\u0E27\ + \u0E32\u0E21\u0E08\u0E23\u0E34\u0E07" + - input_choice_list: + A: "\u0E25\u0E39\u0E01\u0E2B\u0E25\u0E32\u0E19\u0E43\u0E19\u0E1D\u0E31\u0E48\ + \u0E07\u0E21\u0E32\u0E23\u0E14\u0E32\u0E17\u0E38\u0E01\u0E04\u0E19\u0E08\u0E30\ + \u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E1B\u0E01\u0E15\u0E34" + B: "\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E08\u0E30\u0E44\u0E14\u0E49\ + \u0E23\u0E31\u0E1A\u0E1C\u0E25\u0E01\u0E23\u0E30\u0E17\u0E1A\u0E1B\u0E23\u0E30\ + \u0E21\u0E32\u0E13\u0E2A\u0E2D\u0E07\u0E40\u0E17\u0E48\u0E32\u0E02\u0E2D\u0E07\ + \u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22\u0E43\u0E19\u0E04\u0E23\u0E2D\u0E1A\u0E04\ + \u0E23\u0E31\u0E27\u0E19\u0E35\u0E49" + C: "\u0E25\u0E39\u0E01\u0E2A\u0E32\u0E27\u0E02\u0E2D\u0E07\u0E1C\u0E39\u0E49\ + \u0E0A\u0E32\u0E22\u0E17\u0E35\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E1C\ + \u0E25\u0E01\u0E23\u0E30\u0E17\u0E1A\u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\ + \u0E1A\u0E1C\u0E25\u0E01\u0E23\u0E30\u0E17\u0E1A\u0E17\u0E31\u0E49\u0E07\u0E2B\ + \u0E21\u0E14" + D: "\u0E08\u0E30\u0E21\u0E35\u0E01\u0E32\u0E23\u0E41\u0E08\u0E01\u0E41\u0E08\ + \u0E07\u0E1C\u0E39\u0E49\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E1C\u0E25\u0E01\ + \u0E23\u0E30\u0E17\u0E1A\u0E17\u0E31\u0E49\u0E07\u0E0A\u0E32\u0E22\u0E41\u0E25\ + \u0E30\u0E2B\u0E0D\u0E34\u0E07\u0E40\u0E17\u0E48\u0E32\u0E46 \u0E01\u0E31\u0E19" + input_correct_responses: + - C + input_question: "\u0E43\u0E19\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E17\ + \u0E32\u0E07\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E01\u0E23\u0E23\u0E21\u0E02\u0E2D\ + \u0E07\u0E17\u0E32\u0E23\u0E01\u0E41\u0E23\u0E01\u0E40\u0E01\u0E34\u0E14 \u0E1E\ + \u0E1A\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E1B\u0E01\u0E15\u0E34\u0E17\ + \u0E32\u0E07\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E01\u0E23\u0E23\u0E21\u0E17\u0E35\ + \u0E48\u0E2B\u0E32\u0E44\u0E14\u0E49\u0E22\u0E32\u0E01\u0E0B\u0E36\u0E48\u0E07\ + \u0E21\u0E35\u0E01\u0E32\u0E23\u0E16\u0E48\u0E32\u0E22\u0E17\u0E2D\u0E14\u0E41\ + \u0E1A\u0E1A X-linked recessive \u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E43\ + \u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E19\u0E48\u0E32\u0E08\ + \u0E30\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07\u0E40\u0E01\u0E35\u0E48\ + \u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2A\u0E32\u0E22\u0E40\u0E25\u0E37\u0E2D\u0E14\ + \u0E02\u0E2D\u0E07\u0E42\u0E23\u0E04\u0E19\u0E35\u0E49" + - input_choice_list: + A: "\u0E2D\u0E38\u0E13\u0E2B\u0E20\u0E39\u0E21\u0E34\u0E40\u0E1E\u0E34\u0E48\ + \u0E21\u0E02\u0E36\u0E49\u0E19 \u0E42\u0E21\u0E25\u0E02\u0E2D\u0E07\u0E41\u0E01\ + \u0E4A\u0E2A\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + B: "\u0E2D\u0E38\u0E13\u0E2B\u0E20\u0E39\u0E21\u0E34\u0E17\u0E35\u0E48\u0E40\ + \u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19 \u0E1B\u0E23\u0E34\u0E21\u0E32\ + \u0E13\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + C: "\u0E1B\u0E23\u0E34\u0E21\u0E32\u0E13\u0E25\u0E14\u0E25\u0E07 \u0E2D\u0E38\ + \u0E13\u0E2B\u0E20\u0E39\u0E21\u0E34\u0E25\u0E14\u0E25\u0E07" + D: "\u0E42\u0E21\u0E25\u0E02\u0E2D\u0E07\u0E41\u0E01\u0E4A\u0E2A\u0E25\u0E14\ + \u0E25\u0E07 \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1B\u0E23\u0E34\u0E21\u0E32\u0E15\ + \u0E23" + input_correct_responses: + - A + input_question: "\u0E04\u0E23\u0E39\u0E27\u0E34\u0E17\u0E22\u0E32\u0E28\u0E32\u0E2A\ + \u0E15\u0E23\u0E4C\u0E23\u0E30\u0E14\u0E31\u0E1A\u0E21\u0E31\u0E18\u0E22\u0E21\ + \u0E1B\u0E25\u0E32\u0E22\u0E40\u0E15\u0E34\u0E21\u0E44\u0E19\u0E42\u0E15\u0E23\ + \u0E40\u0E08\u0E19\u0E1A\u0E23\u0E34\u0E2A\u0E38\u0E17\u0E18\u0E34\u0E4C\u0E25\ + \u0E07\u0E43\u0E19\u0E02\u0E27\u0E14\u0E02\u0E19\u0E32\u0E14 1 \u0E25\u0E34\u0E15\ + \u0E23\u0E41\u0E25\u0E49\u0E27\u0E1B\u0E34\u0E14\u0E1D\u0E32 \u0E04\u0E27\u0E32\ + \u0E21\u0E14\u0E31\u0E19\u0E04\u0E37\u0E2D 1.70 atm \u0E41\u0E25\u0E30\u0E2D\ + \u0E38\u0E13\u0E2B\u0E20\u0E39\u0E21\u0E34\u0E2B\u0E49\u0E2D\u0E07\u0E04\u0E37\ + \u0E2D 25\xB0C \u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23\u0E2A\u0E2D\u0E07\u0E15\u0E31\ + \u0E27\u0E43\u0E14\u0E08\u0E30\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E04\u0E27\u0E32\ + \u0E21\u0E14\u0E31\u0E19\u0E02\u0E2D\u0E07\u0E23\u0E30\u0E1A\u0E1A \u0E16\u0E49\ + \u0E32\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23\u0E2D\u0E37\u0E48\u0E19\u0E46 \u0E17\ + \u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\u0E21\u0E35\u0E04\u0E48\u0E32\u0E04\u0E07\ + \u0E17\u0E35\u0E48" + - input_choice_list: + A: "\u0E01\u0E25\u0E49\u0E32\u0E21\u0E40\u0E19\u0E37\u0E49\u0E2D\u0E2D\u0E48\ + \u0E2D\u0E19\u0E41\u0E23\u0E07." + B: "\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E21\u0E27\u0E25\u0E01\u0E32\u0E22" + C: "\u0E1B\u0E27\u0E14\u0E01\u0E25\u0E49\u0E32\u0E21\u0E40\u0E19\u0E37\u0E49\ + \u0E2D" + D: "\u0E01\u0E32\u0E23\u0E2A\u0E39\u0E0D\u0E40\u0E2A\u0E35\u0E22\u0E2D\u0E34\ + \u0E40\u0E25\u0E47\u0E01\u0E42\u0E17\u0E23\u0E44\u0E25\u0E15\u0E4C" + input_correct_responses: + - B + input_question: "\u0E1C\u0E25\u0E02\u0E49\u0E32\u0E07\u0E40\u0E04\u0E35\u0E22\u0E07\ + \u0E17\u0E35\u0E48\u0E04\u0E32\u0E14\u0E27\u0E48\u0E32\u0E08\u0E30\u0E44\u0E14\ + \u0E49\u0E23\u0E31\u0E1A\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E40\u0E2A\u0E23\ + \u0E34\u0E21\u0E04\u0E23\u0E35\u0E40\u0E2D\u0E17\u0E35\u0E19\u0E04\u0E37\u0E2D\ + :" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_medicine +tag: mmlu_th_llama_other_tasks +task: mmlu_th_llama_college_medicine +task_alias: college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1332b08bf9f410d5e45977c9c547bfe637e8bc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_college_physics.yaml @@ -0,0 +1,111 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '4' + B: '5' + C: '6' + D: '20' + input_correct_responses: + - A + input_question: "\u0E01\u0E25\u0E49\u0E2D\u0E07\u0E42\u0E17\u0E23\u0E17\u0E23\u0E23\ + \u0E28\u0E19\u0E4C\u0E41\u0E1A\u0E1A\u0E2B\u0E31\u0E01\u0E40\u0E2B\u0E41\u0E2A\ + \u0E07\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E40\u0E25\ + \u0E19\u0E2A\u0E4C\u0E21\u0E32\u0E1A\u0E23\u0E23\u0E08\u0E1A\u0E01\u0E31\u0E19\ + \ 2 \u0E0A\u0E34\u0E49\u0E19 \u0E42\u0E14\u0E22\u0E2B\u0E48\u0E32\u0E07\u0E01\ + \u0E31\u0E19 100 \u0E0B\u0E21. \u0E40\u0E25\u0E19\u0E2A\u0E4C\u0E43\u0E01\u0E25\ + \u0E49\u0E15\u0E32\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E22\u0E32\u0E27\u0E42\ + \u0E1F\u0E01\u0E31\u0E2A 20 \u0E0B\u0E21. \u0E01\u0E33\u0E25\u0E31\u0E07\u0E02\ + \u0E22\u0E32\u0E22\u0E40\u0E0A\u0E34\u0E07\u0E21\u0E38\u0E21\u0E02\u0E2D\u0E07\ + \u0E01\u0E25\u0E49\u0E2D\u0E07\u0E42\u0E17\u0E23\u0E17\u0E23\u0E23\u0E28\u0E19\ + \u0E4C\u0E04\u0E37\u0E2D" + - input_choice_list: + A: "\u0E2D\u0E38\u0E13\u0E2B\u0E20\u0E39\u0E21\u0E34\u0E04\u0E07\u0E17\u0E35\ + \u0E48" + B: "\u0E1B\u0E23\u0E34\u0E21\u0E32\u0E13\u0E04\u0E07\u0E17\u0E35\u0E48" + C: "\u0E04\u0E27\u0E32\u0E21\u0E14\u0E31\u0E19\u0E04\u0E07\u0E17\u0E35\u0E48" + D: "\u0E2D\u0E30\u0E40\u0E14\u0E35\u0E22\u0E41\u0E1A\u0E15\u0E34\u0E01" + input_correct_responses: + - B + input_question: "\u0E01\u0E23\u0E30\u0E1A\u0E27\u0E19\u0E01\u0E32\u0E23\u0E17\u0E32\ + \u0E07\u0E2D\u0E38\u0E13\u0E2B\u0E1E\u0E25\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C\ + \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\ + \u0E40\u0E1B\u0E47\u0E19\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1E\ + \u0E25\u0E31\u0E07\u0E07\u0E32\u0E19\u0E20\u0E32\u0E22\u0E43\u0E19\u0E02\u0E2D\ + \u0E07\u0E01\u0E4A\u0E32\u0E0B\u0E43\u0E19\u0E2D\u0E38\u0E14\u0E21\u0E04\u0E15\ + \u0E34\u0E43\u0E2B\u0E49\u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A\u0E04\u0E27\ + \u0E32\u0E21\u0E23\u0E49\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\ + \u0E21\u0E40\u0E02\u0E49\u0E32\u0E44\u0E1B\u0E43\u0E19\u0E01\u0E4A\u0E32\u0E0B" + - input_choice_list: + A: "2.4 \u0E42\u0E27\u0E25\u0E15\u0E4C" + B: "3.3 \u0E42\u0E27\u0E25\u0E15\u0E4C" + C: "4.5 \u0E42\u0E27\u0E25\u0E15\u0E4C" + D: "5.7 \u0E42\u0E27\u0E25\u0E15\u0E4C" + input_correct_responses: + - A + input_question: "\u0E1B\u0E25\u0E32\u0E22\u0E14\u0E49\u0E32\u0E19\u0E2B\u0E19\u0E36\ + \u0E48\u0E07\u0E02\u0E2D\u0E07\u0E40\u0E2A\u0E49\u0E19\u0E25\u0E27\u0E14 Nichrome\ + \ \u0E04\u0E27\u0E32\u0E21\u0E22\u0E32\u0E27 2L \u0E41\u0E25\u0E30\u0E1E\u0E37\ + \u0E49\u0E19\u0E17\u0E35\u0E48\u0E2B\u0E19\u0E49\u0E32\u0E15\u0E31\u0E14 A \u0E15\ + \u0E48\u0E2D\u0E40\u0E02\u0E49\u0E32\u0E01\u0E31\u0E1A\u0E1B\u0E25\u0E32\u0E22\ + \u0E40\u0E2A\u0E49\u0E19\u0E25\u0E27\u0E14 Nichrome \u0E04\u0E27\u0E32\u0E21\ + \u0E22\u0E32\u0E27 L \u0E41\u0E25\u0E30\u0E1E\u0E37\u0E49\u0E19\u0E17\u0E35\u0E48\ + \u0E2B\u0E19\u0E49\u0E32\u0E15\u0E31\u0E14 2A \u0E2D\u0E35\u0E01\u0E40\u0E2A\ + \u0E49\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07 \u0E16\u0E49\u0E32\u0E1B\u0E25\u0E32\ + \u0E22\u0E14\u0E49\u0E32\u0E19\u0E27\u0E48\u0E32\u0E07\u0E02\u0E2D\u0E07\u0E2A\ + \u0E32\u0E22\u0E17\u0E35\u0E48\u0E22\u0E32\u0E27\u0E01\u0E27\u0E48\u0E32\u0E21\ + \u0E35\u0E28\u0E31\u0E01\u0E22\u0E4C\u0E44\u0E1F\u0E1F\u0E49\u0E32 8.0 \u0E42\ + \u0E27\u0E25\u0E15\u0E4C \u0E41\u0E25\u0E30\u0E1B\u0E25\u0E32\u0E22\u0E14\u0E49\ + \u0E32\u0E19\u0E27\u0E48\u0E32\u0E07\u0E02\u0E2D\u0E07\u0E2A\u0E32\u0E22\u0E17\ + \u0E35\u0E48\u0E2A\u0E31\u0E49\u0E19\u0E01\u0E27\u0E48\u0E32\u0E2D\u0E22\u0E39\ + \u0E48\u0E17\u0E35\u0E48\u0E28\u0E31\u0E01\u0E22\u0E4C\u0E44\u0E1F\u0E1F\u0E49\ + \u0E32 1.0 \u0E42\u0E27\u0E25\u0E15\u0E4C \u0E28\u0E31\u0E01\u0E22\u0E4C\u0E17\ + \u0E35\u0E48\u0E08\u0E38\u0E14\u0E15\u0E48\u0E2D\u0E02\u0E2D\u0E07\u0E2A\u0E32\ + \u0E22\u0E17\u0E31\u0E49\u0E07\u0E2A\u0E2D\u0E07\u0E08\u0E30\u0E40\u0E01\u0E37\ + \u0E2D\u0E1A\u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A" + - input_choice_list: + A: '4' + B: '5' + C: '6' + D: '20' + input_correct_responses: + - A + input_question: "\u0E01\u0E25\u0E49\u0E2D\u0E07\u0E42\u0E17\u0E23\u0E17\u0E23\u0E23\ + \u0E28\u0E19\u0E4C\u0E41\u0E1A\u0E1A\u0E2B\u0E31\u0E01\u0E40\u0E2B\u0E41\u0E2A\ + \u0E07\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E40\u0E25\ + \u0E19\u0E2A\u0E4C\u0E21\u0E32\u0E1A\u0E23\u0E23\u0E08\u0E1A\u0E01\u0E31\u0E19\ + \ 2 \u0E0A\u0E34\u0E49\u0E19 \u0E42\u0E14\u0E22\u0E2B\u0E48\u0E32\u0E07\u0E01\ + \u0E31\u0E19 100 \u0E0B\u0E21. \u0E40\u0E25\u0E19\u0E2A\u0E4C\u0E43\u0E01\u0E25\ + \u0E49\u0E15\u0E32\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E22\u0E32\u0E27\u0E42\ + \u0E1F\u0E01\u0E31\u0E2A 20 \u0E0B\u0E21. \u0E01\u0E33\u0E25\u0E31\u0E07\u0E02\ + \u0E22\u0E32\u0E22\u0E40\u0E0A\u0E34\u0E07\u0E21\u0E38\u0E21\u0E02\u0E2D\u0E07\ + \u0E01\u0E25\u0E49\u0E2D\u0E07\u0E42\u0E17\u0E23\u0E17\u0E23\u0E23\u0E28\u0E19\ + \u0E4C\u0E04\u0E37\u0E2D" + - input_choice_list: + A: "\u0E04\u0E48\u0E32\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22" + B: "\u0E21\u0E27\u0E25" + C: "\u0E1E\u0E25\u0E31\u0E07\u0E07\u0E32\u0E19\u0E41\u0E25\u0E30\u0E42\u0E21\ + \u0E40\u0E21\u0E19\u0E15\u0E31\u0E21" + D: "\u0E2B\u0E21\u0E32\u0E22\u0E40\u0E25\u0E02\u0E40\u0E25\u0E1B\u0E15\u0E31\ + \u0E19" + input_correct_responses: + - D + input_question: "\u0E21\u0E34\u0E27\u0E2D\u0E2D\u0E19\u0E08\u0E30\u0E2A\u0E25\u0E32\ + \u0E22\u0E15\u0E31\u0E27\u0E42\u0E14\u0E22\u0E21\u0E35\u0E25\u0E31\u0E01\u0E29\ + \u0E13\u0E30\u0E2D\u0E32\u0E22\u0E38\u0E02\u0E31\u0E22\u0E1B\u0E23\u0E30\u0E21\ + \u0E32\u0E13 10^-6 \u0E27\u0E34\u0E19\u0E32\u0E17\u0E35\u0E40\u0E1B\u0E47\u0E19\ + \u0E2D\u0E34\u0E40\u0E25\u0E47\u0E01\u0E15\u0E23\u0E2D\u0E19 \u0E21\u0E34\u0E27\ + \u0E2D\u0E2D\u0E19\u0E19\u0E34\u0E27\u0E15\u0E23\u0E34\u0E42\u0E19 \u0E41\u0E25\ + \u0E30\u0E2D\u0E34\u0E40\u0E25\u0E47\u0E01\u0E15\u0E23\u0E2D\u0E19\u0E41\u0E2D\ + \u0E19\u0E15\u0E34\u0E19\u0E34\u0E27\u0E15\u0E23\u0E34\u0E42\u0E19 \u0E21\u0E34\ + \u0E27\u0E2D\u0E2D\u0E19\u0E16\u0E39\u0E01\u0E2B\u0E49\u0E32\u0E21\u0E44\u0E21\ + \u0E48\u0E43\u0E2B\u0E49\u0E2A\u0E25\u0E32\u0E22\u0E15\u0E31\u0E27\u0E40\u0E1B\ + \u0E47\u0E19\u0E2D\u0E34\u0E40\u0E25\u0E47\u0E01\u0E15\u0E23\u0E2D\u0E19\u0E41\ + \u0E25\u0E30\u0E19\u0E34\u0E27\u0E15\u0E23\u0E34\u0E42\u0E19\u0E40\u0E1E\u0E35\ + \u0E22\u0E07\u0E15\u0E31\u0E27\u0E40\u0E14\u0E35\u0E22\u0E27\u0E15\u0E32\u0E21\ + \u0E01\u0E0E\u0E01\u0E32\u0E23\u0E2D\u0E19\u0E38\u0E23\u0E31\u0E01\u0E29\u0E4C" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_college_physics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_college_physics +task_alias: college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d653633a43966d929e81353eb07e8bcb79fa3d80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_computer_security.yaml @@ -0,0 +1,98 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "160 \u0E1A\u0E34\u0E15" + B: "512 \u0E1A\u0E34\u0E15" + C: "628 \u0E1A\u0E34\u0E15" + D: "820 \u0E1A\u0E34\u0E15" + input_correct_responses: + - A + input_question: "SHA-1 \u0E21\u0E35\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E22\ + \u0E48\u0E2D\u0E22\u0E02\u0E2D\u0E07" + - input_choice_list: + A: "IM \u2013 \u0E42\u0E17\u0E23\u0E08\u0E31\u0E19" + B: "\u0E42\u0E17\u0E23\u0E08\u0E31\u0E19\u0E25\u0E31\u0E1A\u0E46" + C: "\u0E42\u0E1B\u0E23\u0E41\u0E01\u0E23\u0E21\u0E14\u0E32\u0E27\u0E19\u0E4C\ + \u0E42\u0E2B\u0E25\u0E14\u0E42\u0E17\u0E23\u0E08\u0E31\u0E19" + D: "\u0E42\u0E17\u0E23\u0E08\u0E31\u0E19\u0E40\u0E23\u0E35\u0E22\u0E01\u0E04\ + \u0E48\u0E32\u0E44\u0E16\u0E48" + input_correct_responses: + - D + input_question: "_____________ \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E41\u0E01\ + \u0E49\u0E44\u0E02\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E43\u0E19\u0E23\u0E30\ + \u0E1A\u0E1A\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13 \u0E40\u0E1E\u0E37\u0E48\u0E2D\ + \u0E43\u0E2B\u0E49\u0E23\u0E30\u0E1A\u0E1A\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13\ + \u0E17\u0E33\u0E07\u0E32\u0E19\u0E44\u0E21\u0E48\u0E16\u0E39\u0E01\u0E15\u0E49\ + \u0E2D\u0E07\u0E2B\u0E23\u0E37\u0E2D\u0E04\u0E38\u0E13\u0E44\u0E21\u0E48\u0E2A\ + \u0E32\u0E21\u0E32\u0E23\u0E16\u0E40\u0E02\u0E49\u0E32\u0E16\u0E36\u0E07\u0E02\ + \u0E49\u0E2D\u0E21\u0E39\u0E25\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E44\u0E14\u0E49\ + \u0E2D\u0E35\u0E01\u0E15\u0E48\u0E2D\u0E44\u0E1B \u0E2B\u0E23\u0E37\u0E2D\u0E2D\ + \u0E32\u0E08\u0E40\u0E23\u0E35\u0E22\u0E01\u0E04\u0E48\u0E32\u0E44\u0E16\u0E48\ + \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E04\u0E38\u0E13\u0E40\u0E02\ + \u0E49\u0E32\u0E16\u0E36\u0E07\u0E44\u0E14\u0E49" + - input_choice_list: + A: "\u0E08\u0E23\u0E34\u0E22\u0E18\u0E23\u0E23\u0E21 "\u0E01\u0E32\u0E23\ + \u0E41\u0E2E\u0E47\u0E01" \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E41\u0E2A\u0E14\ + \u0E07\u0E43\u0E2B\u0E49\u0E40\u0E2B\u0E47\u0E19\u0E16\u0E36\u0E07\u0E1E\u0E24\ + \u0E15\u0E34\u0E01\u0E23\u0E23\u0E21\u0E17\u0E35\u0E48\u0E40\u0E2B\u0E47\u0E19\ + \u0E41\u0E01\u0E48\u0E15\u0E31\u0E27\u0E42\u0E14\u0E22\u0E44\u0E21\u0E48\u0E44\ + \u0E14\u0E49\u0E15\u0E31\u0E49\u0E07\u0E43\u0E08" + B: "\u0E40\u0E08\u0E32\u0E30\u0E23\u0E30\u0E1A\u0E1A (\u0E40\u0E0A\u0E48\u0E19\ + \ \u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E01\u0E32\u0E23\u0E17\u0E14\ + \u0E2A\u0E2D\u0E1A\u0E01\u0E32\u0E23\u0E40\u0E08\u0E32\u0E30\u0E23\u0E30\u0E1A\ + \u0E1A) \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E40\u0E1B\u0E34\u0E14\u0E42\u0E1B\ + \u0E07\u0E0A\u0E48\u0E2D\u0E07\u0E42\u0E2B\u0E27\u0E48\u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E43\u0E2B\u0E49\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E41\u0E01\u0E49\ + \u0E44\u0E02\u0E44\u0E14\u0E49 \u0E41\u0E17\u0E19\u0E17\u0E35\u0E48\u0E08\u0E30\ + \u0E19\u0E33\u0E44\u0E1B\u0E43\u0E0A\u0E49\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\ + \u0E19\u0E4C" + C: "\u0E01\u0E32\u0E23\u0E41\u0E2E\u0E47\u0E01\u0E40\u0E02\u0E49\u0E32\u0E2A\ + \u0E39\u0E48\u0E23\u0E30\u0E1A\u0E1A\u0E17\u0E35\u0E48\u0E14\u0E33\u0E40\u0E19\ + \u0E34\u0E19\u0E01\u0E32\u0E23\u0E42\u0E14\u0E22\u0E1C\u0E39\u0E49\u0E17\u0E35\ + \u0E48\u0E04\u0E38\u0E13\u0E44\u0E21\u0E48\u0E40\u0E2B\u0E47\u0E19\u0E14\u0E49\ + \u0E27\u0E22\u0E01\u0E31\u0E1A\u0E08\u0E23\u0E34\u0E22\u0E18\u0E23\u0E23\u0E21" + D: "\u0E04\u0E33\u0E28\u0E31\u0E1E\u0E17\u0E4C\u0E2A\u0E41\u0E25\u0E07\u0E2A\ + \u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E1E\u0E31\u0E12\u0E19\u0E32\ + \u0E0B\u0E2D\u0E1F\u0E15\u0E4C\u0E41\u0E27\u0E23\u0E4C\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E23\u0E27\u0E14\u0E40\u0E23\u0E47\u0E27 \u0E40\u0E0A\u0E48\u0E19\ + \ \u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\ + \u0E07\u0E02\u0E2D\u0E07\u0E41\u0E2E\u0E47\u0E01\u0E01\u0E32\u0E18\u0E2D\u0E19" + input_correct_responses: + - B + input_question: "\u0E01\u0E32\u0E23\u0E41\u0E2E\u0E47\u0E04\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E21\u0E35\u0E08\u0E23\u0E34\u0E22\u0E18\u0E23\u0E23\u0E21\u0E04\u0E37\ + \u0E2D\u0E2D\u0E30\u0E44\u0E23?" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E40\u0E02\u0E35\u0E22\u0E19\u0E17\u0E31\u0E1A\u0E04\ + \u0E35\u0E22\u0E4C\u0E40\u0E02\u0E49\u0E32\u0E23\u0E2B\u0E31\u0E2A\u0E43\u0E19\ + \u0E2B\u0E19\u0E48\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E08\u0E33" + B: "\u0E01\u0E32\u0E23\u0E09\u0E35\u0E14\u0E23\u0E2B\u0E31\u0E2A\u0E0A\u0E19\ + \u0E34\u0E14\u0E2B\u0E19\u0E36\u0E48\u0E07" + C: "\u0E2D\u0E48\u0E32\u0E19\u0E19\u0E2D\u0E01\u0E02\u0E2D\u0E1A\u0E40\u0E02\ + \u0E15\u0E02\u0E2D\u0E07\u0E1A\u0E31\u0E1F\u0E40\u0E1F\u0E2D\u0E23\u0E4C" + D: "\u0E01\u0E32\u0E23\u0E42\u0E08\u0E21\u0E15\u0E35\u0E2A\u0E15\u0E23\u0E34\ + \u0E07\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A" + input_correct_responses: + - C + input_question: "\u0E2D\u0E19\u0E38\u0E0D\u0E32\u0E15\u0E43\u0E2B\u0E49\u0E43\u0E0A\ + \u0E49\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\u0E19\u0E4C\u0E08\u0E32\u0E01\u0E1A\ + \u0E31\u0E4A\u0E01 Heartbleed" + - input_choice_list: + A: "\u0E40\u0E27\u0E47\u0E1A\u0E1C\u0E35\u0E2A\u0E34\u0E07" + B: "\u0E40\u0E27\u0E34\u0E25\u0E14\u0E4C\u0E44\u0E27\u0E14\u0E4C\u0E40\u0E27\ + \u0E47\u0E1A" + C: "\u0E1E\u0E37\u0E49\u0E19\u0E1C\u0E34\u0E27\u0E40\u0E27\u0E47\u0E1A" + D: "\u0E40\u0E27\u0E47\u0E1A\u0E25\u0E36\u0E01" + input_correct_responses: + - D + input_question: "____________ \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23\u0E01\ + \u0E47\u0E15\u0E32\u0E21\u0E17\u0E35\u0E48\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\ + \u0E07\u0E21\u0E37\u0E2D\u0E04\u0E49\u0E19\u0E2B\u0E32\u0E02\u0E2D\u0E07\u0E04\ + \u0E38\u0E13\u0E44\u0E21\u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E04\u0E49\ + \u0E19\u0E2B\u0E32\u0E44\u0E14\u0E49" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_computer_security +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_computer_security +task_alias: computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed701ff13f3749d4c776bf4d9d7876ee39b53e6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_conceptual_physics.yaml @@ -0,0 +1,83 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E19\u0E49\u0E2D\u0E22" + B: "\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32" + C: "\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19" + D: "\u0E28\u0E39\u0E19\u0E22\u0E4C" + input_correct_responses: + - A + input_question: "\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E1B\u0E23\u0E35\u0E22\u0E1A\ + \u0E40\u0E17\u0E35\u0E22\u0E1A\u0E01\u0E31\u0E1A\u0E21\u0E27\u0E25\u0E02\u0E2D\ + \u0E07\u0E2D\u0E30\u0E15\u0E2D\u0E21\u0E22\u0E39\u0E40\u0E23\u0E40\u0E19\u0E35\ + \u0E22\u0E21\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\u0E1B\u0E0F\u0E34\u0E01\ + \u0E34\u0E23\u0E34\u0E22\u0E32\u0E1F\u0E34\u0E0A\u0E0A\u0E31\u0E19 \u0E21\u0E27\ + \u0E25\u0E23\u0E27\u0E21\u0E02\u0E2D\u0E07\u0E1C\u0E25\u0E34\u0E15\u0E20\u0E31\ + \u0E13\u0E11\u0E4C\u0E2B\u0E25\u0E31\u0E07\u0E40\u0E01\u0E34\u0E14\u0E1B\u0E0F\ + \u0E34\u0E01\u0E34\u0E23\u0E34\u0E22\u0E32\u0E1F\u0E34\u0E0A\u0E0A\u0E31\u0E19\ + \u0E04\u0E37\u0E2D" + - input_choice_list: + A: "\u0E1E\u0E37\u0E49\u0E19\u0E17\u0E35\u0E48\u0E41\u0E25\u0E30\u0E40\u0E27\ + \u0E25\u0E32" + B: "\u0E41\u0E1D\u0E14\u0E40\u0E14\u0E34\u0E19\u0E17\u0E32\u0E07\u0E41\u0E25\ + \u0E30\u0E41\u0E1D\u0E14\u0E2D\u0E22\u0E39\u0E48\u0E1A\u0E49\u0E32\u0E19" + C: "\u0E41\u0E23\u0E07\u0E42\u0E19\u0E49\u0E21\u0E16\u0E48\u0E27\u0E07\u0E41\ + \u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E40\u0E23\u0E48\u0E07" + D: "\u0E21\u0E27\u0E25\u0E41\u0E25\u0E30\u0E1E\u0E25\u0E31\u0E07\u0E07\u0E32\ + \u0E19" + input_correct_responses: + - C + input_question: "\u0E2A\u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E21\u0E21\u0E39\ + \u0E25\u0E01\u0E31\u0E19\u0E15\u0E32\u0E21\u0E2B\u0E25\u0E31\u0E01\u0E2A\u0E21\ + \u0E21\u0E39\u0E25\u0E04\u0E37\u0E2D" + - input_choice_list: + A: "\u0E41\u0E1B\u0E25\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E27\u0E32\u0E21\ + \u0E16\u0E35\u0E48\u0E2D\u0E37\u0E48\u0E19" + B: "\u0E01\u0E32\u0E23\u0E42\u0E01\u0E48\u0E07\u0E15\u0E31\u0E27" + C: "\u0E01\u0E32\u0E23\u0E23\u0E1A\u0E01\u0E27\u0E19" + D: "\u0E42\u0E1E\u0E25\u0E32\u0E44\u0E23\u0E0B\u0E4C" + input_correct_responses: + - C + input_question: "\u0E2A\u0E35\u0E43\u0E19\u0E1F\u0E2D\u0E07\u0E2A\u0E1A\u0E39\u0E48\ + \u0E40\u0E01\u0E34\u0E14\u0E08\u0E32\u0E01\u0E41\u0E2A\u0E07" + - input_choice_list: + A: "\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19" + B: "\u0E21\u0E32\u0E01\u0E02\u0E36\u0E49\u0E19" + C: "\u0E19\u0E49\u0E2D\u0E22" + D: "\u0E21\u0E32\u0E01\u0E2B\u0E23\u0E37\u0E2D\u0E19\u0E49\u0E2D\u0E22\u0E02\ + \u0E36\u0E49\u0E19\u0E2D\u0E22\u0E39\u0E48\u0E01\u0E31\u0E1A\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E23\u0E47\u0E27\u0E25\u0E21" + input_correct_responses: + - B + input_question: "\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E1A\u0E34\u0E19\u0E08\ + \u0E33\u0E25\u0E2D\u0E07\u0E08\u0E30\u0E1A\u0E34\u0E19\u0E0A\u0E49\u0E32\u0E25\ + \u0E07\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E1A\u0E34\u0E19\u0E44\u0E1B\u0E43\u0E19\ + \u0E41\u0E19\u0E27\u0E25\u0E21 \u0E41\u0E25\u0E30\u0E1A\u0E34\u0E19\u0E40\u0E23\ + \u0E47\u0E27\u0E02\u0E36\u0E49\u0E19\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E21\u0E35\ + \u0E25\u0E21\u0E2D\u0E22\u0E39\u0E48\u0E14\u0E49\u0E32\u0E19\u0E2B\u0E25\u0E31\ + \u0E07 \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E1B\u0E25\u0E48\u0E2D\u0E22\u0E43\u0E19\ + \u0E21\u0E38\u0E21\u0E17\u0E35\u0E48\u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07\ + \u0E01\u0E31\u0E1A\u0E25\u0E21 \u0E25\u0E21\u0E17\u0E35\u0E48\u0E1E\u0E31\u0E14\ + \u0E21\u0E32\u0E08\u0E30\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E40\u0E23\u0E47\ + \u0E27\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E1E\u0E37\u0E49\u0E19\u0E14\u0E34\u0E19\ + \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E17\u0E35\u0E22\u0E1A\u0E01\u0E31\u0E1A\ + \u0E01\u0E32\u0E23\u0E1A\u0E34\u0E19\u0E43\u0E19\u0E2D\u0E32\u0E01\u0E32\u0E28\ + \u0E19\u0E34\u0E48\u0E07" + - input_choice_list: + A: "\u0E44\u0E2E\u0E42\u0E14\u0E23\u0E40\u0E08\u0E19" + B: "\u0E40\u0E2B\u0E25\u0E47\u0E01" + C: "\u0E22\u0E39\u0E40\u0E23\u0E40\u0E19\u0E35\u0E22\u0E21" + D: "\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E01\u0E31\u0E19\u0E43\u0E19\u0E41\ + \u0E15\u0E48\u0E25\u0E30" + input_correct_responses: + - A + input_question: "\u0E18\u0E32\u0E15\u0E38\u0E17\u0E31\u0E49\u0E07\u0E2A\u0E32\u0E21\ + \u0E0A\u0E19\u0E34\u0E14\u0E43\u0E14\u0E21\u0E35\u0E21\u0E27\u0E25\u0E15\u0E48\ + \u0E2D\u0E19\u0E34\u0E27\u0E04\u0E25\u0E35\u0E2D\u0E2D\u0E19\u0E21\u0E32\u0E01\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_conceptual_physics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_conceptual_physics +task_alias: conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0193bdadb8d09d30758d0e25eac89d9d63a96ec6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_econometrics.yaml @@ -0,0 +1,117 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E25\u0E49\u0E21\u0E2B\u0E32\u0E22\u0E15\u0E32\u0E22\u0E08\u0E32\u0E01\ + \u0E44\u0E1B\u0E43\u0E19\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" + B: "\u0E04\u0E07\u0E2D\u0E22\u0E39\u0E48\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\ + \u0E21\u0E48\u0E21\u0E35\u0E01\u0E33\u0E2B\u0E19\u0E14" + C: "\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E17\ + \u0E27\u0E35\u0E04\u0E39\u0E13" + D: "\u0E44\u0E21\u0E48\u0E40\u0E04\u0E22\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\ + \u0E49\u0E19" + input_correct_responses: + - A + input_question: "\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E23\u0E30\u0E1A\u0E27\ + \u0E19\u0E01\u0E32\u0E23\u0E16\u0E14\u0E16\u0E2D\u0E22\u0E2D\u0E31\u0E15\u0E42\ + \u0E19\u0E21\u0E31\u0E15\u0E34\u0E17\u0E35\u0E48\u0E2D\u0E22\u0E39\u0E48\u0E19\ + \u0E34\u0E48\u0E07 \u0E41\u0E23\u0E07\u0E01\u0E23\u0E30\u0E41\u0E17\u0E01\u0E08\ + \u0E30\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19" + - input_choice_list: + A: '0.2' + B: '0.4' + C: '0.5' + D: '0.33' + input_correct_responses: + - D + input_question: "\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\u0E41\u0E1A\u0E1A\u0E08\ + \u0E33\u0E25\u0E2D\u0E07 AR(1) \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\ + \u0E0B\u0E36\u0E48\u0E07\u0E01\u0E32\u0E23\u0E23\u0E1A\u0E01\u0E27\u0E19\u0E21\ + \u0E35\u0E04\u0E48\u0E32\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22\u0E40\u0E1B\u0E47\ + \u0E19\u0E28\u0E39\u0E19\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\ + \u0E41\u0E1B\u0E23\u0E1B\u0E23\u0E27\u0E19\u0E02\u0E2D\u0E07\u0E2B\u0E19\u0E48\ + \u0E27\u0E22 yt = 0.2 + 0.4 yt-1 + ut \u0E04\u0E48\u0E32\u0E40\u0E09\u0E25\u0E35\ + \u0E48\u0E22 (\u0E44\u0E21\u0E48\u0E21\u0E35\u0E40\u0E07\u0E37\u0E48\u0E2D\u0E19\ + \u0E44\u0E02) \u0E02\u0E2D\u0E07 y \u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\ + \u0E1A\u0E08\u0E32\u0E01" + - input_choice_list: + A: "(ii) \u0E41\u0E25\u0E30 (iv) \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + B: "(i) \u0E41\u0E25\u0E30 (iii) \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + C: "(i), (ii) \u0E41\u0E25\u0E30 (iii) \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\ + \u0E19" + D: "(i), (ii), (iii) \u0E41\u0E25\u0E30 (iv)" + input_correct_responses: + - C + input_question: "\u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32\u0E2A\u0E16\u0E34\ + \u0E15\u0E34\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E21\ + \u0E42\u0E22\u0E07\u0E01\u0E31\u0E1A\u0E04\u0E48\u0E32 p-value 0.08 \u0E02\u0E49\ + \u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E1B\ + \u0E47\u0E19\u0E08\u0E23\u0E34\u0E07 (i) \u0E2B\u0E32\u0E01\u0E02\u0E19\u0E32\ + \u0E14\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E40\ + \u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A 8% \u0E40\u0E23\u0E32\u0E08\u0E30\u0E44\ + \u0E21\u0E48\u0E2A\u0E19\u0E43\u0E08\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\ + \u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18\u0E41\u0E25\u0E30\u0E44\ + \u0E21\u0E48\u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18\u0E2A\u0E21\u0E21\u0E15\u0E34\ + \u0E10\u0E32\u0E19\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E42\u0E21\u0E06\ + \u0E30 (ii) \u0E2A\u0E21\u0E21\u0E15\u0E34\u0E10\u0E32\u0E19\u0E17\u0E35\u0E48\ + \u0E40\u0E1B\u0E47\u0E19\u0E42\u0E21\u0E06\u0E30\u0E08\u0E30\u0E16\u0E39\u0E01\ + \u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18\u0E2B\u0E32\u0E01\u0E43\u0E0A\u0E49\u0E02\ + \u0E19\u0E32\u0E14\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A 10% (iii)\ + \ \u0E27\u0E48\u0E32\u0E07\u0E40\u0E1B\u0E25\u0E48\u0E32\u0E08\u0E30\u0E44\u0E21\ + \u0E48 \u0E16\u0E39\u0E01\u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18\u0E2B\u0E32\u0E01\ + \u0E43\u0E0A\u0E49\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E02\u0E19\ + \u0E32\u0E14 1% (iv) \u0E04\u0E48\u0E32\u0E27\u0E48\u0E32\u0E07\u0E08\u0E30\u0E16\ + \u0E39\u0E01\u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18\u0E2B\u0E32\u0E01\u0E43\u0E0A\ + \u0E49\u0E02\u0E19\u0E32\u0E14\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A\ + \ 5%" + - input_choice_list: + A: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E25\u0E33\u0E40\u0E2D\u0E35\u0E22\u0E07" + B: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E44\u0E21\u0E48\u0E2A\u0E2D\u0E14\u0E04\ + \u0E25\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E19" + C: "\u0E21\u0E31\u0E19\u0E08\u0E30\u0E02\u0E32\u0E14\u0E1B\u0E23\u0E30\u0E2A\ + \u0E34\u0E17\u0E18\u0E34\u0E20\u0E32\u0E1E" + D: "\u0E02\u0E49\u0E2D (a) (b) \u0E41\u0E25\u0E30 (c) \u0E08\u0E30\u0E40\u0E1B\ + \u0E47\u0E19\u0E08\u0E23\u0E34\u0E07\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14" + input_correct_responses: + - C + input_question: "\u0E2D\u0E30\u0E44\u0E23\u0E08\u0E30\u0E40\u0E01\u0E34\u0E14\u0E02\ + \u0E36\u0E49\u0E19\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E15\u0E31\u0E27\u0E1B\ + \u0E23\u0E30\u0E21\u0E32\u0E13 OLS \u0E2B\u0E32\u0E01 heteroscedasticity \u0E21\ + \u0E35\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E41\u0E1A\u0E1A\u0E08\u0E33\u0E25\ + \u0E2D\u0E07\u0E01\u0E32\u0E23\u0E16\u0E14\u0E16\u0E2D\u0E22 \u0E41\u0E15\u0E48\ + \u0E16\u0E39\u0E01\u0E40\u0E1E\u0E34\u0E01\u0E40\u0E09\u0E22" + - input_choice_list: + A: "1 \u0E04\u0E27\u0E32\u0E21\u0E25\u0E48\u0E32\u0E0A\u0E49\u0E32" + B: "2 \u0E25\u0E48\u0E32\u0E0A\u0E49\u0E32" + C: "3 \u0E25\u0E48\u0E32\u0E0A\u0E49\u0E32" + D: "4 \u0E25\u0E48\u0E32\u0E0A\u0E49\u0E32" + input_correct_responses: + - C + input_question: "\u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32\u0E15\u0E2D\u0E19\ + \u0E19\u0E35\u0E49\u0E19\u0E31\u0E01\u0E27\u0E34\u0E08\u0E31\u0E22\u0E15\u0E49\ + \u0E2D\u0E07\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E40\u0E01\u0E13\u0E11\u0E4C\ + \u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E01\u0E33\ + \u0E2B\u0E19\u0E14\u0E23\u0E30\u0E22\u0E30\u0E40\u0E27\u0E25\u0E32\u0E2B\u0E19\ + \u0E48\u0E27\u0E07\u0E17\u0E35\u0E48\u0E40\u0E2B\u0E21\u0E32\u0E30\u0E2A\u0E21\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A VAR\ + \ \u0E21\u0E35\u0E01\u0E32\u0E23\u0E2A\u0E31\u0E07\u0E40\u0E01\u0E15 500 \u0E23\ + \u0E32\u0E22\u0E01\u0E32\u0E23\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A VAR \u0E41\ + \u0E1A\u0E1A\u0E2A\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23 \u0E41\u0E25\ + \u0E30\u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07\u0E14\u0E35\u0E40\u0E17\u0E2D\u0E23\ + \u0E4C\u0E21\u0E34\u0E41\u0E19\u0E19\u0E15\u0E4C\u0E02\u0E2D\u0E07\u0E40\u0E21\ + \u0E17\u0E23\u0E34\u0E01\u0E0B\u0E4C\u0E04\u0E27\u0E32\u0E21\u0E41\u0E1B\u0E23\ + \u0E1B\u0E23\u0E27\u0E19-\u0E04\u0E27\u0E32\u0E21\u0E41\u0E1B\u0E23\u0E1B\u0E23\ + \u0E27\u0E19\u0E23\u0E48\u0E27\u0E21\u0E02\u0E2D\u0E07\u0E04\u0E48\u0E32\u0E17\ + \u0E35\u0E48\u0E40\u0E2B\u0E25\u0E37\u0E2D\u0E04\u0E37\u0E2D 0.0336, 0.0169,\ + \ 0.0084 \u0E41\u0E25\u0E30 0.0062 \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A 1, 2,\ + \ 3 \u0E41\u0E25\u0E30 4 \u0E04\u0E27\u0E32\u0E21\u0E25\u0E48\u0E32\u0E0A\u0E49\ + \u0E32\u0E15\u0E32\u0E21\u0E25\u0E33\u0E14\u0E31\u0E1A \u0E25\u0E33\u0E14\u0E31\ + \u0E1A\u0E23\u0E38\u0E48\u0E19\u0E17\u0E35\u0E48\u0E40\u0E2B\u0E21\u0E32\u0E30\ + \u0E2A\u0E21\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E15\u0E32\u0E21\u0E40\u0E01\ + \u0E13\u0E11\u0E4C\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E02\u0E2D\u0E07 Akaike\ + \ \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_econometrics +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_econometrics +task_alias: econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0690c5d5138099b5f6847416b4fd9bedf6ef2a87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_electrical_engineering.yaml @@ -0,0 +1,82 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: S=0, R=0 + B: S=0, R=1 + C: S=1, R=0 + D: S=1, R=1 + input_correct_responses: + - D + input_question: "\u0E43\u0E19\u0E2A\u0E25\u0E31\u0E01 SR \u0E17\u0E35\u0E48\u0E2A\ + \u0E23\u0E49\u0E32\u0E07\u0E08\u0E32\u0E01\u0E40\u0E01\u0E17 NOR \u0E0B\u0E36\ + \u0E48\u0E07\u0E44\u0E21\u0E48\u0E2D\u0E19\u0E38\u0E0D\u0E32\u0E15\u0E40\u0E07\ + \u0E37\u0E48\u0E2D\u0E19\u0E44\u0E02" + - input_choice_list: + A: "200\u0E42\u0E2D\u0E2B\u0E4C\u0E21" + B: "100\u0E42\u0E2D\u0E2B\u0E4C\u0E21" + C: "50\u0E42\u0E2D\u0E2B\u0E4C\u0E21" + D: "10\u0E42\u0E2D\u0E2B\u0E4C\u0E21" + input_correct_responses: + - C + input_question: "\u0E43\u0E19\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07 DC \u0E17\ + \u0E35\u0E48\u0E1E\u0E31\u0E19\u0E23\u0E2D\u0E1A\u0E02\u0E14\u0E25\u0E27\u0E14\ + \ 2 \u0E02\u0E31\u0E49\u0E27 \u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E32\u0E19\ + \u0E17\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E19\u0E33\u0E2B\u0E19\ + \u0E36\u0E48\u0E07\u0E15\u0E31\u0E27\u0E04\u0E37\u0E2D 2\u03A9 \u0E41\u0E25\u0E30\ + \u0E08\u0E33\u0E19\u0E27\u0E19\u0E15\u0E31\u0E27\u0E19\u0E33\u0E17\u0E31\u0E49\ + \u0E07\u0E2B\u0E21\u0E14\u0E04\u0E37\u0E2D 100 \u0E08\u0E07\u0E2B\u0E32\u0E04\ + \u0E27\u0E32\u0E21\u0E15\u0E49\u0E32\u0E19\u0E17\u0E32\u0E19\u0E17\u0E31\u0E49\ + \u0E07\u0E2B\u0E21\u0E14" + - input_choice_list: + A: "1 \u0E21." + B: "2 \u0E21\u0E34\u0E25\u0E25\u0E34\u0E41\u0E2D\u0E21\u0E1B\u0E4C" + C: "3 \u0E21\u0E34\u0E25\u0E25\u0E34\u0E41\u0E2D\u0E21\u0E1B\u0E4C" + D: "4 \u0E21." + input_correct_responses: + - B + input_question: "\u0E02\u0E14\u0E25\u0E27\u0E14\u0E02\u0E2D\u0E07\u0E21\u0E34\u0E40\ + \u0E15\u0E2D\u0E23\u0E4C\u0E02\u0E14\u0E25\u0E27\u0E14\u0E40\u0E04\u0E25\u0E37\ + \u0E48\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E21\u0E35 100 \u0E23\u0E2D\u0E1A \u0E22\ + \u0E32\u0E27 40 \u0E21\u0E21. \u0E41\u0E25\u0E30\u0E01\u0E27\u0E49\u0E32\u0E07\ + \ 30 \u0E21\u0E21. \u0E41\u0E23\u0E07\u0E1A\u0E34\u0E14\u0E04\u0E27\u0E1A\u0E04\ + \u0E38\u0E21\u0E2D\u0E22\u0E39\u0E48\u0E17\u0E35\u0E48 240*10-6 Nm \u0E43\u0E19\ + \u0E23\u0E30\u0E14\u0E31\u0E1A\u0E40\u0E15\u0E47\u0E21 \u0E16\u0E49\u0E32\u0E04\ + \u0E27\u0E32\u0E21\u0E2B\u0E19\u0E32\u0E41\u0E19\u0E48\u0E19\u0E1F\u0E25\u0E31\ + \u0E01\u0E0B\u0E4C\u0E41\u0E21\u0E48\u0E40\u0E2B\u0E25\u0E47\u0E01\u0E04\u0E37\ + \u0E2D 1Wb/m2 \u0E0A\u0E48\u0E27\u0E07\u0E02\u0E2D\u0E07\u0E21\u0E34\u0E40\u0E15\ + \u0E2D\u0E23\u0E4C\u0E04\u0E37\u0E2D" + - input_choice_list: + A: "100 \u0E19." + B: "0.1 \u0E19." + C: "1 \u0E40\u0E2D\u0E47\u0E19" + D: "0.01 \u0E19." + input_correct_responses: + - B + input_question: "\u0E15\u0E31\u0E27\u0E19\u0E33\u0E22\u0E32\u0E27\u0E02\u0E19\u0E32\ + \u0E19\u0E01\u0E31\u0E19\u0E2A\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E21\u0E35\u0E04\ + \u0E48\u0E32 100 A \u0E16\u0E49\u0E32\u0E15\u0E31\u0E27\u0E19\u0E33\u0E41\u0E22\ + \u0E01\u0E08\u0E32\u0E01\u0E01\u0E31\u0E19 20 \u0E21\u0E21. \u0E41\u0E23\u0E07\ + \u0E15\u0E48\u0E2D\u0E40\u0E21\u0E15\u0E23\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\ + \u0E21\u0E22\u0E32\u0E27\u0E02\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E19\u0E33\u0E41\ + \u0E15\u0E48\u0E25\u0E30\u0E15\u0E31\u0E27\u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19" + - input_choice_list: + A: "15 \u0E19." + B: "20 \u0E19." + C: "7.5 \u0E19." + D: "3.75 \u0E19." + input_correct_responses: + - A + input_question: "\u0E40\u0E2A\u0E32\u0E08\u0E38\u0E14\u0E21\u0E35\u0E04\u0E27\u0E32\ + \u0E21\u0E41\u0E02\u0E47\u0E07\u0E41\u0E23\u0E07 4\u03C0 * 10^-4 \u0E40\u0E27\ + \u0E40\u0E1A\u0E2D\u0E23\u0E4C \u0E41\u0E23\u0E07\u0E43\u0E19\u0E2B\u0E19\u0E48\ + \u0E27\u0E22\u0E19\u0E34\u0E27\u0E15\u0E31\u0E19\u0E1A\u0E19\u0E40\u0E2A\u0E32\ + \u0E02\u0E19\u0E32\u0E14 4\u03C0 * 1.5 * 10^-4 \u0E40\u0E27\u0E40\u0E1A\u0E2D\ + \u0E23\u0E4C\u0E17\u0E35\u0E48\u0E27\u0E32\u0E07\u0E2B\u0E48\u0E32\u0E07\u0E08\ + \u0E32\u0E01\u0E40\u0E2A\u0E32 10 \u0E0B\u0E21. \u0E08\u0E30\u0E40\u0E1B\u0E47\ + \u0E19" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_electrical_engineering +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_electrical_engineering +task_alias: electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b48d44d9684f25fef4e848f67df52ca7bd92cd72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_elementary_mathematics.yaml @@ -0,0 +1,96 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "5 \u0E1E\u0E31\u0E19" + B: "5 \u0E23\u0E49\u0E2D\u0E22" + C: "5 \u0E2A\u0E34\u0E1A" + D: "5 \u0E2D\u0E31\u0E19" + input_correct_responses: + - A + input_question: "\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E02\u0E2D\u0E07\u0E40\ + \u0E21\u0E37\u0E2D\u0E07\u0E17\u0E35\u0E48\u0E21\u0E34\u0E40\u0E0A\u0E25\u0E25\ + \u0E4C\u0E40\u0E01\u0E34\u0E14\u0E04\u0E37\u0E2D 145,826 \u0E04\u0E19 5 \u0E43\ + \u0E19\u0E08\u0E33\u0E19\u0E27\u0E19 145,826 \u0E21\u0E35\u0E04\u0E48\u0E32\u0E40\ + \u0E17\u0E48\u0E32\u0E43\u0E14" + - input_choice_list: + A: "\u0E15\u0E31\u0E27\u0E40\u0E25\u0E02\u0E17\u0E35\u0E48 10 \u0E43\u0E19\u0E23\ + \u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E25\ + \u0E02\u0E04\u0E39\u0E48" + B: "\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E15\u0E31\u0E27\u0E40\u0E25\u0E02\ + \u0E08\u0E30\u0E44\u0E21\u0E48\u0E21\u0E35\u0E40\u0E25\u0E02\u0E04\u0E39\u0E48\ + \u0E2A\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E15\u0E34\u0E14\u0E01\u0E31\u0E19" + C: "\u0E15\u0E31\u0E27\u0E40\u0E25\u0E02\u0E2A\u0E2D\u0E07\u0E15\u0E31\u0E27\ + \u0E16\u0E31\u0E14\u0E44\u0E1B\u0E43\u0E19\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\ + \u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E25\u0E02\u0E04\u0E39\u0E48\u0E41\ + \u0E25\u0E30\u0E40\u0E25\u0E02\u0E04\u0E35\u0E48" + D: "\u0E2B\u0E32\u0E01\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E15\u0E31\u0E27\ + \u0E40\u0E25\u0E02\u0E40\u0E23\u0E34\u0E48\u0E21\u0E15\u0E49\u0E19\u0E14\u0E49\ + \u0E27\u0E22\u0E40\u0E25\u0E02\u0E04\u0E35\u0E48 \u0E23\u0E39\u0E1B\u0E41\u0E1A\ + \u0E1A\u0E01\u0E47\u0E08\u0E30\u0E21\u0E35\u0E41\u0E15\u0E48\u0E40\u0E25\u0E02\ + \u0E04\u0E35\u0E48\u0E43\u0E19\u0E19\u0E31\u0E49\u0E19" + input_correct_responses: + - B + input_question: "Olivia \u0E43\u0E0A\u0E49\u0E01\u0E0E "\u0E1A\u0E27\u0E01\ + \ 11" \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E23\ + \u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E15\u0E31\u0E27\u0E40\u0E25\u0E02\u0E17\u0E35\ + \u0E48\u0E41\u0E2A\u0E14\u0E07\u0E14\u0E49\u0E32\u0E19\u0E25\u0E48\u0E32\u0E07\ + \ 10, 21, 32, 43, 54 \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E40\u0E01\u0E35\u0E48\u0E22\ + \u0E27\u0E01\u0E31\u0E1A\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E08\u0E33\u0E19\ + \u0E27\u0E19\u0E08\u0E23\u0E34\u0E07" + - input_choice_list: + A: "\u0E40\u0E1E\u0E34\u0E48\u0E21 5 \u0E16\u0E36\u0E07 30 \u0E40\u0E1E\u0E37\ + \u0E48\u0E2D\u0E2B\u0E32 35 \u0E17\u0E35\u0E21" + B: "\u0E2B\u0E32\u0E23 30 \u0E14\u0E49\u0E27\u0E22 5 \u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E2B\u0E32 6 \u0E17\u0E35\u0E21" + C: "\u0E04\u0E39\u0E13 30 \u0E41\u0E25\u0E30 5 \u0E40\u0E1E\u0E37\u0E48\u0E2D\ + \u0E2B\u0E32 150 \u0E17\u0E35\u0E21" + D: "\u0E25\u0E1A 5 \u0E08\u0E32\u0E01 30 \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E2B\ + \u0E32 25 \u0E17\u0E35\u0E21" + input_correct_responses: + - B + input_question: "\u0E1C\u0E39\u0E49\u0E40\u0E25\u0E48\u0E19\u0E17\u0E31\u0E49\u0E07\ + \u0E2B\u0E21\u0E14 30 \u0E04\u0E19\u0E08\u0E30\u0E40\u0E25\u0E48\u0E19\u0E1A\ + \u0E32\u0E2A\u0E40\u0E01\u0E47\u0E15\u0E1A\u0E2D\u0E25\u0E17\u0E35\u0E48\u0E2A\ + \u0E27\u0E19\u0E2A\u0E32\u0E18\u0E32\u0E23\u0E13\u0E30 \u0E08\u0E30\u0E21\u0E35\ + \u0E1C\u0E39\u0E49\u0E40\u0E25\u0E48\u0E19 5 \u0E04\u0E19\u0E43\u0E19\u0E41\u0E15\ + \u0E48\u0E25\u0E30\u0E17\u0E35\u0E21 \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E2D\u0E18\ + \u0E34\u0E1A\u0E32\u0E22\u0E27\u0E34\u0E18\u0E35\u0E01\u0E32\u0E23\u0E2B\u0E32\ + \u0E08\u0E33\u0E19\u0E27\u0E19\u0E17\u0E35\u0E21\u0E17\u0E35\u0E48\u0E15\u0E49\ + \u0E2D\u0E07\u0E01\u0E32\u0E23\u0E44\u0E14\u0E49\u0E16\u0E39\u0E01\u0E15\u0E49\ + \u0E2D\u0E07" + - input_choice_list: + A: '749' + B: 2,675 + C: 2,945 + D: 4,250 + input_correct_responses: + - B + input_question: "\u0E23\u0E49\u0E32\u0E19\u0E02\u0E32\u0E22\u0E2A\u0E35\u0E17\u0E35\ + \u0E48\u0E41\u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E01\u0E31\u0E19 107 \u0E2A\ + \u0E35 \u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E21\u0E35\u0E01\u0E23\u0E30\u0E1B\ + \u0E4B\u0E2D\u0E07\u0E2A\u0E35\u0E25\u0E30 25 \u0E01\u0E23\u0E30\u0E1B\u0E4B\ + \u0E2D\u0E07\u0E43\u0E19\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\u0E40\u0E01\u0E47\ + \u0E1A \u0E08\u0E33\u0E19\u0E27\u0E19\u0E01\u0E23\u0E30\u0E1B\u0E4B\u0E2D\u0E07\ + \u0E2A\u0E35\u0E17\u0E35\u0E48\u0E23\u0E49\u0E32\u0E19\u0E04\u0E49\u0E32\u0E21\ + \u0E35\u0E43\u0E19\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\u0E40\u0E01\u0E47\u0E1A\ + \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E14\u0E39\u0E44\u0E14\u0E49\u0E42\u0E14\ + \u0E22\u0E43\u0E0A\u0E49\u0E19\u0E34\u0E1E\u0E08\u0E19\u0E4C\u0E14\u0E49\u0E32\ + \u0E19\u0E25\u0E48\u0E32\u0E07 107 \xD7 25. \u0E17\u0E35\u0E48\u0E23\u0E49\u0E32\ + \u0E19\u0E21\u0E35\u0E17\u0E35\u0E48\u0E40\u0E01\u0E47\u0E1A\u0E2A\u0E35\u0E01\ + \u0E23\u0E30\u0E1B\u0E4B\u0E2D\u0E07\u0E01\u0E35\u0E48\u0E01\u0E23\u0E30\u0E1B\ + \u0E4B\u0E2D\u0E07?" + - input_choice_list: + A: (5 x 4) x (6 x 5) + B: (5 x 5) + (5 x 4) + C: (5 x 5) + (5 x 9) + D: (5 x 9) x (6 x 9) + input_correct_responses: + - B + input_question: "\u0E19\u0E34\u0E1E\u0E08\u0E19\u0E4C\u0E43\u0E14\u0E40\u0E17\u0E35\ + \u0E22\u0E1A\u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A 5 x 9" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_elementary_mathematics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_elementary_mathematics +task_alias: elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93ea86a295b4594d1f34e25f358d16083cf51fef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_formal_logic.yaml @@ -0,0 +1,126 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "~ \u0E1E\u0E14" + B: "(\u2200x)(Px \u2228 ~Dx)" + C: "(\u2200x)(\u0E1E\u0E34\u0E01\u0E40\u0E0B\u0E25 \u2283 ~Dx)" + D: "~ \u0E1B" + input_correct_responses: + - C + input_question: "\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E04\u0E33\u0E41\u0E1B\u0E25\u0E17\ + \u0E35\u0E48\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E40\u0E1B\u0E47\ + \u0E19\u0E15\u0E23\u0E23\u0E01\u0E30\u0E20\u0E32\u0E04\u0E41\u0E2A\u0E14\u0E07\ + : No people drive on Mars." + - input_choice_list: + A: Blgh + B: Bhlg + C: "\u0E1A\u0E23\u0E36\u0E4B\u0E22" + D: "\u0E1A\u0E35\u0E40\u0E01\u0E25" + input_correct_responses: + - C + input_question: "\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E04\u0E33\u0E41\u0E1B\u0E25\u0E17\ + \u0E35\u0E48\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E43\u0E19\u0E15\ + \u0E23\u0E23\u0E01\u0E30\u0E40\u0E1E\u0E23\u0E14\u0E34\u0E40\u0E04\u0E15 \u0E08\ + \u0E2D\u0E23\u0E4C\u0E08\u0E02\u0E2D\u0E22\u0E37\u0E21\u0E40\u0E04\u0E23\u0E37\ + \u0E48\u0E2D\u0E07\u0E15\u0E31\u0E14\u0E2B\u0E0D\u0E49\u0E32\u0E02\u0E2D\u0E07\ + \u0E40\u0E2E\u0E01\u0E40\u0E15\u0E2D\u0E23\u0E4C (g: George; h: Hector; l: \u0E40\ + \u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E15\u0E31\u0E14\u0E2B\u0E0D\u0E49\u0E32\ + \u0E02\u0E2D\u0E07 Hector; Bxyx: x \u0E22\u0E37\u0E21 y \u0E08\u0E32\u0E01 z)" + - input_choice_list: + A: "\u0E21\u0E32\u0E23\u0E34\u0E19\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E19\ + \u0E31\u0E01\u0E40\u0E15\u0E49\u0E19 \u0E1C\u0E39\u0E49\u0E2D\u0E48\u0E2D\u0E19\ + \u0E41\u0E2D\u0E1A\u0E32\u0E07\u0E04\u0E19\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\ + \u0E19\u0E31\u0E01\u0E40\u0E15\u0E49\u0E19 \u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\ + \u0E17\u0E38\u0E01\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E08\u0E30\u0E2D\u0E48\u0E2D\ + \u0E19\u0E41\u0E2D\u0E2B\u0E23\u0E37\u0E2D\u0E08\u0E2D\u0E23\u0E4C\u0E40\u0E08\ + \u0E35\u0E22\u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\ + \u0E1A\u0E2D\u0E25 \u0E21\u0E35\u0E1A\u0E32\u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\u0E1A\u0E2D\ + \u0E25" + B: "\u0E21\u0E32\u0E23\u0E34\u0E19\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E19\ + \u0E31\u0E01\u0E40\u0E15\u0E49\u0E19 \u0E44\u0E21\u0E48\u0E21\u0E35\u0E1C\u0E39\ + \u0E49\u0E2D\u0E48\u0E2D\u0E19\u0E41\u0E2D\u0E40\u0E1B\u0E47\u0E19\u0E19\u0E31\ + \u0E01\u0E40\u0E15\u0E49\u0E19 \u0E17\u0E38\u0E01\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E40\u0E1B\u0E47\u0E19\u0E08\u0E38\u0E14\u0E2D\u0E48\u0E2D\u0E19\u0E2B\u0E23\ + \u0E37\u0E2D\u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\ + \u0E1A\u0E2D\u0E25 \u0E21\u0E35\u0E1A\u0E32\u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\u0E1A\u0E2D\ + \u0E25" + C: "\u0E21\u0E32\u0E23\u0E34\u0E19\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E19\ + \u0E31\u0E01\u0E40\u0E15\u0E49\u0E19 \u0E1C\u0E39\u0E49\u0E2D\u0E48\u0E2D\u0E19\ + \u0E41\u0E2D\u0E1A\u0E32\u0E07\u0E04\u0E19\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\ + \u0E19\u0E31\u0E01\u0E40\u0E15\u0E49\u0E19 \u0E17\u0E38\u0E01\u0E2D\u0E22\u0E48\ + \u0E32\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E38\u0E14\u0E2D\u0E48\u0E2D\u0E19\ + \u0E2B\u0E23\u0E37\u0E2D\u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\ + \u0E22\u0E4C\u0E1A\u0E2D\u0E25 \u0E21\u0E35\u0E1A\u0E32\u0E07\u0E2D\u0E22\u0E48\ + \u0E32\u0E07\u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\ + \u0E1A\u0E2D\u0E25" + D: "\u0E21\u0E32\u0E23\u0E34\u0E19\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E19\ + \u0E31\u0E01\u0E40\u0E15\u0E49\u0E19 \u0E44\u0E21\u0E48\u0E21\u0E35\u0E1C\u0E39\ + \u0E49\u0E2D\u0E48\u0E2D\u0E19\u0E41\u0E2D\u0E40\u0E1B\u0E47\u0E19\u0E19\u0E31\ + \u0E01\u0E40\u0E15\u0E49\u0E19 \u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\u0E17\u0E38\ + \u0E01\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E08\u0E30\u0E2D\u0E48\u0E2D\u0E19\u0E41\ + \u0E2D\u0E2B\u0E23\u0E37\u0E2D\u0E08\u0E2D\u0E23\u0E4C\u0E40\u0E08\u0E35\u0E22\ + \u0E40\u0E25\u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\u0E1A\u0E2D\ + \u0E25 \u0E21\u0E35\u0E1A\u0E32\u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E40\u0E25\ + \u0E48\u0E19\u0E27\u0E2D\u0E25\u0E40\u0E25\u0E22\u0E4C\u0E1A\u0E2D\u0E25" + input_correct_responses: + - D + input_question: "\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E01\u0E32\u0E23\u0E15\u0E35\u0E04\ + \u0E27\u0E32\u0E21\u0E20\u0E32\u0E29\u0E32\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\ + \u0E17\u0E35\u0E48\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E02\u0E2D\ + \u0E07\u0E2D\u0E32\u0E23\u0E4C\u0E01\u0E34\u0E27\u0E40\u0E21\u0E19\u0E15\u0E4C\ + \u0E17\u0E35\u0E48\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E43\u0E19\u0E15\u0E23\u0E23\ + \u0E01\u0E30\u0E20\u0E32\u0E04\u0E41\u0E2A\u0E14\u0E07 Dm (\u2200x)(Wx \u2283\ + \ ~Dx) (\u2200x)Wx \u2228 Ag / (\u2203x)Ax" + - input_choice_list: + A: "\u0E40\u0E17\u0E35\u0E22\u0E1A\u0E40\u0E17\u0E48\u0E32\u0E17\u0E32\u0E07\ + \u0E15\u0E23\u0E23\u0E01\u0E30" + B: "\u0E02\u0E31\u0E14\u0E41\u0E22\u0E49\u0E07" + C: "\u0E44\u0E21\u0E48\u0E21\u0E35\u0E40\u0E2B\u0E15\u0E38\u0E1C\u0E25\u0E40\ + \u0E17\u0E35\u0E22\u0E1A\u0E40\u0E17\u0E48\u0E32\u0E2B\u0E23\u0E37\u0E2D\u0E02\ + \u0E31\u0E14\u0E41\u0E22\u0E49\u0E07 \u0E41\u0E15\u0E48\u0E2A\u0E2D\u0E14\u0E04\ + \u0E25\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E19" + D: "\u0E44\u0E21\u0E48\u0E2A\u0E2D\u0E14\u0E04\u0E25\u0E49\u0E2D\u0E07\u0E01\ + \u0E31\u0E19" + input_correct_responses: + - C + input_question: "\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E15\u0E32\u0E23\u0E32\u0E07\u0E04\ + \u0E27\u0E32\u0E21\u0E08\u0E23\u0E34\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E21\u0E1A\ + \u0E39\u0E23\u0E13\u0E4C\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E04\u0E39\u0E48\ + \u0E1B\u0E23\u0E30\u0E1E\u0E08\u0E19\u0E4C\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49 \u0E08\u0E32\u0E01\u0E19\u0E31\u0E49\u0E19\u0E43\u0E0A\u0E49\u0E15\ + \u0E32\u0E23\u0E32\u0E07\u0E04\u0E27\u0E32\u0E21\u0E08\u0E23\u0E34\u0E07\u0E40\ + \u0E1E\u0E37\u0E48\u0E2D\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\u0E27\u0E48\ + \u0E32\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E19\u0E31\u0E49\u0E19\u0E21\ + \u0E35\u0E40\u0E2B\u0E15\u0E38\u0E1C\u0E25\u0E40\u0E17\u0E35\u0E22\u0E1A\u0E40\ + \u0E17\u0E48\u0E32\u0E2B\u0E23\u0E37\u0E2D\u0E02\u0E31\u0E14\u0E41\u0E22\u0E49\ + \u0E07\u0E01\u0E31\u0E19 \u0E2B\u0E32\u0E01\u0E44\u0E21\u0E48\u0E21\u0E35 \u0E43\ + \u0E2B\u0E49\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\u0E27\u0E48\u0E32\u0E2A\ + \u0E2D\u0E14\u0E04\u0E25\u0E49\u0E2D\u0E07\u0E2B\u0E23\u0E37\u0E2D\u0E44\u0E21\ + \u0E48\u0E2A\u0E2D\u0E14\u0E04\u0E25\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E19 \u0E1B\ + \u0E23\u0E31\u0E1A\u0E04\u0E33\u0E15\u0E2D\u0E1A\u0E02\u0E2D\u0E07\u0E04\u0E38\ + \u0E13 E \u2283 (F \xB7 E) \u0E41\u0E25\u0E30 ~E \xB7 F" + - input_choice_list: + A: "(L \u2022 H) \u2261 I" + B: "(L \u2022 H) \u2228 I" + C: "L \u2022 (H \u2228 I)" + D: "L \u2022 (\u0E2A\u0E39\u0E07 \u2283 R)" + input_correct_responses: + - B + input_question: "\u0E2A\u0E39\u0E15\u0E23\u0E43\u0E14\u0E02\u0E2D\u0E07 PL \u0E17\ + \u0E35\u0E48\u0E41\u0E2A\u0E14\u0E07\u0E2A\u0E31\u0E0D\u0E25\u0E31\u0E01\u0E29\ + \u0E13\u0E4C\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E04\u0E15\u0E48\ + \u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E44\u0E14\u0E49\u0E14\u0E35\u0E17\u0E35\ + \u0E48\u0E2A\u0E38\u0E14 \u0E40\u0E15\u0E48\u0E32\u0E21\u0E35\u0E2D\u0E32\u0E22\ + \u0E38\u0E22\u0E37\u0E19\u0E22\u0E32\u0E27\u0E41\u0E25\u0E30\u0E40\u0E1B\u0E47\ + \u0E19\u0E2A\u0E31\u0E15\u0E27\u0E4C\u0E17\u0E35\u0E48\u0E21\u0E35\u0E04\u0E27\ + \u0E32\u0E21\u0E2A\u0E38\u0E02 \u0E40\u0E27\u0E49\u0E19\u0E41\u0E15\u0E48\u0E1E\ + \u0E27\u0E01\u0E21\u0E31\u0E19\u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\ + \u0E1A\u0E32\u0E14\u0E40\u0E08\u0E47\u0E1A" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_formal_logic +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_formal_logic +task_alias: formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e74c9716882b5dbcd8790996a85310c3338de58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_global_facts.yaml @@ -0,0 +1,100 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E1C\u0E39\u0E49\u0E04\u0E19\u0E21\u0E31\u0E01\u0E08\u0E30\u0E21\u0E2D\ + \u0E07\u0E42\u0E25\u0E01\u0E43\u0E19\u0E41\u0E07\u0E48\u0E14\u0E35\u0E40\u0E01\ + \u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2D\u0E19\u0E32\u0E04\u0E15\u0E02\ + \u0E2D\u0E07\u0E15\u0E19\u0E40\u0E2D\u0E07\u0E41\u0E25\u0E30\u0E2D\u0E19\u0E32\ + \u0E04\u0E15\u0E02\u0E2D\u0E07\u0E0A\u0E32\u0E15\u0E34\u0E2B\u0E23\u0E37\u0E2D\ + \u0E42\u0E25\u0E01" + B: "\u0E1C\u0E39\u0E49\u0E04\u0E19\u0E21\u0E31\u0E01\u0E08\u0E30\u0E21\u0E2D\ + \u0E07\u0E42\u0E25\u0E01\u0E43\u0E19\u0E41\u0E07\u0E48\u0E14\u0E35\u0E40\u0E01\ + \u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2D\u0E19\u0E32\u0E04\u0E15\u0E02\ + \u0E2D\u0E07\u0E15\u0E19\u0E40\u0E2D\u0E07 \u0E41\u0E15\u0E48\u0E21\u0E2D\u0E07\ + \u0E43\u0E19\u0E41\u0E07\u0E48\u0E23\u0E49\u0E32\u0E22\u0E40\u0E01\u0E35\u0E48\ + \u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2D\u0E19\u0E32\u0E04\u0E15\u0E02\u0E2D\u0E07\ + \u0E0A\u0E32\u0E15\u0E34\u0E2B\u0E23\u0E37\u0E2D\u0E42\u0E25\u0E01" + C: "\u0E1C\u0E39\u0E49\u0E04\u0E19\u0E21\u0E31\u0E01\u0E21\u0E2D\u0E07\u0E42\ + \u0E25\u0E01\u0E43\u0E19\u0E41\u0E07\u0E48\u0E23\u0E49\u0E32\u0E22\u0E40\u0E01\ + \u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2D\u0E19\u0E32\u0E04\u0E15\u0E02\ + \u0E2D\u0E07\u0E15\u0E19\u0E40\u0E2D\u0E07 \u0E41\u0E15\u0E48\u0E21\u0E2D\u0E07\ + \u0E43\u0E19\u0E41\u0E07\u0E48\u0E14\u0E35\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\ + \u0E01\u0E31\u0E1A\u0E2D\u0E19\u0E32\u0E04\u0E15\u0E02\u0E2D\u0E07\u0E0A\u0E32\ + \u0E15\u0E34\u0E2B\u0E23\u0E37\u0E2D\u0E42\u0E25\u0E01" + D: "\u0E1C\u0E39\u0E49\u0E04\u0E19\u0E21\u0E31\u0E01\u0E08\u0E30\u0E21\u0E2D\ + \u0E07\u0E42\u0E25\u0E01\u0E43\u0E19\u0E41\u0E07\u0E48\u0E23\u0E49\u0E32\u0E22\ + \u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2D\u0E19\u0E32\u0E04\ + \u0E15\u0E02\u0E2D\u0E07\u0E15\u0E19\u0E40\u0E2D\u0E07\u0E41\u0E25\u0E30\u0E2D\ + \u0E19\u0E32\u0E04\u0E15\u0E02\u0E2D\u0E07\u0E0A\u0E32\u0E15\u0E34\u0E2B\u0E23\ + \u0E37\u0E2D\u0E42\u0E25\u0E01" + input_correct_responses: + - B + input_question: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E04\u0E39\u0E48\u0E43\ + \u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\ + \u0E08\u0E23\u0E34\u0E07\u0E17\u0E31\u0E49\u0E07\u0E04\u0E39\u0E48 (\u0E02\u0E49\ + \u0E2D\u0E21\u0E39\u0E25 \u0E13 \u0E1B\u0E35 2019)" + - input_choice_list: + A: 31% + B: 46% + C: 61% + D: 76% + input_correct_responses: + - B + input_question: "\u0E43\u0E19\u0E1B\u0E35 2019 \u0E04\u0E19\u0E2D\u0E40\u0E21\u0E23\ + \u0E34\u0E01\u0E31\u0E19\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13\u0E01\u0E35\u0E48\ + \u0E40\u0E1B\u0E2D\u0E23\u0E4C\u0E40\u0E0B\u0E47\u0E19\u0E15\u0E4C\u0E17\u0E35\ + \u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E27\u0E48\u0E32\u0E23\u0E31\u0E10\ + \u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\u0E19\u0E4C\u0E02\u0E2D\u0E07\u0E1B\ + \u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E19\u0E17\u0E38\u0E01\u0E04\u0E19" + - input_choice_list: + A: "\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19, \u0E40\u0E1E\u0E34\ + \u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + B: "\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E25\u0E14\u0E25\ + \u0E07" + C: "\u0E25\u0E14\u0E25\u0E07, \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\ + \u0E19" + D: "\u0E25\u0E14\u0E25\u0E07\u0E25\u0E14\u0E25\u0E07" + input_correct_responses: + - B + input_question: "\u0E13 \u0E1B\u0E35 2015 \u0E15\u0E31\u0E49\u0E07\u0E41\u0E15\ + \u0E48\u0E1B\u0E35 1990 \u0E1B\u0E48\u0E32\u0E21\u0E35 ____ \u0E43\u0E19\u0E22\ + \u0E38\u0E42\u0E23\u0E1B\u0E41\u0E25\u0E30\u0E21\u0E35 ____ \u0E43\u0E19\u0E41\ + \u0E2D\u0E1F\u0E23\u0E34\u0E01\u0E32\u0E41\u0E25\u0E30\u0E2D\u0E40\u0E21\u0E23\ + \u0E34\u0E01\u0E32" + - input_choice_list: + A: 38% + B: 53% + C: 68% + D: 83% + input_correct_responses: + - A + input_question: "\u0E43\u0E19\u0E1B\u0E35 2019 \u0E0A\u0E32\u0E27\u0E23\u0E31\u0E2A\ + \u0E40\u0E0B\u0E35\u0E22\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13\u0E01\u0E35\u0E48\ + \u0E40\u0E1B\u0E2D\u0E23\u0E4C\u0E40\u0E0B\u0E47\u0E19\u0E15\u0E4C\u0E17\u0E35\ + \u0E48\u0E01\u0E25\u0E48\u0E32\u0E27\u0E27\u0E48\u0E32\u0E01\u0E32\u0E23\u0E21\ + \u0E35\u0E2A\u0E37\u0E48\u0E2D\u0E40\u0E2A\u0E23\u0E35\u0E43\u0E19\u0E1B\u0E23\ + \u0E30\u0E40\u0E17\u0E28\u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E42\u0E14\u0E22\ + \u0E1B\u0E23\u0E32\u0E28\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E40\u0E0B\u0E47\ + \u0E19\u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\ + \u0E32\u0E25/\u0E23\u0E31\u0E10\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E34\u0E48\u0E07\ + \u0E2A\u0E33\u0E04\u0E31\u0E0D\u0E21\u0E32\u0E01" + - input_choice_list: + A: 80% + B: 60% + C: 40% + D: 20% + input_correct_responses: + - A + input_question: "\u0E13 \u0E1B\u0E35 2560 \u0E21\u0E35\u0E40\u0E14\u0E47\u0E01\ + \u0E2D\u0E32\u0E22\u0E38 1 \u0E02\u0E27\u0E1A\u0E17\u0E31\u0E48\u0E27\u0E42\u0E25\ + \u0E01\u0E01\u0E35\u0E48\u0E04\u0E19\u0E17\u0E35\u0E48\u0E44\u0E14\u0E49\u0E23\ + \u0E31\u0E1A\u0E01\u0E32\u0E23\u0E09\u0E35\u0E14\u0E27\u0E31\u0E04\u0E0B\u0E35\ + \u0E19\u0E1B\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E19\u0E42\u0E23\u0E04\u0E1A\u0E32\ + \u0E07\u0E0A\u0E19\u0E34\u0E14? *" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_global_facts +tag: mmlu_th_llama_other_tasks +task: mmlu_th_llama_global_facts +task_alias: global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e99ee252d2d2a4764532856288bb2e6f4742b03d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_biology.yaml @@ -0,0 +1,121 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E41\u0E1B\u0E25" + B: "\u0E01\u0E32\u0E23\u0E1C\u0E31\u0E19\u0E04\u0E33\u0E01\u0E23\u0E34\u0E22\ + \u0E32" + C: "\u0E01\u0E32\u0E23\u0E16\u0E48\u0E32\u0E22\u0E42\u0E2D\u0E19\u0E40\u0E09\ + \u0E1E\u0E32\u0E30\u0E17\u0E32\u0E07" + D: "\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E41\u0E1B\ + \u0E25\u0E07" + input_correct_responses: + - A + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\ + \u0E27\u0E34\u0E18\u0E35\u0E2A\u0E23\u0E49\u0E32\u0E07 recombinant DNA" + - input_choice_list: + A: "\u0E42\u0E14\u0E22\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\ + \u0E19\u0E04\u0E48\u0E32 pH \u0E17\u0E35\u0E48\u0E40\u0E2B\u0E21\u0E32\u0E30\ + \u0E2A\u0E21\u0E02\u0E2D\u0E07\u0E40\u0E2D\u0E19\u0E44\u0E0B\u0E21\u0E4C" + B: "\u0E42\u0E14\u0E22\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\ + \u0E19\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E02\u0E2D\u0E07\u0E40\u0E2D\ + \u0E19\u0E44\u0E0B\u0E21\u0E4C\u0E43\u0E19\u0E40\u0E0B\u0E25\u0E25\u0E4C" + C: "\u0E42\u0E14\u0E22\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\ + \u0E19\u0E23\u0E39\u0E1B\u0E23\u0E48\u0E32\u0E07\u0E02\u0E2D\u0E07\u0E42\u0E1B\ + \u0E23\u0E15\u0E35\u0E19" + D: "\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E01\u0E23\ + \u0E14\u0E2D\u0E30\u0E21\u0E34\u0E42\u0E19\u0E08\u0E32\u0E01\u0E15\u0E33\u0E41\ + \u0E2B\u0E19\u0E48\u0E07\u0E17\u0E35\u0E48\u0E43\u0E0A\u0E49\u0E07\u0E32\u0E19\ + \u0E2D\u0E22\u0E39\u0E48\u0E44\u0E21\u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\ + \u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E04\u0E27\u0E32\u0E21\u0E08\u0E33\ + \u0E40\u0E1E\u0E32\u0E30\u0E02\u0E2D\u0E07\u0E0B\u0E31\u0E1A\u0E2A\u0E40\u0E15\ + \u0E23\u0E15\u0E02\u0E2D\u0E07\u0E40\u0E2D\u0E19\u0E44\u0E0B\u0E21\u0E4C\u0E44\ + \u0E14\u0E49" + input_correct_responses: + - C + input_question: "\u0E01\u0E32\u0E23\u0E01\u0E25\u0E32\u0E22\u0E1E\u0E31\u0E19\u0E18\ + \u0E38\u0E4C\u0E43\u0E19\u0E40\u0E2D\u0E19\u0E44\u0E0B\u0E21\u0E4C\u0E02\u0E2D\ + \u0E07\u0E41\u0E1A\u0E04\u0E17\u0E35\u0E40\u0E23\u0E35\u0E22\u0E40\u0E1B\u0E25\ + \u0E35\u0E48\u0E22\u0E19\u0E01\u0E23\u0E14\u0E2D\u0E30\u0E21\u0E34\u0E42\u0E19\ + \u0E17\u0E35\u0E48\u0E21\u0E35\u0E02\u0E31\u0E49\u0E27\u0E01\u0E48\u0E2D\u0E19\ + \u0E2B\u0E19\u0E49\u0E32\u0E19\u0E35\u0E49\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\ + \u0E19\u0E01\u0E23\u0E14\u0E2D\u0E30\u0E21\u0E34\u0E42\u0E19\u0E17\u0E35\u0E48\ + \u0E44\u0E21\u0E48\u0E21\u0E35\u0E02\u0E31\u0E49\u0E27 \u0E01\u0E23\u0E14\u0E2D\ + \u0E30\u0E21\u0E34\u0E42\u0E19\u0E19\u0E35\u0E49\u0E2D\u0E22\u0E39\u0E48\u0E17\ + \u0E35\u0E48\u0E44\u0E0B\u0E15\u0E4C\u0E17\u0E35\u0E48\u0E2B\u0E48\u0E32\u0E07\ + \u0E44\u0E01\u0E25\u0E08\u0E32\u0E01\u0E44\u0E0B\u0E15\u0E4C\u0E17\u0E35\u0E48\ + \u0E43\u0E0A\u0E49\u0E07\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E2D\u0E19\u0E44\ + \u0E0B\u0E21\u0E4C \u0E01\u0E32\u0E23\u0E01\u0E25\u0E32\u0E22\u0E1E\u0E31\u0E19\ + \u0E18\u0E38\u0E4C\u0E19\u0E35\u0E49\u0E2D\u0E32\u0E08\u0E40\u0E1B\u0E25\u0E35\ + \u0E48\u0E22\u0E19\u0E41\u0E1B\u0E25\u0E07\u0E04\u0E27\u0E32\u0E21\u0E08\u0E33\ + \u0E40\u0E1E\u0E32\u0E30\u0E02\u0E2D\u0E07\u0E0B\u0E31\u0E1A\u0E2A\u0E40\u0E15\ + \u0E23\u0E15\u0E02\u0E2D\u0E07\u0E40\u0E2D\u0E19\u0E44\u0E0B\u0E21\u0E4C\u0E44\ + \u0E14\u0E49\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E23" + - input_choice_list: + A: "\u0E1E\u0E25\u0E32\u0E2A\u0E21\u0E32\u0E40\u0E21\u0E21\u0E40\u0E1A\u0E23\ + \u0E19 \u2013 \u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E21\u0E37\u0E2D\ + \ Golgi \u2013 \u0E44\u0E23\u0E42\u0E1A\u0E42\u0E0B\u0E21 \u2013 \u0E16\u0E38\ + \u0E07\u0E2B\u0E25\u0E31\u0E48\u0E07 \u2013 ER \u0E2B\u0E22\u0E32\u0E1A" + B: "Ribosome\u2013Golgi apparatus\u2013Rough ER\u2013secretory vesicle\u2013\ + plasma membrane" + C: "\u0E1E\u0E25\u0E32\u0E2A\u0E21\u0E32\u0E40\u0E21\u0E21\u0E40\u0E1A\u0E23\ + \u0E19 \u2013 \u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E21\u0E37\u0E2D\ + \ Golgi \u2013 \u0E44\u0E23\u0E42\u0E1A\u0E42\u0E0B\u0E21 \u2013 \u0E16\u0E38\ + \u0E07\u0E2B\u0E25\u0E31\u0E48\u0E07 \u2013 ER \u0E2B\u0E22\u0E32\u0E1A" + D: "\u0E44\u0E23\u0E42\u0E1A\u0E42\u0E0B\u0E21-\u0E2B\u0E22\u0E32\u0E1A ER-Golgi\ + \ \u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E21\u0E37\u0E2D-\u0E16\u0E38\ + \u0E07\u0E2B\u0E25\u0E31\u0E48\u0E07-\u0E1E\u0E25\u0E32\u0E2A\u0E21\u0E32\u0E40\ + \u0E21\u0E21\u0E40\u0E1A\u0E23\u0E19" + input_correct_responses: + - D + input_question: "\u0E43\u0E19\u0E40\u0E0B\u0E25\u0E25\u0E4C\u0E2A\u0E31\u0E15\u0E27\ + \u0E4C \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\ + \u0E49\u0E41\u0E2A\u0E14\u0E07\u0E16\u0E36\u0E07\u0E27\u0E34\u0E16\u0E35\u0E17\ + \u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E44\u0E1B\u0E44\u0E14\u0E49\u0E21\u0E32\ + \u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E17\u0E35\u0E48\u0E42\u0E1B\u0E23\ + \u0E15\u0E35\u0E19\u0E17\u0E35\u0E48\u0E2B\u0E25\u0E31\u0E48\u0E07\u0E2D\u0E2D\ + \u0E01\u0E21\u0E32\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E16\u0E39\u0E01\u0E2A\u0E31\ + \u0E07\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\u0E43\u0E19\u0E40\u0E0B\u0E25\ + \u0E25\u0E4C" + - input_choice_list: + A: "\u0E44\u0E0B\u0E04\u0E25\u0E34\u0E19" + B: "\u0E42\u0E1B\u0E23\u0E15\u0E35\u0E19\u0E44\u0E04\u0E40\u0E19\u0E2A" + C: "\u0E08\u0E38\u0E14\u0E15\u0E23\u0E27\u0E08" + D: "\u0E40\u0E0B\u0E25\u0E25\u0E4C\u0E44\u0E1F\u0E42\u0E1A\u0E23\u0E1A\u0E25\ + \u0E32\u0E2A\u0E15\u0E4C" + input_correct_responses: + - D + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E44\u0E21\u0E48\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\ + \u0E2D\u0E07\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E04\u0E27\u0E1A\u0E04\u0E38\ + \u0E21\u0E01\u0E32\u0E23\u0E41\u0E1A\u0E48\u0E07\u0E40\u0E0B\u0E25\u0E25\u0E4C" + - input_choice_list: + A: "\u0E1B\u0E35\u0E01\u0E02\u0E2D\u0E07\u0E19\u0E01\u0E41\u0E25\u0E30\u0E1B\ + \u0E35\u0E01\u0E02\u0E2D\u0E07\u0E04\u0E49\u0E32\u0E07\u0E04\u0E32\u0E27" + B: "\u0E04\u0E23\u0E35\u0E1A\u0E1B\u0E25\u0E32\u0E27\u0E32\u0E2C\u0E41\u0E25\ + \u0E30\u0E41\u0E02\u0E19\u0E02\u0E2D\u0E07\u0E04\u0E19" + C: "\u0E04\u0E23\u0E35\u0E1A\u0E2D\u0E01\u0E02\u0E2D\u0E07\u0E1B\u0E25\u0E32\ + \u0E42\u0E25\u0E21\u0E32\u0E41\u0E25\u0E30\u0E04\u0E23\u0E35\u0E1A\u0E02\u0E2D\ + \u0E07\u0E41\u0E21\u0E27\u0E19\u0E49\u0E33" + D: "\u0E02\u0E32\u0E2B\u0E19\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E41\u0E21\u0E25\ + \u0E07\u0E41\u0E25\u0E30\u0E02\u0E32\u0E2B\u0E19\u0E49\u0E32\u0E02\u0E2D\u0E07\ + \u0E2A\u0E38\u0E19\u0E31\u0E02" + input_correct_responses: + - D + input_question: "\u0E42\u0E04\u0E23\u0E07\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E17\u0E35\ + \u0E48\u0E04\u0E25\u0E49\u0E32\u0E22\u0E04\u0E25\u0E36\u0E07\u0E01\u0E31\u0E19\ + \u0E21\u0E31\u0E01\u0E16\u0E39\u0E01\u0E2D\u0E49\u0E32\u0E07\u0E16\u0E36\u0E07\ + \u0E40\u0E1B\u0E47\u0E19\u0E2B\u0E25\u0E31\u0E01\u0E10\u0E32\u0E19\u0E2A\u0E33\ + \u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E23\u0E30\u0E1A\u0E27\u0E19\u0E01\u0E32\u0E23\ + \u0E04\u0E31\u0E14\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E42\u0E14\u0E22\u0E18\u0E23\ + \u0E23\u0E21\u0E0A\u0E32\u0E15\u0E34 \u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E15\ + \u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E02\u0E2D\u0E07\u0E42\u0E04\u0E23\ + \u0E07\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E17\u0E35\u0E48\u0E04\u0E25\u0E49\u0E32\ + \u0E22\u0E04\u0E25\u0E36\u0E07\u0E01\u0E31\u0E19 \u0E22\u0E01\u0E40\u0E27\u0E49\ + \u0E19" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_biology +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_high_school_biology +task_alias: high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d357b17c4981b53701afe5ac66411bdcaba8021 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_computer_science.yaml @@ -0,0 +1,198 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E23\u0E16\u0E08\u0E30\u0E40\u0E15\u0E37\u0E2D\u0E19\u0E04\u0E19\u0E02\ + \u0E31\u0E1A\u0E27\u0E48\u0E32\u0E01\u0E33\u0E25\u0E31\u0E07\u0E08\u0E30\u0E0A\ + \u0E19\u0E27\u0E31\u0E15\u0E16\u0E38" + B: "\u0E19\u0E31\u0E01\u0E1B\u0E35\u0E19\u0E40\u0E02\u0E32\u0E43\u0E0A\u0E49\ + \u0E19\u0E32\u0E2C\u0E34\u0E01\u0E32 GPS \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E15\ + \u0E34\u0E14\u0E15\u0E32\u0E21\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E02\ + \u0E2D\u0E07\u0E40\u0E18\u0E2D" + C: "\u0E15\u0E39\u0E49\u0E40\u0E22\u0E47\u0E19\u0E2A\u0E31\u0E48\u0E07\u0E19\ + \u0E21\u0E08\u0E32\u0E01\u0E1A\u0E23\u0E34\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\ + \u0E2A\u0E48\u0E07\u0E2D\u0E2D\u0E19\u0E44\u0E25\u0E19\u0E4C\u0E40\u0E21\u0E37\ + \u0E48\u0E2D\u0E19\u0E21\u0E43\u0E19\u0E15\u0E39\u0E49\u0E40\u0E22\u0E47\u0E19\ + \u0E43\u0E01\u0E25\u0E49\u0E08\u0E30\u0E2B\u0E21\u0E14" + D: "\u0E19\u0E31\u0E01\u0E27\u0E34\u0E48\u0E07\u0E43\u0E0A\u0E49\u0E19\u0E32\ + \u0E2C\u0E34\u0E01\u0E32\u0E17\u0E35\u0E48\u0E21\u0E35\u0E40\u0E0B\u0E47\u0E19\ + \u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E2D\u0E2D\u0E1B\u0E15\u0E34\u0E04\u0E31\u0E25\ + \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E15\u0E34\u0E14\u0E15\u0E32\u0E21\u0E2D\u0E31\ + \u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E15\u0E49\u0E19\u0E02\u0E2D\u0E07\ + \u0E2B\u0E31\u0E27\u0E43\u0E08" + input_correct_responses: + - C + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E04\u0E37\u0E2D\u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E2D\u0E38\u0E1B\u0E01\u0E23\u0E13\u0E4C\ + \u0E1A\u0E19 Internet of Things (IoT)" + - input_choice_list: + A: "\u0E01\u0E34\u0E08\u0E01\u0E23\u0E23\u0E21\u0E02\u0E2D\u0E07\u0E1C\u0E39\ + \u0E49\u0E43\u0E0A\u0E49\u0E17\u0E35\u0E48\u0E40\u0E23\u0E35\u0E22\u0E01\u0E14\ + \u0E39\u0E43\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E15\u0E48\u0E32\u0E07\u0E17\u0E35\ + \u0E48\u0E44\u0E21\u0E48\u0E23\u0E30\u0E1A\u0E38\u0E15\u0E31\u0E27\u0E15\u0E19\ + \u0E08\u0E30\u0E44\u0E21\u0E48\u0E1B\u0E23\u0E32\u0E01\u0E0F\u0E41\u0E01\u0E48\ + \u0E1A\u0E38\u0E04\u0E04\u0E25\u0E17\u0E35\u0E48\u0E15\u0E23\u0E27\u0E08\u0E2A\ + \u0E2D\u0E1A\u0E40\u0E04\u0E23\u0E37\u0E2D\u0E02\u0E48\u0E32\u0E22\u0E02\u0E2D\ + \u0E07\u0E1C\u0E39\u0E49\u0E43\u0E0A\u0E49 \u0E40\u0E0A\u0E48\u0E19 \u0E1C\ + \u0E39\u0E49\u0E14\u0E39\u0E41\u0E25\u0E23\u0E30\u0E1A\u0E1A" + B: "\u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\u0E17\u0E35\u0E48\u0E27\u0E32\u0E07\ + \u0E43\u0E19\u0E15\u0E30\u0E01\u0E23\u0E49\u0E32\u0E2A\u0E34\u0E19\u0E04\u0E49\ + \u0E32\u0E02\u0E2D\u0E07\u0E40\u0E27\u0E47\u0E1A\u0E2A\u0E42\u0E15\u0E23\u0E4C\ + \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E0B\u0E37\u0E49\u0E2D\ + \u0E43\u0E19\u0E2D\u0E19\u0E32\u0E04\u0E15\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\ + \u0E07\u0E40\u0E0B\u0E2A\u0E0A\u0E31\u0E19\u0E01\u0E32\u0E23\u0E40\u0E23\u0E35\ + \u0E22\u0E01\u0E14\u0E39\u0E41\u0E1A\u0E1A\u0E44\u0E21\u0E48\u0E23\u0E30\u0E1A\ + \u0E38\u0E15\u0E31\u0E27\u0E15\u0E19\u0E08\u0E30\u0E44\u0E21\u0E48\u0E16\u0E39\ + \u0E01\u0E1A\u0E31\u0E19\u0E17\u0E36\u0E01\u0E44\u0E27\u0E49\u0E43\u0E19\u0E04\ + \u0E2D\u0E21\u0E1E\u0E34\u0E27\u0E40\u0E15\u0E2D\u0E23\u0E4C\u0E02\u0E2D\u0E07\ + \u0E1C\u0E39\u0E49\u0E43\u0E0A\u0E49" + C: "\u0E1C\u0E39\u0E49\u0E43\u0E0A\u0E49\u0E08\u0E30\u0E44\u0E21\u0E48\u0E2A\ + \u0E32\u0E21\u0E32\u0E23\u0E16\u0E25\u0E07\u0E0A\u0E37\u0E48\u0E2D\u0E40\u0E02\ + \u0E49\u0E32\u0E43\u0E0A\u0E49\u0E1A\u0E31\u0E0D\u0E0A\u0E35\u0E2D\u0E35\u0E40\ + \u0E21\u0E25\u0E2B\u0E23\u0E37\u0E2D\u0E42\u0E0B\u0E40\u0E0A\u0E35\u0E22\u0E25\ + \u0E21\u0E35\u0E40\u0E14\u0E35\u0E22\u0E44\u0E14\u0E49\u0E43\u0E19\u0E23\u0E30\ + \u0E2B\u0E27\u0E48\u0E32\u0E07\u0E40\u0E0B\u0E2A\u0E0A\u0E31\u0E19\u0E01\u0E32\ + \u0E23\u0E17\u0E48\u0E2D\u0E07\u0E40\u0E27\u0E47\u0E1A\u0E41\u0E1A\u0E1A\u0E44\ + \u0E21\u0E48\u0E23\u0E30\u0E1A\u0E38\u0E15\u0E31\u0E27\u0E15\u0E19" + D: "\u0E1C\u0E39\u0E49\u0E43\u0E0A\u0E49\u0E17\u0E35\u0E48\u0E40\u0E23\u0E35\ + \u0E22\u0E01\u0E14\u0E39\u0E43\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E15\u0E48\u0E32\ + \u0E07\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E23\u0E30\u0E1A\u0E38\u0E0A\u0E37\ + \u0E48\u0E2D\u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\ + \u0E1B\u0E01\u0E1B\u0E49\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E44\u0E27\u0E23\u0E31\ + \u0E2A\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E34\u0E14\u0E15\u0E31\u0E27\u0E08\u0E32\ + \u0E01\u0E40\u0E27\u0E47\u0E1A\u0E44\u0E0B\u0E15\u0E4C\u0E17\u0E35\u0E48\u0E40\ + \u0E22\u0E35\u0E48\u0E22\u0E21\u0E0A\u0E21\u0E2B\u0E23\u0E37\u0E2D\u0E44\u0E1F\ + \u0E25\u0E4C\u0E17\u0E35\u0E48\u0E14\u0E32\u0E27\u0E19\u0E4C\u0E42\u0E2B\u0E25\ + \u0E14" + input_correct_responses: + - B + input_question: "\u0E40\u0E27\u0E47\u0E1A\u0E40\u0E1A\u0E23\u0E32\u0E27\u0E4C\u0E40\ + \u0E0B\u0E2D\u0E23\u0E4C\u0E08\u0E33\u0E19\u0E27\u0E19\u0E21\u0E32\u0E01\u0E2D\ + \u0E19\u0E38\u0E0D\u0E32\u0E15\u0E43\u0E2B\u0E49\u0E1C\u0E39\u0E49\u0E43\u0E0A\ + \u0E49\u0E40\u0E1B\u0E34\u0E14\u0E2B\u0E19\u0E49\u0E32\u0E15\u0E48\u0E32\u0E07\ + \u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E23\u0E30\u0E1A\u0E38\u0E15\u0E31\u0E27\ + \u0E15\u0E19 \u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E40\u0E0B\u0E2A\u0E0A\ + \u0E31\u0E19\u0E01\u0E32\u0E23\u0E40\u0E23\u0E35\u0E22\u0E01\u0E14\u0E39\u0E43\ + \u0E19\u0E2B\u0E19\u0E49\u0E32\u0E15\u0E48\u0E32\u0E07\u0E17\u0E35\u0E48\u0E44\ + \u0E21\u0E48\u0E23\u0E30\u0E1A\u0E38\u0E15\u0E31\u0E27\u0E15\u0E19 \u0E40\u0E1A\ + \u0E23\u0E32\u0E27\u0E4C\u0E40\u0E0B\u0E2D\u0E23\u0E4C\u0E08\u0E30\u0E44\u0E21\ + \u0E48\u0E1A\u0E31\u0E19\u0E17\u0E36\u0E01\u0E1B\u0E23\u0E30\u0E27\u0E31\u0E15\ + \u0E34\u0E01\u0E32\u0E23\u0E40\u0E23\u0E35\u0E22\u0E01\u0E14\u0E39\u0E2B\u0E23\ + \u0E37\u0E2D\u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\u0E44\u0E1F\u0E25\u0E4C\u0E17\ + \u0E35\u0E48\u0E14\u0E32\u0E27\u0E19\u0E4C\u0E42\u0E2B\u0E25\u0E14 \u0E40\u0E21\ + \u0E37\u0E48\u0E2D\u0E2D\u0E2D\u0E01\u0E08\u0E32\u0E01\u0E2B\u0E19\u0E49\u0E32\ + \u0E15\u0E48\u0E32\u0E07\u0E19\u0E34\u0E23\u0E19\u0E32\u0E21 \u0E04\u0E38\u0E01\ + \u0E01\u0E35\u0E49\u0E17\u0E35\u0E48\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E02\u0E36\ + \u0E49\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E40\u0E0B\u0E2A\u0E0A\ + \u0E31\u0E19\u0E08\u0E30\u0E16\u0E39\u0E01\u0E25\u0E1A \u0E02\u0E49\u0E2D\u0E43\ + \u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E01\u0E35\u0E48\ + \u0E22\u0E27\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E40\u0E23\u0E35\u0E22\u0E01\ + \u0E14\u0E39\u0E40\u0E0B\u0E2A\u0E0A\u0E31\u0E19\u0E43\u0E19\u0E2B\u0E19\u0E49\ + \u0E32\u0E15\u0E48\u0E32\u0E07\u0E19\u0E34\u0E23\u0E19\u0E32\u0E21\u0E17\u0E35\ + \u0E48\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07" + - input_choice_list: + A: "\u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14" + B: "\u0E40\u0E2D\u0E1A\u0E35\u0E0B\u0E35" + C: "\u0E2A.\u0E1A.\u0E17" + D: "\u0E04" + input_correct_responses: + - C + input_question: "\u0E1C\u0E25\u0E25\u0E31\u0E1E\u0E18\u0E4C\u0E02\u0E2D\u0E07\ + \ "abc"[::-1] \u0E43\u0E19 Python 3 \u0E04\u0E37\u0E2D\u0E2D\u0E30\ + \u0E44\u0E23" + - input_choice_list: + A: "\u0E1F\u0E47\u0E2D\u0E01\u0E0B\u0E4C\u0E17\u0E23\u0E47\u0E2D\u0E15" + B: "\u0E42\u0E23\u0E07\u0E41\u0E23\u0E21" + C: "\u0E1E\u0E24\u0E28\u0E08\u0E34\u0E01\u0E32\u0E22\u0E19" + D: "\u0E41\u0E22\u0E07\u0E01\u0E35\u0E49" + input_correct_responses: + - C + input_question: "\u0E43\u0E19\u0E42\u0E1B\u0E23\u0E41\u0E01\u0E23\u0E21\u0E14\u0E49\ + \u0E32\u0E19\u0E25\u0E48\u0E32\u0E07 \u0E04\u0E48\u0E32\u0E40\u0E23\u0E34\u0E48\ + \u0E21\u0E15\u0E49\u0E19\u0E02\u0E2D\u0E07 x \u0E04\u0E37\u0E2D 5 \u0E41\u0E25\ + \u0E30\u0E04\u0E48\u0E32\u0E40\u0E23\u0E34\u0E48\u0E21\u0E15\u0E49\u0E19\u0E02\ + \u0E2D\u0E07 y \u0E04\u0E37\u0E2D 10 IF (X < O) { DISPLAY ("Foxtrot")\ + \ } ELSE { IF (X > y) { DISPLAY ("\u0E42\u0E23\u0E07\u0E41\u0E23\u0E21\ + ") } ELSE { IF (y > O) { DISPLAY ("\u0E1E\u0E24\u0E28\u0E08\u0E34\ + \u0E01\u0E32\u0E22\u0E19") } ELSE { DISPLAY ("Yankee") } } }\ + \ \u0E2A\u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E41\u0E2A\u0E14\u0E07\u0E02\u0E36\ + \u0E49\u0E19\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E40\u0E23\u0E35\u0E22\u0E01\ + \u0E43\u0E0A\u0E49\u0E42\u0E1B\u0E23\u0E41\u0E01\u0E23\u0E21" + - input_choice_list: + A: "\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 3: \u0E40\u0E1E\ + \u0E34\u0E48\u0E21\u0E21\u0E39\u0E25\u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07\u0E15\ + \u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E17\u0E35\u0E25\u0E30 1 \u0E02\u0E31\ + \u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 4: \u0E17\u0E33\u0E0B\u0E49\ + \u0E33\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 2 \u0E41\ + \u0E25\u0E30 3 \u0E08\u0E19\u0E01\u0E27\u0E48\u0E32\u0E21\u0E39\u0E25\u0E04\ + \u0E48\u0E32\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E19\u0E31\u0E1A\u0E08\u0E30\ + \u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32 100" + B: "\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 3: \u0E40\u0E1E\ + \u0E34\u0E48\u0E21\u0E21\u0E39\u0E25\u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07\u0E15\ + \u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E02\u0E36\u0E49\u0E19 1 \u0E02\u0E31\ + \u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 4: \u0E17\u0E33\u0E0B\u0E49\ + \u0E33\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 2 \u0E41\ + \u0E25\u0E30 3 \u0E08\u0E19\u0E01\u0E27\u0E48\u0E32\u0E04\u0E48\u0E32\u0E02\ + \u0E2D\u0E07\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E08\u0E30\u0E21\u0E32\ + \u0E01\u0E01\u0E27\u0E48\u0E32 n" + C: "\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 3: \u0E17\u0E33\ + \u0E0B\u0E49\u0E33\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48\ + \ 2 \u0E08\u0E19\u0E01\u0E27\u0E48\u0E32\u0E21\u0E39\u0E25\u0E04\u0E48\u0E32\ + \u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E19\u0E31\u0E1A\u0E08\u0E30\u0E21\u0E32\ + \u0E01\u0E01\u0E27\u0E48\u0E32 100 \u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\ + \u0E17\u0E35\u0E48 4: \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E21\u0E39\u0E25\u0E04\ + \u0E48\u0E32\u0E02\u0E2D\u0E07\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E17\ + \u0E35\u0E25\u0E30 1" + D: "\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 3: \u0E17\u0E33\ + \u0E0B\u0E49\u0E33\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48\ + \ 2 \u0E08\u0E19\u0E01\u0E27\u0E48\u0E32\u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07\ + \u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E08\u0E30\u0E21\u0E32\u0E01\u0E01\ + \u0E27\u0E48\u0E32 n \u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\ + \u0E48 4: \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E21\u0E39\u0E25\u0E04\u0E48\u0E32\ + \u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E19\u0E31\u0E1A\u0E17\u0E35\u0E25\u0E30\ + \ 1" + input_correct_responses: + - D + input_question: "\u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\u0E15\u0E31\u0E27\u0E40\u0E25\ + \u0E02\u0E21\u0E35\u0E2D\u0E07\u0E04\u0E4C\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\ + \ n \u0E23\u0E32\u0E22\u0E01\u0E32\u0E23 \u0E08\u0E31\u0E14\u0E17\u0E33\u0E14\ + \u0E31\u0E0A\u0E19\u0E35\u0E15\u0E31\u0E49\u0E07\u0E41\u0E15\u0E48 1 \u0E16\u0E36\ + \u0E07 n \u0E2D\u0E31\u0E25\u0E01\u0E2D\u0E23\u0E34\u0E17\u0E36\u0E21\u0E15\u0E48\ + \u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E21\u0E35\u0E27\u0E31\u0E15\u0E16\u0E38\ + \u0E1B\u0E23\u0E30\u0E2A\u0E07\u0E04\u0E4C\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E41\ + \u0E2A\u0E14\u0E07\u0E08\u0E33\u0E19\u0E27\u0E19\u0E2D\u0E07\u0E04\u0E4C\u0E1B\ + \u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E43\u0E19\u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\ + \u0E17\u0E35\u0E48\u0E21\u0E35\u0E04\u0E48\u0E32\u0E21\u0E32\u0E01\u0E01\u0E27\ + \u0E48\u0E32 100 \u0E2D\u0E31\u0E25\u0E01\u0E2D\u0E23\u0E34\u0E17\u0E36\u0E21\ + \u0E43\u0E0A\u0E49\u0E01\u0E32\u0E23\u0E19\u0E31\u0E1A\u0E41\u0E25\u0E30\u0E15\ + \u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E02\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E41\ + \u0E1B\u0E23 \u0E44\u0E21\u0E48\u0E21\u0E35\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\ + \u0E19\u0E17\u0E35\u0E48 3 \u0E41\u0E25\u0E30 4 \u0E02\u0E31\u0E49\u0E19\u0E15\ + \u0E2D\u0E19\u0E17\u0E35\u0E48 1: \u0E15\u0E31\u0E49\u0E07\u0E04\u0E48\u0E32\ + \u0E01\u0E32\u0E23\u0E19\u0E31\u0E1A\u0E40\u0E1B\u0E47\u0E19 0 \u0E41\u0E25\u0E30\ + \u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E40\u0E1B\u0E47\u0E19 1 \u0E02\u0E31\ + \u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 2: \u0E2B\u0E32\u0E01\u0E04\ + \u0E48\u0E32\u0E02\u0E2D\u0E07\u0E2D\u0E07\u0E04\u0E4C\u0E1B\u0E23\u0E30\u0E01\ + \u0E2D\u0E1A\u0E17\u0E35\u0E48\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E14\ + \u0E31\u0E0A\u0E19\u0E35\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32 100 \u0E43\ + \u0E2B\u0E49\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07\ + \u0E01\u0E32\u0E23\u0E19\u0E31\u0E1A\u0E40\u0E1B\u0E47\u0E19 1 \u0E02\u0E31\u0E49\ + \u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 3: (\u0E02\u0E31\u0E49\u0E19\u0E15\ + \u0E2D\u0E19\u0E17\u0E35\u0E48\u0E2B\u0E32\u0E22\u0E44\u0E1B) \u0E02\u0E31\u0E49\ + \u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 4: (\u0E02\u0E31\u0E49\u0E19\u0E15\ + \u0E2D\u0E19\u0E17\u0E35\u0E48\u0E02\u0E32\u0E14\u0E2B\u0E32\u0E22\u0E44\u0E1B\ + \ ) \u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48 5: \u0E41\u0E2A\ + \u0E14\u0E07\u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E19\u0E31\ + \u0E1A \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\ + \u0E49\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E43\u0E0A\u0E49\u0E41\u0E17\u0E19\ + \u0E17\u0E35\u0E48\u0E02\u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48\ + \ 3 \u0E41\u0E25\u0E30 4 \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E2D\ + \u0E31\u0E25\u0E01\u0E2D\u0E23\u0E34\u0E17\u0E36\u0E21\u0E17\u0E33\u0E07\u0E32\ + \u0E19\u0E44\u0E14\u0E49\u0E15\u0E32\u0E21\u0E17\u0E35\u0E48\u0E15\u0E31\u0E49\ + \u0E07\u0E43\u0E08\u0E44\u0E27\u0E49" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_computer_science +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_high_school_computer_science +task_alias: high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..009d15a0510fc505f089c410ff30f4ea3fa316a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_european_history.yaml @@ -0,0 +1,748 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E0C\u0E2D\u0E07 \u0E0C\u0E32\u0E04\u0E2A\u0E4C \u0E23\u0E38\u0E2A\u0E42\ + \u0E0B" + B: "\u0E1A\u0E32\u0E23\u0E2D\u0E19 \u0E21\u0E2D\u0E07\u0E40\u0E15\u0E2A\u0E01\ + \u0E34\u0E40\u0E2D\u0E2D" + C: Mary Wollstonecraft + D: "\u0E2D\u0E14\u0E31\u0E21 \u0E2A\u0E21\u0E34\u0E18" + input_correct_responses: + - B + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E02\u0E49\u0E2D\u0E04\u0E27\ + \u0E32\u0E21\u0E17\u0E35\u0E48\u0E15\u0E31\u0E14\u0E15\u0E2D\u0E19\u0E21\u0E32\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E21\u0E32\u0E08\u0E32\u0E01\ + \u0E08\u0E38\u0E25\u0E2A\u0E32\u0E23 \u0E04\u0E38\u0E13\u0E08\u0E30\u0E43\u0E2B\ + \u0E49\u0E04\u0E27\u0E32\u0E21\u0E22\u0E38\u0E15\u0E34\u0E18\u0E23\u0E23\u0E21\ + \u0E41\u0E01\u0E48\u0E09\u0E31\u0E19\u0E43\u0E19\u0E01\u0E32\u0E23\u0E08\u0E14\ + \u0E08\u0E33\u0E27\u0E48\u0E32\u0E09\u0E31\u0E19\u0E44\u0E14\u0E49\u0E2A\u0E19\ + \u0E31\u0E1A\u0E2A\u0E19\u0E38\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E41\u0E02\ + \u0E47\u0E07\u0E02\u0E31\u0E19\u0E40\u0E2A\u0E21\u0E2D\u0E21\u0E32\u0E43\u0E19\ + \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E02\u0E2D\u0E07\u0E17\u0E38\u0E01\u0E04\u0E19\ + \u0E43\u0E19\u0E04\u0E27\u0E32\u0E21\u0E04\u0E34\u0E14\u0E40\u0E2B\u0E47\u0E19\ + \u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32\u0E40\u0E2D\u0E07 \u0E44\u0E21\u0E48\u0E27\ + \u0E48\u0E32\u0E04\u0E27\u0E32\u0E21\u0E04\u0E34\u0E14\u0E40\u0E2B\u0E47\u0E19\ + \u0E19\u0E31\u0E49\u0E19\u0E08\u0E30\u0E41\u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\ + \u0E01\u0E31\u0E19\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E09\u0E31\u0E19\u0E01\ + \u0E47\u0E15\u0E32\u0E21 \u0E1C\u0E39\u0E49\u0E17\u0E35\u0E48\u0E1B\u0E0F\u0E34\ + \u0E40\u0E2A\u0E18\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E19\u0E35\u0E49 \u0E17\u0E33\ + \u0E43\u0E2B\u0E49\u0E15\u0E19\u0E40\u0E2D\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E17\ + \u0E32\u0E2A\u0E15\u0E48\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E04\u0E34\u0E14\u0E40\ + \u0E2B\u0E47\u0E19\u0E43\u0E19\u0E1B\u0E31\u0E08\u0E08\u0E38\u0E1A\u0E31\u0E19\ + \ \u0E40\u0E1E\u0E23\u0E32\u0E30\u0E40\u0E02\u0E32\u0E1B\u0E34\u0E14\u0E01\u0E31\ + \u0E49\u0E19\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E43\u0E19\u0E01\u0E32\u0E23\u0E40\ + \u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E41\u0E1B\u0E25\u0E07 \u0E2D\u0E32\u0E27\ + \u0E38\u0E18\u0E17\u0E35\u0E48\u0E19\u0E48\u0E32\u0E01\u0E25\u0E31\u0E27\u0E17\ + \u0E35\u0E48\u0E2A\u0E38\u0E14\u0E43\u0E19\u0E01\u0E32\u0E23\u0E15\u0E48\u0E2D\ + \u0E15\u0E49\u0E32\u0E19\u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\ + \u0E14\u0E17\u0E38\u0E01\u0E0A\u0E19\u0E34\u0E14\u0E04\u0E37\u0E2D\u0E40\u0E2B\ + \u0E15\u0E38\u0E1C\u0E25 \u0E09\u0E31\u0E19\u0E44\u0E21\u0E48\u0E40\u0E04\u0E22\ + \u0E43\u0E0A\u0E49\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2D\u0E37\u0E48\u0E19\u0E41\ + \u0E25\u0E30\u0E09\u0E31\u0E19\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E27\u0E48\u0E32\ + \u0E09\u0E31\u0E19\u0E08\u0E30\u0E44\u0E21\u0E48\u0E17\u0E33 \u0E2A\u0E16\u0E32\ + \u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\ + \u0E02\u0E36\u0E49\u0E19\u0E43\u0E19\u0E1D\u0E23\u0E31\u0E48\u0E07\u0E40\u0E28\ + \u0E2A\u0E43\u0E19\u0E02\u0E13\u0E30\u0E19\u0E35\u0E49\u0E40\u0E01\u0E35\u0E48\ + \u0E22\u0E27\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E22\u0E01\u0E40\u0E25\u0E34\ + \u0E01\u0E23\u0E30\u0E40\u0E1A\u0E35\u0E22\u0E1A\u0E10\u0E32\u0E19\u0E30\u0E1B\ + \u0E38\u0E42\u0E23\u0E2B\u0E34\u0E15\u0E41\u0E2B\u0E48\u0E07\u0E0A\u0E32\u0E15\ + \u0E34\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 \u0E41\u0E25\u0E30\u0E17\u0E38\ + \u0E01\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E17\u0E35\u0E48\u0E40\u0E01\u0E35\u0E48\ + \u0E22\u0E27\u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E1A\u0E23\u0E30\u0E1A\u0E1A\ + \u0E1A\u0E31\u0E07\u0E04\u0E31\u0E1A\u0E02\u0E2D\u0E07\u0E28\u0E32\u0E2A\u0E19\ + \u0E32 \u0E41\u0E25\u0E30\u0E2B\u0E25\u0E31\u0E01\u0E04\u0E27\u0E32\u0E21\u0E40\ + \u0E0A\u0E37\u0E48\u0E2D\u0E17\u0E35\u0E48\u0E1A\u0E35\u0E1A\u0E1A\u0E31\u0E07\ + \u0E04\u0E31\u0E1A \u0E44\u0E21\u0E48\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E17\u0E33\ + \u0E43\u0E2B\u0E49\u0E04\u0E27\u0E32\u0E21\u0E15\u0E31\u0E49\u0E07\u0E43\u0E08\ + \u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E15\u0E01\u0E15\u0E30\u0E01\u0E2D\u0E19\ + \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19 \u0E41\u0E15\u0E48\u0E22\u0E31\ + \u0E07\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E07\u0E32\u0E19\u0E19\u0E35\u0E49\u0E2A\ + \u0E33\u0E40\u0E23\u0E47\u0E08\u0E14\u0E49\u0E27\u0E22 \u0E14\u0E49\u0E27\u0E22\ + \u0E04\u0E27\u0E32\u0E21\u0E01\u0E23\u0E38\u0E13\u0E32\u0E17\u0E35\u0E48\u0E08\ + \u0E33\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E22\u0E34\u0E48\ + \u0E07 \u0E40\u0E01\u0E23\u0E07\u0E27\u0E48\u0E32\u0E04\u0E27\u0E32\u0E21\u0E2B\ + \u0E32\u0E22\u0E19\u0E30\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E02\u0E2D\u0E07\ + \u0E04\u0E27\u0E32\u0E21\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E42\u0E0A\u0E04\u0E25\ + \u0E32\u0E07 \u0E23\u0E30\u0E1A\u0E1A\u0E01\u0E32\u0E23\u0E1B\u0E01\u0E04\u0E23\ + \u0E2D\u0E07\u0E17\u0E35\u0E48\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14 \u0E41\ + \u0E25\u0E30\u0E40\u0E17\u0E27\u0E27\u0E34\u0E17\u0E22\u0E32\u0E40\u0E17\u0E47\ + \u0E08 \u0E40\u0E23\u0E32\u0E08\u0E30\u0E21\u0E2D\u0E07\u0E44\u0E21\u0E48\u0E40\ + \u0E2B\u0E47\u0E19\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21 \u0E21\u0E19\u0E38\ + \u0E29\u0E22\u0E0A\u0E32\u0E15\u0E34 \u0E41\u0E25\u0E30\u0E40\u0E17\u0E27\u0E27\ + \u0E34\u0E17\u0E22\u0E32\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E27\ + \u0E32\u0E21\u0E08\u0E23\u0E34\u0E07 \u0E09\u0E31\u0E19\u0E40\u0E0A\u0E37\u0E48\ + \u0E2D\u0E43\u0E19\u0E1E\u0E23\u0E30\u0E40\u0E08\u0E49\u0E32\u0E2D\u0E07\u0E04\ + \u0E4C\u0E40\u0E14\u0E35\u0E22\u0E27\u0E41\u0E25\u0E30\u0E44\u0E21\u0E48\u0E40\ + \u0E0A\u0E37\u0E48\u0E2D\u0E2D\u0E35\u0E01\u0E15\u0E48\u0E2D\u0E44\u0E1B \u0E41\ + \u0E25\u0E30\u0E09\u0E31\u0E19\u0E2B\u0E27\u0E31\u0E07\u0E27\u0E48\u0E32\u0E08\ + \u0E30\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E38\u0E02\u0E43\u0E19\u0E0A\ + \u0E35\u0E27\u0E34\u0E15\u0E19\u0E35\u0E49 \u0E09\u0E31\u0E19\u0E40\u0E0A\u0E37\ + \u0E48\u0E2D\u0E43\u0E19\u0E04\u0E27\u0E32\u0E21\u0E40\u0E17\u0E48\u0E32\u0E40\ + \u0E17\u0E35\u0E22\u0E21\u0E01\u0E31\u0E19\u0E02\u0E2D\u0E07\u0E21\u0E19\u0E38\ + \u0E29\u0E22\u0E4C \u0E41\u0E25\u0E30\u0E02\u0E49\u0E32\u0E1E\u0E40\u0E08\u0E49\ + \u0E32\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E27\u0E48\u0E32\u0E2B\u0E19\u0E49\u0E32\ + \u0E17\u0E35\u0E48\u0E17\u0E32\u0E07\u0E28\u0E32\u0E2A\u0E19\u0E32\u0E1B\u0E23\ + \u0E30\u0E01\u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E01\u0E32\u0E23\u0E17\u0E33\ + \u0E04\u0E27\u0E32\u0E21\u0E22\u0E38\u0E15\u0E34\u0E18\u0E23\u0E23\u0E21 \u0E23\ + \u0E31\u0E01\u0E04\u0E27\u0E32\u0E21\u0E40\u0E21\u0E15\u0E15\u0E32 \u0E41\u0E25\ + \u0E30\u0E1E\u0E22\u0E32\u0E22\u0E32\u0E21\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E40\ + \u0E1E\u0E37\u0E48\u0E2D\u0E19\u0E23\u0E48\u0E27\u0E21\u0E42\u0E25\u0E01\u0E21\ + \u0E35\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E38\u0E02 \u0E09\u0E31\u0E19\u0E44\u0E21\ + \u0E48\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E43\u0E19\u0E25\u0E31\u0E17\u0E18\u0E34\ + \u0E17\u0E35\u0E48\u0E1B\u0E23\u0E30\u0E01\u0E32\u0E28\u0E42\u0E14\u0E22\u0E04\ + \u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E22\u0E34\u0E27 \u0E04\u0E23\ + \u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E42\u0E23\u0E21\u0E31\u0E19 \u0E04\ + \u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E01\u0E23\u0E35\u0E01 \u0E04\ + \u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E15\u0E38\u0E23\u0E01\u0E35\ + \ \u0E04\u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E42\u0E1B\u0E23\u0E40\ + \u0E15\u0E2A\u0E41\u0E15\u0E19\u0E15\u0E4C \u0E2B\u0E23\u0E37\u0E2D\u0E04\u0E23\ + \u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E43\u0E14 \u0E46 \u0E17\u0E35\u0E48\ + \u0E09\u0E31\u0E19\u0E23\u0E39\u0E49\u0E08\u0E31\u0E01 \u0E08\u0E34\u0E15\u0E43\ + \u0E08\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E04\u0E37\u0E2D\u0E04\u0E23\u0E34\ + \u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E40\ + \u0E2D\u0E07 \u0E2A\u0E16\u0E32\u0E1A\u0E31\u0E19\u0E04\u0E23\u0E34\u0E2A\u0E15\ + \u0E08\u0E31\u0E01\u0E23\u0E23\u0E30\u0E14\u0E31\u0E1A\u0E0A\u0E32\u0E15\u0E34\ + \u0E17\u0E38\u0E01\u0E41\u0E2B\u0E48\u0E07 \u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\ + \u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E22\u0E34\u0E27 \u0E04\u0E23\u0E34\u0E2A\ + \u0E15\u0E4C \u0E2B\u0E23\u0E37\u0E2D\u0E15\u0E38\u0E23\u0E01\u0E35 \u0E44\u0E21\ + \u0E48\u0E1B\u0E23\u0E32\u0E01\u0E0F\u0E41\u0E01\u0E48\u0E02\u0E49\u0E32\u0E1E\ + \u0E40\u0E08\u0E49\u0E32\u0E19\u0E2D\u0E01\u0E08\u0E32\u0E01\u0E2A\u0E34\u0E48\ + \u0E07\u0E1B\u0E23\u0E30\u0E14\u0E34\u0E29\u0E10\u0E4C\u0E02\u0E2D\u0E07\u0E21\ + \u0E19\u0E38\u0E29\u0E22\u0E4C \u0E08\u0E31\u0E14\u0E15\u0E31\u0E49\u0E07\u0E02\ + \u0E36\u0E49\u0E19\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E2A\u0E23\u0E49\u0E32\u0E07\ + \u0E04\u0E27\u0E32\u0E21\u0E2B\u0E27\u0E32\u0E14\u0E01\u0E25\u0E31\u0E27\u0E41\ + \u0E25\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E32\u0E2A\u0E02\u0E2D\u0E07\u0E21\ + \u0E19\u0E38\u0E29\u0E22\u0E0A\u0E32\u0E15\u0E34 \u0E41\u0E25\u0E30\u0E1C\u0E39\ + \u0E01\u0E02\u0E32\u0E14\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E41\u0E25\u0E30\u0E1C\ + \u0E25\u0E01\u0E33\u0E44\u0E23 \u0E02\u0E49\u0E32\u0E1E\u0E40\u0E08\u0E49\u0E32\ + \u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E2B\u0E21\u0E32\u0E22\u0E04\u0E27\u0E32\ + \u0E21\u0E15\u0E32\u0E21\u0E04\u0E33\u0E1B\u0E23\u0E30\u0E01\u0E32\u0E28\u0E19\ + \u0E35\u0E49\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E1B\u0E23\u0E30\u0E13\u0E32\u0E21\ + \u0E1C\u0E39\u0E49\u0E17\u0E35\u0E48\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E40\u0E1B\ + \u0E47\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2D\u0E37\u0E48\u0E19 \u0E1E\u0E27\ + \u0E01\u0E40\u0E02\u0E32\u0E21\u0E35\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C\u0E43\ + \u0E19\u0E04\u0E27\u0E32\u0E21\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E02\u0E2D\u0E07\ + \u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E40\u0E0A\u0E48\u0E19\u0E40\u0E14\u0E35\ + \u0E22\u0E27\u0E01\u0E31\u0E1A\u0E09\u0E31\u0E19 \u2014Thomas Paine, The Age\ + \ of Reason, 1794\u20131795 \u0E1B\u0E23\u0E31\u0E0A\u0E0D\u0E32\u0E01\u0E32\ + \u0E23\u0E23\u0E39\u0E49\u0E41\u0E08\u0E49\u0E07\u0E02\u0E49\u0E2D\u0E43\u0E14\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E2D\u0E2D\u0E01\u0E41\u0E1A\ + \u0E1A\u0E23\u0E30\u0E1A\u0E1A\u0E01\u0E32\u0E23\u0E15\u0E23\u0E27\u0E08\u0E2A\ + \u0E2D\u0E1A\u0E41\u0E25\u0E30\u0E16\u0E48\u0E27\u0E07\u0E14\u0E38\u0E25\u0E2A\ + \u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E40\u0E1E\ + \u0E37\u0E48\u0E2D\u0E2B\u0E25\u0E35\u0E01\u0E40\u0E25\u0E35\u0E48\u0E22\u0E07\ + \u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E42\u0E14\ + \u0E22\u0E21\u0E34\u0E0A\u0E2D\u0E1A" + - input_choice_list: + A: "\u0E41\u0E19\u0E27\u0E04\u0E34\u0E14\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\ + \u0E01\u0E31\u0E1A\u0E40\u0E2A\u0E23\u0E35\u0E20\u0E32\u0E1E\u0E2A\u0E48\u0E27\ + \u0E19\u0E1A\u0E38\u0E04\u0E04\u0E25\u0E41\u0E25\u0E30\u0E25\u0E31\u0E17\u0E18\ + \u0E34\u0E0A\u0E32\u0E15\u0E34\u0E19\u0E34\u0E22\u0E21\u0E17\u0E35\u0E48\u0E40\ + \u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\ + \u0E22\u0E38\u0E04\u0E15\u0E23\u0E31\u0E2A\u0E23\u0E39\u0E49\u0E2A\u0E48\u0E07\ + \u0E1C\u0E25\u0E43\u0E2B\u0E49\u0E40\u0E01\u0E34\u0E14\u0E01\u0E32\u0E23\u0E1B\ + \u0E0F\u0E34\u0E27\u0E31\u0E15\u0E34\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E23\u0E38\ + \u0E19\u0E41\u0E23\u0E07\u0E0B\u0E36\u0E48\u0E07\u0E2D\u0E32\u0E08\u0E41\u0E1E\ + \u0E23\u0E48\u0E01\u0E23\u0E30\u0E08\u0E32\u0E22\u0E44\u0E1B\u0E17\u0E31\u0E48\ + \u0E27\u0E22\u0E38\u0E42\u0E23\u0E1B" + B: "\u0E01\u0E32\u0E23\u0E1E\u0E34\u0E0A\u0E34\u0E15\u0E22\u0E38\u0E42\u0E23\ + \u0E1B\u0E42\u0E14\u0E22\u0E19\u0E42\u0E1B\u0E40\u0E25\u0E35\u0E22\u0E19\u0E19\ + \u0E33\u0E44\u0E1B\u0E2A\u0E39\u0E48\u0E01\u0E32\u0E23\u0E2A\u0E23\u0E49\u0E32\ + \u0E07\u0E01\u0E25\u0E38\u0E48\u0E21\u0E43\u0E2B\u0E21\u0E48\u0E41\u0E25\u0E30\ + \u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E14\u0E38\u0E25\u0E2D\u0E33\u0E19\ + \u0E32\u0E08\u0E02\u0E2D\u0E07\u0E22\u0E38\u0E42\u0E23\u0E1B" + C: "\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E02\u0E2D\u0E07\u0E1E\u0E23\u0E30\u0E21\ + \u0E2B\u0E32\u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E40\u0E15\u0E34\ + \u0E1A\u0E42\u0E15\u0E02\u0E36\u0E49\u0E19\u0E08\u0E19\u0E16\u0E36\u0E07\u0E08\ + \u0E38\u0E14\u0E17\u0E35\u0E48\u0E08\u0E33\u0E40\u0E1B\u0E47\u0E19\u0E15\u0E49\ + \u0E2D\u0E07\u0E15\u0E23\u0E27\u0E08\u0E2A\u0E2D\u0E1A\u0E42\u0E14\u0E22\u0E2D\ + \u0E33\u0E19\u0E32\u0E08\u0E2D\u0E37\u0E48\u0E19 \u0E46 \u0E43\u0E19\u0E41\ + \u0E15\u0E48\u0E25\u0E30\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28 \u0E21\u0E34\u0E09\ + \u0E30\u0E19\u0E31\u0E49\u0E19\u0E01\u0E32\u0E23\u0E04\u0E23\u0E2D\u0E1A\u0E07\ + \u0E33\u0E02\u0E2D\u0E07\u0E1E\u0E25\u0E40\u0E23\u0E37\u0E2D\u0E19\u0E08\u0E30\ + \u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19" + D: "\u0E27\u0E07\u0E08\u0E23\u0E40\u0E28\u0E23\u0E29\u0E10\u0E01\u0E34\u0E08\ + \u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E41\ + \u0E25\u0E30\u0E25\u0E14\u0E25\u0E07\u0E02\u0E2D\u0E07\u0E23\u0E30\u0E1A\u0E1A\ + \u0E40\u0E28\u0E23\u0E29\u0E10\u0E01\u0E34\u0E08\u0E17\u0E38\u0E19\u0E19\u0E34\ + \u0E22\u0E21\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\ + \u0E43\u0E2B\u0E21\u0E48\u0E2D\u0E32\u0E08\u0E19\u0E33\u0E44\u0E1B\u0E2A\u0E39\ + \u0E48\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E2A\u0E07\u0E1A\u0E02\u0E2D\ + \u0E07\u0E1E\u0E25\u0E40\u0E23\u0E37\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E15\u0E49\ + \u0E2D\u0E07\u0E16\u0E39\u0E01\u0E23\u0E30\u0E07\u0E31\u0E1A" + input_correct_responses: + - A + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E2D\u0E48\u0E32\u0E19\u0E02\ + \u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E17\u0E35\u0E48\u0E15\u0E31\u0E14\u0E15\ + \u0E2D\u0E19\u0E21\u0E32\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E40\ + \u0E21\u0E25\u0E47\u0E14\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E4C\u0E41\u0E2B\u0E48\ + \u0E07\u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E27\u0E31\u0E15\u0E34\u0E44\u0E14\ + \u0E49\u0E41\u0E17\u0E23\u0E01\u0E0B\u0E36\u0E21\u0E40\u0E02\u0E49\u0E32\u0E44\ + \u0E1B\u0E43\u0E19\u0E17\u0E38\u0E01\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E41\ + \u0E25\u0E30\u0E41\u0E1E\u0E23\u0E48\u0E01\u0E23\u0E30\u0E08\u0E32\u0E22\u0E44\ + \u0E21\u0E48\u0E21\u0E32\u0E01\u0E01\u0E47\u0E19\u0E49\u0E2D\u0E22 \u0E21\u0E31\ + \u0E19\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E1E\u0E31\u0E12\ + \u0E19\u0E32\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E21\u0E32\u0E01\u0E20\u0E32\u0E22\ + \u0E43\u0E15\u0E49\u0E23\u0E30\u0E1A\u0E2D\u0E1A\u0E40\u0E1C\u0E14\u0E47\u0E08\ + \u0E01\u0E32\u0E23\u0E17\u0E2B\u0E32\u0E23\u0E02\u0E2D\u0E07\u0E42\u0E1A\u0E19\ + \u0E32\u0E1B\u0E32\u0E23\u0E4C\u0E15 \u0E01\u0E32\u0E23\u0E1E\u0E34\u0E0A\u0E34\ + \u0E15\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E01\ + \u0E0E\u0E2B\u0E21\u0E32\u0E22 \u0E2A\u0E16\u0E32\u0E1A\u0E31\u0E19 \u0E41\u0E25\ + \u0E30\u0E02\u0E19\u0E1A\u0E18\u0E23\u0E23\u0E21\u0E40\u0E19\u0E35\u0E22\u0E21\ + \u0E2B\u0E25\u0E32\u0E22\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E40\u0E02\u0E49\u0E32\ + \u0E21\u0E32\u0E41\u0E17\u0E19\u0E17\u0E35\u0E48 \u0E17\u0E33\u0E25\u0E32\u0E22\ + \u0E1E\u0E31\u0E19\u0E18\u0E19\u0E32\u0E01\u0E32\u0E23\u0E2D\u0E31\u0E19\u0E28\ + \u0E31\u0E01\u0E14\u0E34\u0E4C\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C\u0E02\u0E2D\ + \u0E07\u0E17\u0E38\u0E01\u0E0A\u0E32\u0E15\u0E34 \u0E41\u0E02\u0E47\u0E07\u0E41\ + \u0E01\u0E23\u0E48\u0E07\u0E1E\u0E2D\u0E17\u0E35\u0E48\u0E08\u0E30\u0E15\u0E49\ + \u0E32\u0E19\u0E17\u0E32\u0E19\u0E01\u0E32\u0E25\u0E40\u0E27\u0E25\u0E32\u0E44\ + \u0E14\u0E49 \u0E0B\u0E36\u0E48\u0E07\u0E21\u0E32\u0E01\u0E40\u0E01\u0E34\u0E19\ + \u0E01\u0E27\u0E48\u0E32\u0E08\u0E30\u0E01\u0E25\u0E48\u0E32\u0E27\u0E44\u0E14\ + \u0E49\u0E27\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E25\u0E1B\u0E23\u0E30\ + \u0E42\u0E22\u0E0A\u0E19\u0E4C\u0E1A\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E01\u0E32\ + \u0E23\u0E17\u0E35\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E08\u0E32\u0E01\ + \u0E19\u0E31\u0E01\u0E1B\u0E23\u0E30\u0E14\u0E34\u0E29\u0E10\u0E4C\u0E40\u0E2B\ + \u0E25\u0E48\u0E32\u0E19\u0E35\u0E49 \u0E1E\u0E23\u0E30\u0E21\u0E2B\u0E32\u0E01\ + \u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E08\u0E30\u0E1B\u0E0F\u0E34\u0E1A\ + \u0E31\u0E15\u0E34\u0E15\u0E32\u0E21\u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\ + \u0E17\u0E35\u0E48\u0E1E\u0E23\u0E30\u0E2D\u0E07\u0E04\u0E4C\u0E44\u0E14\u0E49\ + \u0E17\u0E23\u0E07\u0E21\u0E2D\u0E1A\u0E2B\u0E21\u0E32\u0E22\u0E44\u0E27\u0E49\ + \u0E41\u0E01\u0E48\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E42\u0E14\u0E22\u0E21\ + \u0E2D\u0E1A\u0E2B\u0E21\u0E32\u0E22\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E43\u0E2B\ + \u0E49\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E14\u0E39\u0E41\u0E25\u0E01\u0E32\ + \u0E23\u0E23\u0E31\u0E01\u0E29\u0E32\u0E04\u0E27\u0E32\u0E21\u0E22\u0E38\u0E15\ + \u0E34\u0E18\u0E23\u0E23\u0E21\u0E41\u0E25\u0E30\u0E2A\u0E34\u0E17\u0E18\u0E34\ + \u0E02\u0E2D\u0E07\u0E17\u0E38\u0E01\u0E04\u0E19\u0E40\u0E1E\u0E37\u0E48\u0E2D\ + \u0E2B\u0E25\u0E35\u0E01\u0E40\u0E25\u0E35\u0E48\u0E22\u0E07\u0E40\u0E2A\u0E49\ + \u0E19\u0E17\u0E32\u0E07\u0E41\u0E2B\u0E48\u0E07\u0E04\u0E27\u0E32\u0E21\u0E1C\ + \u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\u0E41\u0E25\u0E30\u0E40\u0E14\u0E34\u0E19\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E21\u0E31\u0E48\u0E19\u0E04\u0E07\u0E43\u0E19\ + \u0E17\u0E32\u0E07\u0E41\u0E2B\u0E48\u0E07 \u0E04\u0E27\u0E32\u0E21\u0E08\u0E23\ + \u0E34\u0E07. \u0E2D\u0E22\u0E39\u0E48\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E01\u0E34\ + \u0E40\u0E25\u0E2A\u0E15\u0E31\u0E13\u0E2B\u0E32\u0E17\u0E35\u0E48\u0E1B\u0E31\ + \u0E48\u0E19\u0E1B\u0E48\u0E27\u0E19\u0E2A\u0E31\u0E07\u0E04\u0E21 \u0E43\u0E19\ + \u0E22\u0E38\u0E04\u0E41\u0E2B\u0E48\u0E07\u0E01\u0E32\u0E23\u0E17\u0E14\u0E25\ + \u0E2D\u0E07 \u0E2A\u0E48\u0E27\u0E19\u0E43\u0E2B\u0E0D\u0E48\u0E1E\u0E27\u0E01\ + \u0E40\u0E02\u0E32\u0E16\u0E39\u0E01\u0E40\u0E23\u0E35\u0E22\u0E01\u0E23\u0E49\ + \u0E2D\u0E07\u0E43\u0E2B\u0E49\u0E17\u0E33\u0E25\u0E32\u0E22\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07\u0E02\u0E2D\u0E07\u0E23\ + \u0E39\u0E1B\u0E25\u0E31\u0E01\u0E29\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E1C\u0E34\ + \u0E14\u0E46 \u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32 \u0E41\u0E25\ + \u0E30\u0E41\u0E2A\u0E14\u0E07\u0E15\u0E19\u0E15\u0E32\u0E21\u0E17\u0E35\u0E48\ + \u0E40\u0E1B\u0E47\u0E19 \u0E1A\u0E34\u0E14\u0E32\u0E25\u0E07\u0E17\u0E38\u0E19\ + \u0E01\u0E31\u0E1A\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E17\u0E35\u0E48\u0E40\u0E1B\ + \u0E47\u0E19\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E02\u0E2D\u0E07\u0E2B\u0E31\u0E27\ + \u0E2B\u0E19\u0E49\u0E32\u0E04\u0E23\u0E2D\u0E1A\u0E04\u0E23\u0E31\u0E27 \u0E40\ + \u0E1E\u0E37\u0E48\u0E2D\u0E1E\u0E34\u0E2A\u0E39\u0E08\u0E19\u0E4C\u0E27\u0E48\ + \u0E32\u0E43\u0E19\u0E27\u0E31\u0E19\u0E41\u0E2B\u0E48\u0E07\u0E04\u0E27\u0E32\ + \u0E21\u0E42\u0E28\u0E01\u0E40\u0E28\u0E23\u0E49\u0E32 \u0E1E\u0E27\u0E01\u0E40\ + \u0E02\u0E32\u0E23\u0E39\u0E49\u0E27\u0E34\u0E18\u0E35\u0E17\u0E35\u0E48\u0E08\ + \u0E30\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E19\u0E22\u0E38\u0E15\u0E34\u0E18\u0E23\ + \u0E23\u0E21 \u0E09\u0E25\u0E32\u0E14 \u0E41\u0E25\u0E30\u0E40\u0E02\u0E49\u0E21\ + \u0E41\u0E02\u0E47\u0E07 \u0E41\u0E25\u0E30\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\ + \u0E08\u0E30\u0E44\u0E21\u0E48\u0E25\u0E30\u0E17\u0E34\u0E49\u0E07\u0E1C\u0E39\ + \u0E49\u0E04\u0E19\u0E17\u0E35\u0E48\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E04\ + \u0E27\u0E23\u0E1B\u0E01\u0E04\u0E23\u0E2D\u0E07\u0E43\u0E2B\u0E49\u0E40\u0E1B\ + \u0E47\u0E19\u0E01\u0E35\u0E2C\u0E32\u0E02\u0E2D\u0E07\u0E1D\u0E31\u0E01\u0E1D\ + \u0E48\u0E32\u0E22 \u0E2A\u0E39\u0E48\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\ + \u0E1E\u0E25\u0E32\u0E14\u0E41\u0E25\u0E30\u0E1C\u0E25\u0E17\u0E35\u0E48\u0E15\ + \u0E32\u0E21\u0E21\u0E32 \u0E0B\u0E36\u0E48\u0E07\u0E08\u0E30\u0E15\u0E49\u0E2D\ + \u0E07 \u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\ + \u0E1A\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E39\u0E0D\u0E40\u0E2A\u0E35\u0E22\u0E02\ + \u0E2D\u0E07\u0E2A\u0E31\u0E07\u0E04\u0E21 \u0E2A\u0E2B\u0E20\u0E32\u0E1E\u0E23\ + \u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E1E\u0E23\u0E30\u0E21\u0E2B\u0E32\u0E01\ + \u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E37\ + \u0E49\u0E19\u0E10\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E19\u0E42\u0E22\u0E1A\u0E32\ + \u0E22\u0E17\u0E35\u0E48\u0E15\u0E49\u0E2D\u0E07\u0E1B\u0E0F\u0E34\u0E1A\u0E31\ + \u0E15\u0E34\u0E15\u0E32\u0E21\u0E43\u0E19\u0E02\u0E13\u0E30\u0E19\u0E35\u0E49\ + \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E0A\u0E48\u0E27\u0E22\u0E2A\u0E31\u0E07\u0E04\ + \u0E21\u0E08\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E34\u0E19\u0E32\u0E28\ + \u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 . . . \u0E2D\u0E22\u0E48\u0E32\u0E1B\ + \u0E25\u0E48\u0E2D\u0E22\u0E43\u0E2B\u0E49\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\ + \u0E2A\u0E31\u0E1A\u0E2A\u0E19\u0E01\u0E31\u0E1A\u0E2A\u0E31\u0E21\u0E1B\u0E17\ + \u0E32\u0E19\u0E17\u0E35\u0E48\u0E17\u0E33\u0E01\u0E31\u0E1A\u0E1D\u0E48\u0E32\ + \u0E22\u0E15\u0E48\u0E32\u0E07\u0E46 \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E1B\u0E23\ + \u0E30\u0E42\u0E22\u0E0A\u0E19\u0E4C\u0E17\u0E35\u0E48\u0E1E\u0E27\u0E01\u0E40\ + \u0E02\u0E32\u0E04\u0E27\u0E23\u0E08\u0E30\u0E17\u0E33\u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E19\u0E02\u0E2D\u0E07\u0E1E\u0E27\ + \u0E01\u0E40\u0E02\u0E32 \u0E43\u0E19\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E31\u0E1A\ + \u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E2A\u0E32\u0E02\u0E32\u0E02\u0E2D\ + \u0E07\u0E01\u0E32\u0E23\u0E1A\u0E23\u0E34\u0E2B\u0E32\u0E23\u0E15\u0E32\u0E21\ + \u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E17\u0E35\ + \u0E48\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E35\u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\ + \u0E1A\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E15\u0E32\u0E21\ + \u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23 \u0E43\u0E2B\ + \u0E49\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E19\ + \u0E40\u0E17\u0E35\u0E48\u0E22\u0E07\u0E18\u0E23\u0E23\u0E21\u0E41\u0E15\u0E48\ + \u0E40\u0E02\u0E49\u0E21\u0E41\u0E02\u0E47\u0E07 \u0E43\u0E08\u0E14\u0E35 \u0E41\ + \u0E15\u0E48\u0E40\u0E02\u0E49\u0E21\u0E07\u0E27\u0E14 \u0E1B\u0E25\u0E48\u0E2D\ + \u0E22\u0E43\u0E2B\u0E49\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E23\u0E31\u0E01\ + \u0E29\u0E32\u0E2B\u0E25\u0E31\u0E01\u0E01\u0E32\u0E23\u0E17\u0E32\u0E07\u0E28\ + \u0E32\u0E2A\u0E19\u0E32\u0E14\u0E49\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E1A\ + \u0E23\u0E34\u0E2A\u0E38\u0E17\u0E18\u0E34\u0E4C\u0E17\u0E31\u0E49\u0E07\u0E2B\ + \u0E21\u0E14 \u0E2D\u0E22\u0E48\u0E32\u0E43\u0E2B\u0E49\u0E04\u0E27\u0E32\u0E21\ + \u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E16\u0E39\u0E01\u0E42\u0E08\u0E21\u0E15\u0E35\ + \u0E41\u0E25\u0E30\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21\u0E16\u0E39\u0E01\ + \u0E15\u0E35\u0E04\u0E27\u0E32\u0E21\u0E15\u0E32\u0E21\u0E2A\u0E31\u0E0D\u0E0D\ + \u0E32\u0E17\u0E32\u0E07\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E2B\u0E23\u0E37\u0E2D\ + \u0E27\u0E34\u0E2A\u0E31\u0E22\u0E17\u0E31\u0E28\u0E19\u0E4C\u0E02\u0E2D\u0E07\ + \u0E19\u0E34\u0E01\u0E32\u0E22\u0E17\u0E35\u0E48\u0E42\u0E07\u0E48\u0E40\u0E02\ + \u0E25\u0E32 \u0E43\u0E2B\u0E49\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E1B\u0E23\ + \u0E32\u0E1A\u0E1B\u0E23\u0E32\u0E21\u0E2A\u0E21\u0E32\u0E04\u0E21\u0E25\u0E31\ + \u0E1A \u0E04\u0E27\u0E32\u0E21\u0E40\u0E19\u0E48\u0E32\u0E40\u0E1F\u0E30\u0E02\ + \u0E2D\u0E07\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E19\u0E31\u0E49\u0E19 \u2014Klemens\ + \ von Metternich, Political Confession of Faith, 1820 \u0E02\u0E49\u0E2D\u0E43\ + \u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E17\u0E35\u0E48\u0E40\ + \u0E1B\u0E47\u0E19\u0E2A\u0E32\u0E40\u0E2B\u0E15\u0E38\u0E43\u0E2B\u0E0D\u0E48\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\ + \u0E01\u0E25\u0E31\u0E27\u0E17\u0E35\u0E48 Metternich \u0E41\u0E2A\u0E14\u0E07\ + \u0E44\u0E27\u0E49\u0E43\u0E19\u0E40\u0E2D\u0E01\u0E2A\u0E32\u0E23\u0E02\u0E49\ + \u0E32\u0E07\u0E15\u0E49\u0E19" + - input_choice_list: + A: "\u0E19\u0E32\u0E22\u0E17\u0E38\u0E19" + B: "\u0E27\u0E34\u0E17\u0E22\u0E32\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C" + C: "\u0E04\u0E2D\u0E21\u0E21\u0E34\u0E27\u0E19\u0E34\u0E2A\u0E15\u0E4C" + D: "\u0E2D\u0E31\u0E15\u0E16\u0E34\u0E20\u0E32\u0E27\u0E19\u0E34\u0E22\u0E21" + input_correct_responses: + - C + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E43\u0E19\u0E23\u0E31\u0E2A\ + \u0E40\u0E0B\u0E35\u0E22\u0E44\u0E21\u0E48\u0E21\u0E35\u0E2D\u0E30\u0E44\u0E23\ + \u0E14\u0E35\u0E40\u0E25\u0E22 \u0E41\u0E25\u0E30 [Souvarine] \u0E01\u0E47\u0E2A\ + \u0E34\u0E49\u0E19\u0E2B\u0E27\u0E31\u0E07\u0E01\u0E31\u0E1A\u0E02\u0E48\u0E32\ + \u0E27\u0E17\u0E35\u0E48\u0E40\u0E02\u0E32\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\ + \ \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E19\u0E40\u0E01\u0E48\u0E32\u0E02\u0E2D\u0E07\ + \u0E40\u0E02\u0E32\u0E2B\u0E31\u0E19\u0E44\u0E1B\u0E2B\u0E32\u0E19\u0E31\u0E01\ + \u0E01\u0E32\u0E23\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E01\u0E31\u0E19\u0E2B\u0E21\ + \u0E14 \u0E1E\u0E27\u0E01 Nihilists \u0E17\u0E35\u0E48\u0E21\u0E35\u0E0A\u0E37\ + \u0E48\u0E2D\u0E40\u0E2A\u0E35\u0E22\u0E07\u0E17\u0E35\u0E48\u0E17\u0E33\u0E43\ + \u0E2B\u0E49\u0E22\u0E38\u0E42\u0E23\u0E1B\u0E15\u0E49\u0E2D\u0E07\u0E2A\u0E31\ + \u0E48\u0E19\u0E2A\u0E30\u0E17\u0E49\u0E32\u0E19 - \u0E1A\u0E38\u0E15\u0E23\u0E0A\ + \u0E32\u0E22\u0E02\u0E2D\u0E07\u0E19\u0E31\u0E01\u0E1A\u0E27\u0E0A\u0E1B\u0E23\ + \u0E30\u0E08\u0E33\u0E2B\u0E21\u0E39\u0E48\u0E1A\u0E49\u0E32\u0E19, \u0E0A\u0E19\ + \u0E0A\u0E31\u0E49\u0E19\u0E01\u0E25\u0E32\u0E07\u0E23\u0E30\u0E14\u0E31\u0E1A\ + \u0E25\u0E48\u0E32\u0E07, \u0E1E\u0E48\u0E2D\u0E04\u0E49\u0E32 - \u0E44\u0E21\ + \u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E2D\u0E22\u0E39\u0E48\u0E40\u0E2B\ + \u0E19\u0E37\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E04\u0E34\u0E14\u0E40\u0E23\u0E37\ + \u0E48\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E1B\u0E25\u0E14\u0E1B\u0E25\u0E48\u0E2D\ + \u0E22\u0E0A\u0E32\u0E15\u0E34\u0E44\u0E14\u0E49 \u0E41\u0E25\u0E30\u0E14\u0E39\ + \u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E08\u0E30\u0E40\u0E0A\u0E37\u0E48\u0E2D\ + \u0E27\u0E48\u0E32\u0E42\u0E25\u0E01\u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\ + \u0E1A\u0E01\u0E32\u0E23\u0E1B\u0E25\u0E14\u0E1B\u0E25\u0E48\u0E2D\u0E22 - \u0E40\ + \u0E21\u0E37\u0E48\u0E2D\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E2A\u0E31\u0E07\ + \u0E2B\u0E32\u0E23\u0E40\u0E1C\u0E14\u0E47\u0E08\u0E01\u0E32\u0E23\u0E02\u0E2D\ + \u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32&... "\u0E04\u0E19\u0E42\u0E07\ + \u0E48\u0E40\u0E02\u0E25\u0E32! \u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E08\u0E30\ + \u0E44\u0E21\u0E48\u0E21\u0E35\u0E27\u0E31\u0E19\u0E2B\u0E25\u0E38\u0E14\u0E1E\ + \u0E49\u0E19\u0E08\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E42\u0E07\u0E48\u0E40\ + \u0E02\u0E25\u0E32\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32"\ + \ \u0E08\u0E32\u0E01\u0E19\u0E31\u0E49\u0E19 \u0E25\u0E14\u0E40\u0E2A\u0E35\u0E22\ + \u0E07\u0E25\u0E07\u0E2D\u0E35\u0E01 \u0E1A\u0E23\u0E23\u0E22\u0E32\u0E22\u0E04\ + \u0E27\u0E32\u0E21\u0E1D\u0E31\u0E19\u0E40\u0E01\u0E48\u0E32 \u0E46 \u0E02\u0E2D\ + \u0E07\u0E40\u0E02\u0E32\u0E43\u0E19\u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E04\ + \u0E27\u0E32\u0E21\u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E35\u0E48\u0E19\u0E49\u0E2D\ + \u0E07 \u0E40\u0E02\u0E32\u0E2A\u0E25\u0E30\u0E22\u0E28\u0E41\u0E25\u0E30\u0E17\ + \u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E2A\u0E21\u0E1A\u0E31\u0E15\u0E34\u0E02\u0E2D\ + \u0E07\u0E40\u0E02\u0E32\u0E41\u0E25\u0E49\u0E27 \u0E40\u0E02\u0E32\u0E44\u0E1B\ + \u0E43\u0E19\u0E2B\u0E21\u0E39\u0E48\u0E04\u0E19\u0E07\u0E32\u0E19\u0E40\u0E1E\ + \u0E35\u0E22\u0E07\u0E14\u0E49\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E27\ + \u0E31\u0E07\u0E27\u0E48\u0E32\u0E08\u0E30\u0E44\u0E14\u0E49\u0E40\u0E2B\u0E47\ + \u0E19\u0E23\u0E32\u0E01\u0E10\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E2A\u0E31\u0E07\ + \u0E04\u0E21\u0E41\u0E23\u0E07\u0E07\u0E32\u0E19\u0E43\u0E2B\u0E21\u0E48\u0E17\ + \u0E35\u0E48\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E01\u0E31\u0E19\u0E43\u0E19\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14 Sous \u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\ + \u0E14\u0E43\u0E19\u0E01\u0E23\u0E30\u0E40\u0E1B\u0E4B\u0E32\u0E02\u0E2D\u0E07\ + \u0E40\u0E02\u0E32\u0E44\u0E1B\u0E17\u0E35\u0E48\u0E40\u0E21\u0E48\u0E19\u0E17\ + \u0E30\u0E40\u0E25\u0E02\u0E2D\u0E07\u0E19\u0E34\u0E04\u0E21\u0E21\u0E32\u0E19\ + \u0E32\u0E19\u0E41\u0E25\u0E49\u0E27 \u0E40\u0E02\u0E32\u0E2D\u0E48\u0E2D\u0E19\ + \u0E42\u0E22\u0E19\u0E23\u0E32\u0E27\u0E01\u0E31\u0E1A\u0E40\u0E1B\u0E47\u0E19\ + \u0E1E\u0E35\u0E48\u0E19\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E1A\u0E04\u0E19\u0E07\ + \u0E32\u0E19\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E07 \u0E40\u0E02\u0E32\u0E22\u0E34\ + \u0E49\u0E21\u0E43\u0E2B\u0E49\u0E01\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E2A\ + \u0E07\u0E2A\u0E31\u0E22\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\ + \ \u0E40\u0E2D\u0E32\u0E0A\u0E19\u0E30\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E14\ + \u0E49\u0E27\u0E22\u0E27\u0E34\u0E18\u0E35\u0E01\u0E32\u0E23\u0E17\u0E33\u0E07\ + \u0E32\u0E19\u0E41\u0E1A\u0E1A\u0E04\u0E19\u0E17\u0E33\u0E07\u0E32\u0E19\u0E40\ + \u0E07\u0E35\u0E22\u0E1A\u0E46 \u0E41\u0E25\u0E30\u0E40\u0E02\u0E32\u0E44\u0E21\ + \u0E48\u0E0A\u0E2D\u0E1A\u0E1E\u0E39\u0E14\u0E1E\u0E25\u0E48\u0E32\u0E21 \u0E41\ + \u0E15\u0E48\u0E1F\u0E34\u0E27\u0E0A\u0E31\u0E48\u0E19\u0E44\u0E21\u0E48\u0E44\ + \u0E14\u0E49\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E2D\u0E22\u0E48\ + \u0E32\u0E07\u0E41\u0E19\u0E48\u0E19\u0E2D\u0E19 \u0E19\u0E49\u0E33\u0E40\u0E2A\ + \u0E35\u0E22\u0E07\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32\u0E40\u0E1B\u0E25\u0E35\ + \u0E48\u0E22\u0E19\u0E44\u0E1B \u0E14\u0E27\u0E07\u0E15\u0E32\u0E02\u0E2D\u0E07\ + \u0E40\u0E02\u0E32\u0E2A\u0E14\u0E43\u0E2A\u0E02\u0E36\u0E49\u0E19 \u0E40\u0E02\ + \u0E32\u0E08\u0E31\u0E1A\u0E08\u0E49\u0E2D\u0E07\u0E44\u0E1B\u0E17\u0E35\u0E48\ + \ \xE9tienne \u0E41\u0E25\u0E49\u0E27\u0E1E\u0E39\u0E14\u0E01\u0E31\u0E1A\u0E40\ + \u0E02\u0E32\u0E42\u0E14\u0E22\u0E15\u0E23\u0E07: "\u0E15\u0E2D\u0E19\u0E19\ + \u0E35\u0E49 \u0E04\u0E38\u0E13\u0E40\u0E02\u0E49\u0E32\u0E43\u0E08\u0E41\u0E25\ + \u0E49\u0E27\u0E43\u0E0A\u0E48\u0E44\u0E2B\u0E21 \u0E04\u0E19\u0E17\u0E33\u0E2B\ + \u0E21\u0E27\u0E01\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E35\u0E49\u0E17\u0E35\ + \u0E48 Marseilles \u0E17\u0E35\u0E48\u0E16\u0E39\u0E01\u0E23\u0E32\u0E07\u0E27\ + \u0E31\u0E25\u0E25\u0E2D\u0E15\u0E40\u0E15\u0E2D\u0E23\u0E35\u0E48\u0E23\u0E32\ + \u0E07\u0E27\u0E31\u0E25\u0E43\u0E2B\u0E0D\u0E48\u0E21\u0E39\u0E25\u0E04\u0E48\ + \u0E32 1 \u0E41\u0E2A\u0E19\u0E1F\u0E23\u0E31\u0E07\u0E01\u0E4C \u0E44\u0E14\ + \u0E49\u0E2D\u0E2D\u0E01\u0E44\u0E1B\u0E17\u0E31\u0E19\u0E17\u0E35\u0E41\u0E25\ + \u0E30\u0E25\u0E07\u0E17\u0E38\u0E19 \u0E21\u0E31\u0E19\u0E1B\u0E23\u0E30\u0E01\ + \u0E32\u0E28\u0E27\u0E48\u0E32\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E08\u0E30\ + \u0E2D\u0E22\u0E39\u0E48\u0E42\u0E14\u0E22\u0E44\u0E21\u0E48\u0E17\u0E33\u0E2D\ + \u0E30\u0E44\u0E23\u0E40\u0E25\u0E22 \u0E43\u0E0A\u0E48 \u0E19\u0E31\u0E48\u0E19\ + \u0E40\u0E1B\u0E47\u0E19\u0E04\u0E27\u0E32\u0E21\u0E04\u0E34\u0E14\u0E02\u0E2D\ + \u0E07\u0E04\u0E38\u0E13 \u0E04\u0E19\u0E07\u0E32\u0E19\u0E1D\u0E23\u0E31\u0E48\ + \u0E07\u0E40\u0E28\u0E2A \u0E17\u0E38\u0E01\u0E04\u0E19 \u0E04\u0E38\u0E13\u0E2D\ + \u0E22\u0E32\u0E01\u0E08\u0E30\u0E02\u0E38\u0E14\u0E2A\u0E21\u0E1A\u0E31\u0E15\ + \u0E34\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E01\u0E34\u0E19\u0E21\u0E31\u0E19\u0E04\ + \u0E19\u0E40\u0E14\u0E35\u0E22\u0E27\u0E43\u0E19\u0E20\u0E32\u0E22\u0E2B\u0E25\ + \u0E31\u0E07 \u0E43\u0E19\u0E21\u0E38\u0E21\u0E02\u0E35\u0E49\u0E40\u0E01\u0E35\ + \u0E22\u0E08 \u0E40\u0E2B\u0E47\u0E19\u0E41\u0E01\u0E48\u0E15\u0E31\u0E27 \u0E04\ + \u0E38\u0E13\u0E2D\u0E32\u0E08\u0E23\u0E49\u0E2D\u0E07\u0E44\u0E2B\u0E49\u0E2D\ + \u0E2D\u0E01\u0E21\u0E32 \u0E21\u0E32\u0E01\u0E40\u0E17\u0E48\u0E32\u0E17\u0E35\ + \u0E48\u0E04\u0E38\u0E13\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E01\u0E31\ + \u0E1A\u0E04\u0E19\u0E23\u0E27\u0E22 \u0E04\u0E38\u0E13\u0E44\u0E21\u0E48\u0E21\ + \u0E35\u0E04\u0E27\u0E32\u0E21\u0E01\u0E25\u0E49\u0E32\u0E1E\u0E2D\u0E17\u0E35\ + \u0E48\u0E08\u0E30\u0E04\u0E37\u0E19\u0E40\u0E07\u0E34\u0E19\u0E17\u0E35\u0E48\ + \u0E42\u0E0A\u0E04\u0E14\u0E35\u0E19\u0E33\u0E21\u0E32\u0E43\u0E2B\u0E49\u0E04\ + \u0E38\u0E13 \u0E04\u0E19\u0E08\u0E19 \u0E04\u0E38\u0E13\u0E08\u0E30\u0E44\u0E21\ + \u0E48\u0E21\u0E35\u0E17\u0E32\u0E07\u0E04\u0E39\u0E48\u0E04\u0E27\u0E23\u0E01\ + \u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E38\u0E02\u0E15\u0E23\u0E32\u0E1A\ + \u0E40\u0E17\u0E48\u0E32\u0E17\u0E35\u0E48\u0E04\u0E38\u0E13\u0E40\u0E1B\u0E47\ + \u0E19\u0E40\u0E08\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E2A\u0E34\u0E48\u0E07\u0E43\ + \u0E14 \u0E46 \u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E40\u0E01\u0E25\u0E35\ + \u0E22\u0E14\u0E0A\u0E31\u0E07\u0E0A\u0E19\u0E0A\u0E31\u0E49\u0E19\u0E19\u0E32\ + \u0E22\u0E17\u0E38\u0E19\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13 \u0E08\u0E32\u0E01\ + \u0E04\u0E27\u0E32\u0E21\u0E1B\u0E23\u0E32\u0E23\u0E16\u0E19\u0E32\u0E2D\u0E31\ + \u0E19\u0E41\u0E23\u0E07\u0E01\u0E25\u0E49\u0E32\u0E17\u0E35\u0E48\u0E08\u0E30\ + \u0E40\u0E1B\u0E47\u0E19\u0E0A\u0E19\u0E0A\u0E31\u0E49\u0E19\u0E19\u0E32\u0E22\ + \u0E17\u0E38\u0E19\u0E41\u0E17\u0E19\u0E15\u0E19\u0E40\u0E17\u0E48\u0E32\u0E19\ + \u0E31\u0E49\u0E19\u201D \xE9mile Zola, \u0E19\u0E31\u0E01\u0E40\u0E02\u0E35\ + \u0E22\u0E19\u0E0A\u0E32\u0E27\u0E1D\u0E23\u0E31\u0E48\u0E07\u0E40\u0E28\u0E2A\ + , Germinal, 1885 \u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E19\u0E35\u0E49\ + \u0E41\u0E2A\u0E14\u0E07\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E48\u0E27\u0E07\u0E43\ + \u0E22\u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E15\u0E48\u0E2D\u0E2A\u0E27\u0E31\ + \u0E2A\u0E14\u0E34\u0E20\u0E32\u0E1E\u0E02\u0E2D\u0E07\u0E0A\u0E19\u0E0A\u0E31\ + \u0E49\u0E19\u0E41\u0E23\u0E07\u0E07\u0E32\u0E19 \u0E0B\u0E36\u0E48\u0E07\u0E42\ + \u0E14\u0E22\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E41\u0E25\u0E49\u0E27\u0E40\ + \u0E1B\u0E47\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E02\ + \u0E2D\u0E07\u0E02\u0E1A\u0E27\u0E19\u0E01\u0E32\u0E23\u0E43\u0E14?" + - input_choice_list: + A: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E17\u0E33\u0E2B\u0E19\u0E49\u0E32\ + \u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E15\u0E31\u0E27\u0E40\u0E23\u0E48\ + \u0E07\u0E01\u0E32\u0E23\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15\u0E02\u0E2D\u0E07\ + \u0E01\u0E32\u0E23\u0E02\u0E19\u0E2A\u0E48\u0E07\u0E17\u0E32\u0E07\u0E40\u0E23\ + \u0E37\u0E2D\u0E02\u0E2D\u0E07\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E41\u0E25\ + \u0E30\u0E01\u0E32\u0E23\u0E04\u0E49\u0E32\u0E43\u0E19\u0E15\u0E48\u0E32\u0E07\ + \u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28 \u0E41\u0E15\u0E48\u0E01\u0E47\u0E41\u0E17\ + \u0E1A\u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E08\u0E33\u0E01\u0E31\u0E14\u0E42\ + \u0E2D\u0E01\u0E32\u0E2A\u0E02\u0E2D\u0E07\u0E0A\u0E32\u0E27\u0E14\u0E31\u0E15\ + \u0E0A\u0E4C\u0E43\u0E19\u0E28\u0E15\u0E27\u0E23\u0E23\u0E29\u0E17\u0E35\u0E48\ + \u0E2A\u0E34\u0E1A\u0E40\u0E08\u0E47\u0E14" + B: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E19\u0E33\u0E21\u0E32\u0E0B\u0E36\ + \u0E48\u0E07\u0E04\u0E27\u0E32\u0E21\u0E22\u0E32\u0E01\u0E25\u0E33\u0E1A\u0E32\ + \u0E01\u0E40\u0E01\u0E37\u0E2D\u0E1A\u0E08\u0E30\u0E43\u0E19\u0E17\u0E31\u0E19\ + \u0E17\u0E35\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E40\u0E28\u0E23\u0E29\u0E10\ + \u0E01\u0E34\u0E08\u0E02\u0E2D\u0E07\u0E40\u0E19\u0E40\u0E18\u0E2D\u0E23\u0E4C\ + \u0E41\u0E25\u0E19\u0E14\u0E4C\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\ + \u0E01\u0E01\u0E32\u0E23\u0E04\u0E23\u0E2D\u0E1A\u0E07\u0E33\u0E01\u0E32\u0E23\ + \u0E04\u0E49\u0E32\u0E43\u0E19\u0E15\u0E48\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E40\ + \u0E17\u0E28\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E2A\u0E34\ + \u0E49\u0E19\u0E2A\u0E38\u0E14\u0E25\u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E23\ + \u0E27\u0E14\u0E40\u0E23\u0E47\u0E27" + C: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E16\u0E39\u0E01\u0E22\u0E01\u0E40\ + \u0E25\u0E34\u0E01\u0E43\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E01\ + \u0E32\u0E23\u0E1F\u0E37\u0E49\u0E19\u0E1F\u0E39 Stuarts \u0E40\u0E19\u0E37\ + \u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E41\ + \u0E2A\u0E27\u0E07\u0E2B\u0E32\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\ + \u0E31\u0E19\u0E18\u0E4C\u0E17\u0E32\u0E07\u0E01\u0E32\u0E23\u0E17\u0E39\u0E15\ + \u0E15\u0E32\u0E21\u0E1B\u0E01\u0E15\u0E34\u0E01\u0E31\u0E1A\u0E0A\u0E32\u0E27\ + \u0E14\u0E31\u0E15\u0E0A\u0E4C\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E44\u0E21\u0E48\ + \u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E01\u0E32\u0E23\u0E2A\u0E19\u0E31\ + \u0E1A\u0E2A\u0E19\u0E38\u0E19\u0E17\u0E32\u0E07\u0E01\u0E32\u0E23\u0E40\u0E07\ + \u0E34\u0E19\u0E08\u0E32\u0E01\u0E23\u0E31\u0E10\u0E2A\u0E20\u0E32\u0E43\u0E19\ + \u0E01\u0E32\u0E23\u0E17\u0E33\u0E2A\u0E07\u0E04\u0E23\u0E32\u0E21" + D: "\u0E2A\u0E34\u0E48\u0E07\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E35\u0E49\ + \u0E19\u0E33\u0E44\u0E1B\u0E2A\u0E39\u0E48\u0E2A\u0E07\u0E04\u0E23\u0E32\u0E21\ + \u0E0B\u0E49\u0E33\u0E0B\u0E32\u0E01\u0E40\u0E01\u0E37\u0E2D\u0E1A\u0E2B\u0E19\ + \u0E36\u0E48\u0E07\u0E28\u0E15\u0E27\u0E23\u0E23\u0E29\u0E23\u0E30\u0E2B\u0E27\ + \u0E48\u0E32\u0E07\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E41\u0E25\u0E30\u0E40\ + \u0E19\u0E40\u0E18\u0E2D\u0E23\u0E4C\u0E41\u0E25\u0E19\u0E14\u0E4C \u0E0B\u0E36\ + \u0E48\u0E07\u0E08\u0E30\u0E44\u0E21\u0E48\u0E2A\u0E34\u0E49\u0E19\u0E2A\u0E38\ + \u0E14\u0E08\u0E19\u0E01\u0E27\u0E48\u0E32\u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\ + \u0E31\u0E1A\u0E40\u0E2D\u0E01\u0E23\u0E32\u0E0A\u0E08\u0E32\u0E01\u0E2D\u0E40\ + \u0E21\u0E23\u0E34\u0E01\u0E32" + input_correct_responses: + - A + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E02\u0E49\u0E2D\u0E04\u0E27\ + \u0E32\u0E21\u0E17\u0E35\u0E48\u0E15\u0E31\u0E14\u0E15\u0E2D\u0E19\u0E21\u0E32\ + \u0E14\u0E49\u0E32\u0E19\u0E25\u0E48\u0E32\u0E07\u0E21\u0E32\u0E08\u0E32\u0E01\ + \u0E1E\u0E23\u0E30\u0E23\u0E32\u0E0A\u0E1A\u0E31\u0E0D\u0E0D\u0E31\u0E15\u0E34\ + \u0E01\u0E32\u0E23\u0E40\u0E14\u0E34\u0E19\u0E40\u0E23\u0E37\u0E2D \u0E1E.\u0E28\ + . 2194 [\u0E01] \u0E2B\u0E25\u0E31\u0E07\u0E08\u0E32\u0E01\u0E27\u0E31\u0E19\ + \u0E41\u0E23\u0E01\u0E02\u0E2D\u0E07\u0E40\u0E14\u0E37\u0E2D\u0E19\u0E18\u0E31\ + \u0E19\u0E27\u0E32\u0E04\u0E21 \u0E2B\u0E19\u0E36\u0E48\u0E07\u0E1E\u0E31\u0E19\ + \u0E2B\u0E01\u0E23\u0E49\u0E2D\u0E22\u0E2B\u0E49\u0E32\u0E2A\u0E34\u0E1A\u0E40\ + \u0E2D\u0E47\u0E14 \u0E41\u0E25\u0E30\u0E19\u0E31\u0E1A\u0E08\u0E32\u0E01\u0E19\ + \u0E31\u0E49\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E15\u0E49\u0E19\u0E44\u0E1B \u0E2A\ + \u0E34\u0E19\u0E04\u0E49\u0E32\u0E2B\u0E23\u0E37\u0E2D\u0E42\u0E20\u0E04\u0E20\ + \u0E31\u0E13\u0E11\u0E4C\u0E43\u0E14\u0E46 \u0E01\u0E47\u0E15\u0E32\u0E21\u0E17\ + \u0E35\u0E48\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15 \u0E01\u0E32\u0E23\u0E1C\u0E25\ + \u0E34\u0E15 \u0E2B\u0E23\u0E37\u0E2D\u0E01\u0E32\u0E23\u0E1C\u0E25\u0E34\u0E15\ + \u0E43\u0E19\u0E40\u0E2D\u0E40\u0E0A\u0E35\u0E22 \u0E41\u0E2D\u0E1F\u0E23\u0E34\ + \u0E01\u0E32 \u0E2B\u0E23\u0E37\u0E2D\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\ + \ \u0E2B\u0E23\u0E37\u0E2D\u0E2A\u0E48\u0E27\u0E19\u0E43\u0E14\u0E2A\u0E48\u0E27\ + \u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07; \u0E2B\u0E23\u0E37\u0E2D\u0E02\u0E2D\u0E07\ + \u0E40\u0E01\u0E32\u0E30\u0E43\u0E14 \u0E46 \u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\ + \u0E19\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32 \u0E2B\u0E23\u0E37\ + \u0E2D\u0E17\u0E35\u0E48\u0E2D\u0E18\u0E34\u0E1A\u0E32\u0E22\u0E2B\u0E23\u0E37\ + \u0E2D\u0E27\u0E32\u0E07\u0E44\u0E27\u0E49\u0E43\u0E19\u0E41\u0E1C\u0E19\u0E17\ + \u0E35\u0E48\u0E2B\u0E23\u0E37\u0E2D\u0E01\u0E32\u0E23\u0E4C\u0E14\u0E15\u0E32\ + \u0E21\u0E1B\u0E01\u0E15\u0E34\u0E02\u0E2D\u0E07\u0E2A\u0E16\u0E32\u0E19\u0E17\ + \u0E35\u0E48\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19 \u0E23\u0E27\ + \u0E21\u0E17\u0E31\u0E49\u0E07\u0E2A\u0E27\u0E19\u0E02\u0E2D\u0E07\u0E2D\u0E31\ + \u0E07\u0E01\u0E24\u0E29\u0E41\u0E25\u0E30\u0E17\u0E35\u0E48\u0E2D\u0E37\u0E48\ + \u0E19 \u0E46 \u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E19\u0E33\u0E40\u0E02\u0E49\ + \u0E32\u0E2B\u0E23\u0E37\u0E2D\u0E19\u0E33\u0E40\u0E02\u0E49\u0E32\u0E40\u0E04\ + \u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\u0E23\u0E20\u0E1E\u0E41\u0E2B\u0E48\u0E07\ + \u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E2B\u0E23\u0E37\u0E2D\u0E43\u0E19\u0E44\ + \u0E2D\u0E23\u0E4C\u0E41\u0E25\u0E19\u0E14\u0E4C \u0E2B\u0E23\u0E37\u0E2D \u0E17\ + \u0E35\u0E48\u0E14\u0E34\u0E19 \u0E40\u0E01\u0E32\u0E30 \u0E1E\u0E37\u0E49\u0E19\ + \u0E17\u0E35\u0E48\u0E40\u0E1E\u0E32\u0E30\u0E1B\u0E25\u0E39\u0E01 \u0E2B\u0E23\ + \u0E37\u0E2D\u0E14\u0E34\u0E19\u0E41\u0E14\u0E19\u0E2D\u0E37\u0E48\u0E19\u0E43\ + \u0E14\u0E02\u0E2D\u0E07\u0E40\u0E04\u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\u0E23\ + \u0E20\u0E1E\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E02\u0E2D\u0E07\u0E40\ + \u0E04\u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\u0E23\u0E20\u0E1E\u0E19\u0E35\u0E49\ + \ \u0E2B\u0E23\u0E37\u0E2D\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E04\u0E27\u0E32\ + \u0E21\u0E04\u0E23\u0E2D\u0E1A\u0E04\u0E23\u0E2D\u0E07\u0E02\u0E2D\u0E07\u0E1E\ + \u0E27\u0E01\u0E21\u0E31\u0E19 \u0E43\u0E19\u0E40\u0E23\u0E37\u0E2D\u0E25\u0E33\ + \u0E2D\u0E37\u0E48\u0E19 \u0E40\u0E23\u0E37\u0E2D \u0E40\u0E23\u0E37\u0E2D\u0E2B\ + \u0E23\u0E37\u0E2D\u0E40\u0E23\u0E37\u0E2D\u0E43\u0E14\u0E46 \u0E01\u0E47\u0E15\ + \u0E32\u0E21 \u0E41\u0E15\u0E48\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E43\u0E19\u0E2A\ + \u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E17\u0E33\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E41\u0E17\u0E49\u0E08\u0E23\u0E34\u0E07\u0E41\u0E25\u0E30\u0E1B\u0E23\u0E32\ + \u0E28\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E09\u0E49\u0E2D\u0E09\u0E25\u0E40\ + \u0E1B\u0E47\u0E19\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E19\ + \u0E43\u0E19\u0E40\u0E04\u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\u0E23\u0E20\u0E1E\ + \u0E19\u0E35\u0E49\u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19 \u0E2B\u0E23\ + \u0E37\u0E2D\u0E2A\u0E27\u0E19\u0E1B\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E40\ + \u0E08\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E2B\u0E23\u0E37\u0E2D\u0E40\u0E08\u0E49\ + \u0E32\u0E02\u0E2D\u0E07\u0E2A\u0E34\u0E17\u0E18\u0E34; \u0E41\u0E25\u0E30\u0E1C\ + \u0E39\u0E49\u0E0B\u0E36\u0E48\u0E07\u0E19\u0E32\u0E22\u0E40\u0E23\u0E37\u0E2D\ + \u0E41\u0E25\u0E30\u0E01\u0E30\u0E25\u0E32\u0E2A\u0E35\u0E40\u0E1B\u0E47\u0E19\ + \u0E04\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E04\u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\ + \u0E23\u0E20\u0E1E\u0E19\u0E35\u0E49\u0E14\u0E49\u0E27\u0E22 \u0E20\u0E32\u0E22\ + \u0E43\u0E15\u0E49\u0E1A\u0E17\u0E25\u0E07\u0E42\u0E17\u0E29\u0E02\u0E2D\u0E07\ + \u0E01\u0E32\u0E23\u0E23\u0E34\u0E1A\u0E41\u0E25\u0E30\u0E01\u0E32\u0E23\u0E2A\ + \u0E39\u0E0D\u0E40\u0E2A\u0E35\u0E22\u0E02\u0E2D\u0E07\u0E2A\u0E34\u0E19\u0E04\ + \u0E49\u0E32\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\u0E17\u0E35\u0E48\u0E08\ + \u0E30\u0E19\u0E33\u0E40\u0E02\u0E49\u0E32\u0E02\u0E31\u0E14\u0E15\u0E48\u0E2D\ + \u0E1E\u0E23\u0E30\u0E23\u0E32\u0E0A\u0E1A\u0E31\u0E0D\u0E0D\u0E31\u0E15\u0E34\ + \u0E19\u0E35\u0E49 , , , [N]o \u0E2A\u0E34\u0E19\u0E04\u0E49\u0E32\u0E2B\u0E23\ + \u0E37\u0E2D\u0E2A\u0E34\u0E19\u0E04\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E01\u0E32\ + \u0E23\u0E40\u0E08\u0E23\u0E34\u0E0D\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15 \u0E01\ + \u0E32\u0E23\u0E1C\u0E25\u0E34\u0E15\u0E2B\u0E23\u0E37\u0E2D\u0E01\u0E32\u0E23\ + \u0E1C\u0E25\u0E34\u0E15\u0E02\u0E2D\u0E07\u0E22\u0E38\u0E42\u0E23\u0E1B\u0E2B\ + \u0E23\u0E37\u0E2D\u0E2A\u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E2A\ + \u0E48\u0E27\u0E19\u0E43\u0E14\u0E02\u0E2D\u0E07\u0E22\u0E38\u0E42\u0E23\u0E1B\ + \u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E19\u0E33\u0E40\u0E02\u0E49\u0E32\u0E2B\ + \u0E23\u0E37\u0E2D\u0E19\u0E33\u0E40\u0E02\u0E49\u0E32\u0E21\u0E32\u0E43\u0E19\ + \u0E40\u0E04\u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\u0E23\u0E20\u0E1E\u0E41\u0E2B\ + \u0E48\u0E07\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E2B\u0E23\u0E37\u0E2D\u0E14\ + \u0E34\u0E19\u0E41\u0E14\u0E19\u0E2B\u0E23\u0E37\u0E2D\u0E14\u0E34\u0E19\u0E41\ + \u0E14\u0E19\u0E2D\u0E37\u0E48\u0E19 \u0E46 \u0E43\u0E19\u0E40\u0E04\u0E23\u0E37\ + \u0E2D\u0E08\u0E31\u0E01\u0E23\u0E20\u0E1E\u0E41\u0E2B\u0E48\u0E07\u0E19\u0E35\ + \u0E49\u0E2B\u0E25\u0E31\u0E07\u0E08\u0E32\u0E01\u0E27\u0E31\u0E19\u0E41\u0E23\ + \u0E01\u0E02\u0E2D\u0E07\u0E40\u0E14\u0E37\u0E2D\u0E19\u0E18\u0E31\u0E19\u0E27\ + \u0E32\u0E04\u0E21 \u0E2B\u0E23\u0E37\u0E2D\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\ + \u0E04\u0E27\u0E32\u0E21\u0E04\u0E23\u0E2D\u0E1A\u0E04\u0E23\u0E2D\u0E07\u0E02\ + \u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32 \u0E43\u0E19\u0E40\u0E23\u0E37\ + \u0E2D\u0E2B\u0E23\u0E37\u0E2D\u0E40\u0E23\u0E37\u0E2D \u0E40\u0E23\u0E37\u0E2D\ + \ \u0E2B\u0E23\u0E37\u0E2D\u0E40\u0E23\u0E37\u0E2D\u0E43\u0E14\u0E46 \u0E01\u0E47\ + \u0E15\u0E32\u0E21 \u0E41\u0E15\u0E48\u0E43\u0E19\u0E2A\u0E34\u0E48\u0E07\u0E17\ + \u0E35\u0E48\u0E17\u0E33\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E41\u0E17\u0E49\u0E08\ + \u0E23\u0E34\u0E07\u0E41\u0E25\u0E30\u0E1B\u0E23\u0E32\u0E28\u0E08\u0E32\u0E01\ + \u0E01\u0E32\u0E23\u0E09\u0E49\u0E2D\u0E09\u0E25\u0E40\u0E1B\u0E47\u0E19\u0E02\ + \u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E19\u0E43\u0E19\u0E40\u0E04\ + \u0E23\u0E37\u0E2D\u0E08\u0E31\u0E01\u0E23\u0E20\u0E1E\u0E19\u0E35\u0E49\u0E40\ + \u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19 \u0E41\u0E25\u0E30\u0E44\u0E21\u0E48\ + \u0E43\u0E0A\u0E48\u0E02\u0E2D\u0E07\u0E2D\u0E37\u0E48\u0E19\u0E43\u0E14 \u0E22\ + \u0E01\u0E40\u0E27\u0E49\u0E19\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E40\u0E23\u0E37\ + \u0E2D\u0E41\u0E25\u0E30\u0E40\u0E23\u0E37\u0E2D\u0E15\u0E48\u0E32\u0E07\u0E1B\ + \u0E23\u0E30\u0E40\u0E17\u0E28\u0E17\u0E35\u0E48\u0E17\u0E33\u0E2D\u0E22\u0E48\ + \u0E32\u0E07\u0E41\u0E17\u0E49\u0E08\u0E23\u0E34\u0E07 \u0E41\u0E25\u0E30 \u0E42\ + \u0E14\u0E22\u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07\u0E40\u0E1B\u0E47\u0E19\ + \u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E19\u0E43\u0E19\u0E1B\ + \u0E23\u0E30\u0E40\u0E17\u0E28\u0E2B\u0E23\u0E37\u0E2D\u0E2A\u0E16\u0E32\u0E19\ + \u0E17\u0E35\u0E48\u0E19\u0E31\u0E49\u0E19 \u0E46 \u0E0B\u0E36\u0E48\u0E07\u0E02\ + \u0E2D\u0E07\u0E14\u0E31\u0E07\u0E01\u0E25\u0E48\u0E32\u0E27\u0E40\u0E1B\u0E47\ + \u0E19\u0E04\u0E27\u0E32\u0E21\u0E40\u0E08\u0E23\u0E34\u0E0D \u0E01\u0E32\u0E23\ + \u0E1C\u0E25\u0E34\u0E15 \u0E2B\u0E23\u0E37\u0E2D\u0E01\u0E32\u0E23\u0E1C\u0E25\ + \u0E34\u0E15 \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E2D\u0E18\u0E34\u0E1A\u0E32\u0E22\u0E1C\u0E25\u0E02\u0E2D\u0E07\ + \u0E1E\u0E23\u0E30\u0E23\u0E32\u0E0A\u0E1A\u0E31\u0E0D\u0E0D\u0E31\u0E15\u0E34\ + \u0E01\u0E32\u0E23\u0E40\u0E14\u0E34\u0E19\u0E40\u0E23\u0E37\u0E2D\u0E1B\u0E35\ + \ 1651 \u0E44\u0E14\u0E49\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" + - input_choice_list: + A: "\u0E21\u0E2D\u0E1A\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E2D\u0E33\ + \u0E19\u0E32\u0E08\u0E43\u0E2B\u0E21\u0E48\u0E43\u0E2B\u0E49\u0E01\u0E31\u0E1A\ + \u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E2D\u0E31\u0E07\u0E01\u0E24\ + \u0E29" + B: "\u0E43\u0E2B\u0E49\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E2B\u0E31\ + \u0E27\u0E2B\u0E19\u0E49\u0E32\u0E04\u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\ + \u0E23\u0E41\u0E2B\u0E48\u0E07\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E41\u0E01\ + \u0E48 Henry VIII \u0E40\u0E1E\u0E35\u0E22\u0E07\u0E1C\u0E39\u0E49\u0E40\u0E14\ + \u0E35\u0E22\u0E27\u0E41\u0E25\u0E30\u0E44\u0E21\u0E48\u0E23\u0E27\u0E21\u0E17\ + \u0E32\u0E22\u0E32\u0E17\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32" + C: "\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E43\u0E2B\u0E49\u0E25\u0E31\u0E17\u0E18\ + \u0E34\u0E04\u0E32\u0E25\u0E27\u0E34\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E28\u0E32\ + \u0E2A\u0E19\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C\u0E17\u0E35\u0E48\u0E41\u0E17\ + \u0E49\u0E08\u0E23\u0E34\u0E07\u0E41\u0E2B\u0E48\u0E07\u0E40\u0E14\u0E35\u0E22\ + \u0E27\u0E43\u0E19\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29" + D: "\u0E22\u0E38\u0E15\u0E34\u0E01\u0E32\u0E23\u0E04\u0E2D\u0E23\u0E31\u0E1B\ + \u0E0A\u0E31\u0E48\u0E19\u0E43\u0E19\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E15\ + \u0E48\u0E32\u0E07 \u0E46 \u0E17\u0E35\u0E48\u0E01\u0E48\u0E2D\u0E01\u0E27\ + \u0E19\u0E04\u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E43\u0E19\u0E2D\ + \u0E31\u0E07\u0E01\u0E24\u0E29" + input_correct_responses: + - D + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E41\u0E21\u0E49\u0E27\u0E48\ + \u0E32\u0E1E\u0E23\u0E30\u0E1A\u0E32\u0E17\u0E2A\u0E21\u0E40\u0E14\u0E47\u0E08\ + \u0E1E\u0E23\u0E30\u0E40\u0E08\u0E49\u0E32\u0E2D\u0E22\u0E39\u0E48\u0E2B\u0E31\ + \u0E27\u0E08\u0E30\u0E17\u0E23\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E1B\u0E23\u0E30\ + \u0E21\u0E38\u0E02\u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14\u0E02\u0E2D\u0E07\u0E19\ + \u0E34\u0E01\u0E32\u0E22\u0E40\u0E0A\u0E34\u0E23\u0E4C\u0E0A\u0E2D\u0E2D\u0E1F\ + \u0E2D\u0E34\u0E07\u0E41\u0E25\u0E19\u0E14\u0E4C\u0E42\u0E14\u0E22\u0E0A\u0E2D\ + \u0E1A\u0E18\u0E23\u0E23\u0E21\u0E41\u0E25\u0E30\u0E0A\u0E2D\u0E1A\u0E18\u0E23\ + \u0E23\u0E21\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E41\u0E25\u0E30\u0E14\u0E49\u0E27\ + \u0E22\u0E40\u0E2B\u0E15\u0E38\u0E19\u0E35\u0E49\u0E08\u0E36\u0E07\u0E44\u0E14\ + \u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\ + \u0E08\u0E32\u0E01\u0E01\u0E25\u0E38\u0E48\u0E21\u0E19\u0E31\u0E01\u0E1A\u0E27\ + \u0E0A\u0E41\u0E2B\u0E48\u0E07\u0E2D\u0E32\u0E13\u0E32\u0E08\u0E31\u0E01\u0E23\ + \u0E19\u0E35\u0E49\u0E43\u0E19\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E0A\u0E38\ + \u0E21\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32 \u0E2D\u0E22\u0E48\ + \u0E32\u0E07\u0E44\u0E23\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E22\u0E37\u0E19\u0E22\u0E31\u0E19\u0E41\u0E25\u0E30\u0E22\u0E37\u0E19\ + \u0E22\u0E31\u0E19\u0E2A\u0E34\u0E48\u0E07\u0E19\u0E31\u0E49\u0E19 \u0E41\u0E25\ + \u0E30\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1E\u0E39\ + \u0E19\u0E04\u0E38\u0E13\u0E18\u0E23\u0E23\u0E21\u0E43\u0E19 \u0E28\u0E32\u0E2A\ + \u0E19\u0E32\u0E04\u0E23\u0E34\u0E2A\u0E15\u0E4C\u0E43\u0E19\u0E2D\u0E32\u0E13\ + \u0E32\u0E08\u0E31\u0E01\u0E23\u0E19\u0E35\u0E49\u0E02\u0E2D\u0E07\u0E2D\u0E31\ + \u0E07\u0E01\u0E24\u0E29 \u0E41\u0E25\u0E30\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E1B\ + \u0E23\u0E32\u0E1A\u0E1B\u0E23\u0E32\u0E21\u0E41\u0E25\u0E30\u0E01\u0E33\u0E08\ + \u0E31\u0E14\u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14 \u0E25\ + \u0E31\u0E17\u0E18\u0E34\u0E19\u0E2D\u0E01\u0E23\u0E35\u0E15 \u0E41\u0E25\u0E30\ + \u0E04\u0E27\u0E32\u0E21\u0E0A\u0E31\u0E48\u0E27\u0E23\u0E49\u0E32\u0E22\u0E2D\ + \u0E37\u0E48\u0E19\u0E46 \u0E41\u0E25\u0E30\u0E01\u0E32\u0E23\u0E25\u0E48\u0E27\ + \u0E07\u0E25\u0E30\u0E40\u0E21\u0E34\u0E14\u0E2D\u0E37\u0E48\u0E19\u0E46 \u0E17\ + \u0E35\u0E48\u0E40\u0E04\u0E22\u0E43\u0E0A\u0E49\u0E43\u0E19\u0E2A\u0E34\u0E48\ + \u0E07\u0E40\u0E14\u0E35\u0E22\u0E27\u0E01\u0E31\u0E19\u0E19\u0E35\u0E49 \u0E44\ + \u0E21\u0E48\u0E27\u0E48\u0E32\u0E08\u0E30\u0E15\u0E23\u0E32\u0E02\u0E36\u0E49\ + \u0E19\u0E42\u0E14\u0E22\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E02\u0E2D\u0E07\u0E23\ + \u0E31\u0E10\u0E2A\u0E20\u0E32\u0E1B\u0E31\u0E08\u0E08\u0E38\u0E1A\u0E31\u0E19\ + \u0E19\u0E35\u0E49 \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E27\u0E48\u0E32\u0E01\u0E29\ + \u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C \u0E40\u0E08\u0E49\u0E32\u0E19\u0E32\u0E22\ + \u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14\u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32 \u0E23\ + \u0E31\u0E0A\u0E17\u0E32\u0E22\u0E32\u0E17\u0E02\u0E2D\u0E07\u0E1E\u0E23\u0E30\ + \u0E2D\u0E07\u0E04\u0E4C \u0E41\u0E25\u0E30\u0E1C\u0E39\u0E49\u0E2A\u0E37\u0E1A\ + \u0E17\u0E2D\u0E14 \u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E41\u0E2B\ + \u0E48\u0E07\u0E2D\u0E32\u0E13\u0E32\u0E08\u0E31\u0E01\u0E23\u0E19\u0E35\u0E49\ + \u0E08\u0E30\u0E16\u0E39\u0E01\u0E22\u0E36\u0E14\u0E04\u0E23\u0E2D\u0E07 \u0E22\ + \u0E2D\u0E21\u0E23\u0E31\u0E1A \u0E41\u0E25\u0E30\u0E02\u0E36\u0E49\u0E19\u0E0A\ + \u0E37\u0E48\u0E2D\u0E27\u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E1B\u0E23\u0E30\ + \u0E21\u0E38\u0E02\u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14\u0E04\u0E19\u0E40\u0E14\ + \u0E35\u0E22\u0E27\u0E43\u0E19\u0E42\u0E25\u0E01\u0E02\u0E2D\u0E07\u0E19\u0E34\ + \u0E01\u0E32\u0E22\u0E40\u0E0A\u0E34\u0E23\u0E4C\u0E0A\u0E2D\u0E2D\u0E1F\u0E2D\ + \u0E34\u0E07\u0E41\u0E25\u0E19\u0E14\u0E4C \u0E17\u0E35\u0E48\u0E40\u0E23\u0E35\ + \u0E22\u0E01\u0E27\u0E48\u0E32\u0E41\u0E2D\u0E07\u0E01\u0E25\u0E34\u0E04\u0E31\ + \u0E19\u0E40\u0E2D\u0E04\u0E40\u0E04\u0E34\u0E25\u0E40\u0E0B\u0E35\u0E22 \u0E41\ + \u0E25\u0E30\u0E08\u0E30\u0E21\u0E35\u0E41\u0E25\u0E30\u0E40\u0E1E\u0E25\u0E34\ + \u0E14\u0E40\u0E1E\u0E25\u0E34\u0E19\u0E44\u0E1B\u0E01\u0E31\u0E1A \u0E1C\u0E19\ + \u0E27\u0E01\u0E41\u0E25\u0E30\u0E23\u0E27\u0E21\u0E40\u0E1B\u0E47\u0E19\u0E2B\ + \u0E19\u0E36\u0E48\u0E07\u0E01\u0E31\u0E1A\u0E21\u0E07\u0E01\u0E38\u0E0E\u0E02\ + \u0E2D\u0E07\u0E2D\u0E32\u0E13\u0E32\u0E08\u0E31\u0E01\u0E23\u0E19\u0E35\u0E49\ + \ \u0E15\u0E25\u0E2D\u0E14\u0E08\u0E19\u0E0A\u0E37\u0E48\u0E2D\u0E41\u0E25\u0E30\ + \u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E02\u0E2D\u0E07\u0E21\u0E31\u0E19 \u0E40\ + \u0E0A\u0E48\u0E19\u0E40\u0E14\u0E35\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E40\u0E01\ + \u0E35\u0E22\u0E23\u0E15\u0E34\u0E22\u0E28 \u0E28\u0E31\u0E01\u0E14\u0E34\u0E4C\ + \u0E28\u0E23\u0E35 \u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E48\u0E07\u0E43\u0E2B\ + \u0E0D\u0E48 \u0E40\u0E02\u0E15\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E28\u0E32\u0E25\ + \ \u0E40\u0E2D\u0E01\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C \u0E2D\u0E33\u0E19\u0E32\ + \u0E08 \u0E04\u0E27\u0E32\u0E21\u0E04\u0E38\u0E49\u0E21\u0E01\u0E31\u0E19 \u0E1C\ + \u0E25\u0E01\u0E33\u0E44\u0E23 \u0E41\u0E25\u0E30\u0E2A\u0E34\u0E19\u0E04\u0E49\ + \u0E32\u0E02\u0E2D\u0E07\u0E28\u0E31\u0E01\u0E14\u0E34\u0E4C\u0E28\u0E23\u0E35\ + \u0E14\u0E31\u0E07\u0E01\u0E25\u0E48\u0E32\u0E27\u0E02\u0E2D\u0E07 \u0E2B\u0E31\ + \u0E27\u0E2B\u0E19\u0E49\u0E32\u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14\u0E02\u0E2D\ + \u0E07\u0E04\u0E23\u0E34\u0E2A\u0E15\u0E08\u0E31\u0E01\u0E23\u0E40\u0E14\u0E35\ + \u0E22\u0E27\u0E01\u0E31\u0E19\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E02\ + \u0E2D\u0E07\u0E41\u0E25\u0E30\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\ + \u0E2D\u0E07; \u0E41\u0E25\u0E30\u0E27\u0E48\u0E32\u0E25\u0E2D\u0E23\u0E4C\u0E14\ + \u0E2D\u0E07\u0E04\u0E4C\u0E2D\u0E18\u0E34\u0E1B\u0E44\u0E15\u0E22\u0E02\u0E2D\ + \u0E07\u0E40\u0E23\u0E32 \u0E17\u0E32\u0E22\u0E32\u0E17\u0E41\u0E25\u0E30\u0E1C\ + \u0E39\u0E49\u0E2A\u0E37\u0E1A\u0E17\u0E2D\u0E14\u0E02\u0E2D\u0E07\u0E40\u0E02\ + \u0E32 \u0E23\u0E32\u0E0A\u0E32\u0E41\u0E2B\u0E48\u0E07\u0E2D\u0E32\u0E13\u0E32\ + \u0E08\u0E31\u0E01\u0E23\u0E19\u0E35\u0E49 \u0E08\u0E30\u0E21\u0E35\u0E2D\u0E33\ + \u0E19\u0E32\u0E08\u0E40\u0E15\u0E47\u0E21\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\ + \u0E19\u0E04\u0E23\u0E31\u0E49\u0E07\u0E04\u0E23\u0E32\u0E27\u0E43\u0E19\u0E01\ + \u0E32\u0E23\u0E40\u0E22\u0E35\u0E48\u0E22\u0E21\u0E40\u0E22\u0E35\u0E22\u0E19\ + \ \u0E1B\u0E23\u0E32\u0E1A\u0E1B\u0E23\u0E32\u0E21 \u0E41\u0E01\u0E49\u0E44\u0E02\ + \ \u0E41\u0E01\u0E49\u0E44\u0E02 \u0E1A\u0E31\u0E19\u0E17\u0E36\u0E01 \u0E04\ + \u0E33\u0E2A\u0E31\u0E48\u0E07 \u0E41\u0E01\u0E49\u0E44\u0E02 \u0E22\u0E31\u0E1A\ + \u0E22\u0E31\u0E49\u0E07 \u0E41\u0E25\u0E30\u0E41\u0E01\u0E49\u0E44\u0E02\u0E02\ + \u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\u0E17\u0E31\u0E49\u0E07\ + \u0E2B\u0E21\u0E14 \u0E19\u0E2D\u0E01\u0E23\u0E35\u0E15 \u0E01\u0E32\u0E23\u0E02\ + \u0E48\u0E21\u0E40\u0E2B\u0E07 \u0E01\u0E32\u0E23\u0E23\u0E38\u0E01\u0E23\u0E32\ + \u0E19 \u0E01\u0E32\u0E23\u0E14\u0E39\u0E2B\u0E21\u0E34\u0E48\u0E19 \u0E41\u0E25\ + \u0E30\u0E04\u0E27\u0E32\u0E21\u0E0A\u0E31\u0E48\u0E27\u0E23\u0E49\u0E32\u0E22\ + \ \u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\u0E21\u0E31\u0E19\u0E08\u0E30\u0E40\u0E1B\ + \u0E47\u0E19\u0E2D\u0E30\u0E44\u0E23\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E0B\u0E36\ + \u0E48\u0E07\u0E42\u0E14\u0E22\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\u0E43\u0E14\ + \u0E01\u0E47\u0E15\u0E32\u0E21\u0E02\u0E2D\u0E07\u0E2D\u0E33\u0E19\u0E32\u0E08\ + \u0E17\u0E32\u0E07\u0E27\u0E34\u0E0D\u0E0D\u0E32\u0E13\u0E2B\u0E23\u0E37\u0E2D\ + \u0E40\u0E02\u0E15\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E28\u0E32\u0E25\u0E04\u0E27\ + \u0E23\u0E2B\u0E23\u0E37\u0E2D\u0E2D\u0E32\u0E08\u0E44\u0E14\u0E49\u0E23\u0E31\ + \u0E1A\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E31\u0E1A\u0E1B\u0E23\u0E38\u0E07 \u0E1B\ + \u0E23\u0E32\u0E1A\u0E1B\u0E23\u0E32\u0E21 \u0E2A\u0E31\u0E48\u0E07\u0E01\u0E32\ + \u0E23 \u0E41\u0E01\u0E49\u0E44\u0E02 \u0E22\u0E31\u0E1A\u0E22\u0E31\u0E49\u0E07\ + \ \u0E2B\u0E23\u0E37\u0E2D\u0E41\u0E01\u0E49\u0E44\u0E02\u0E42\u0E14\u0E22\u0E0A\ + \u0E2D\u0E1A\u0E14\u0E49\u0E27\u0E22\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22 \u0E42\ + \u0E14\u0E22\u0E2A\u0E48\u0E27\u0E19\u0E43\u0E2B\u0E0D\u0E48\u0E40\u0E1E\u0E37\ + \u0E48\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E2D\u0E1E\u0E23\u0E30\u0E17\u0E31\ + \u0E22\u0E02\u0E2D\u0E07\u0E1E\u0E23\u0E30\u0E1C\u0E39\u0E49\u0E40\u0E1B\u0E47\ + \u0E19\u0E40\u0E08\u0E49\u0E32\u0E1C\u0E39\u0E49\u0E17\u0E23\u0E07\u0E24\u0E17\ + \u0E18\u0E32\u0E19\u0E38\u0E20\u0E32\u0E1E \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1E\ + \u0E39\u0E19\u0E04\u0E38\u0E13\u0E18\u0E23\u0E23\u0E21\u0E43\u0E19\u0E28\u0E32\ + \u0E2A\u0E19\u0E32\u0E04\u0E23\u0E34\u0E2A\u0E15\u0E4C\u0E41\u0E25\u0E30\u0E40\ + \u0E1E\u0E37\u0E48\u0E2D\u0E23\u0E31\u0E01\u0E29\u0E32\u0E2A\u0E31\u0E19\u0E15\ + \u0E34\u0E20\u0E32\u0E1E \u0E04\u0E27\u0E32\u0E21\u0E2A\u0E32\u0E21\u0E31\u0E04\ + \u0E04\u0E35 \u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E23\u0E48\u0E21\u0E40\ + \u0E22\u0E47\u0E19\u0E02\u0E2D\u0E07\u0E2D\u0E32\u0E13\u0E32\u0E08\u0E31\u0E01\ + \u0E23\u0E19\u0E35\u0E49 \u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E07\u0E32\u0E19\ + \u0E43\u0E14 \u0E46 \u0E17\u0E35\u0E48\u0E14\u0E34\u0E19\u0E15\u0E48\u0E32\u0E07\ + \u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28 \u0E2B\u0E19\u0E48\u0E27\u0E22\u0E07\u0E32\ + \u0E19\u0E15\u0E48\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28 \u0E43\u0E1A\ + \u0E2A\u0E31\u0E48\u0E07\u0E22\u0E32 \u0E2B\u0E23\u0E37\u0E2D\u0E2A\u0E34\u0E48\ + \u0E07\u0E2D\u0E37\u0E48\u0E19 \u0E46 \u0E2B\u0E23\u0E37\u0E2D\u0E2A\u0E34\u0E48\ + \u0E07\u0E15\u0E48\u0E32\u0E07 \u0E46 \u0E17\u0E35\u0E48\u0E15\u0E23\u0E07\u0E01\ + \u0E31\u0E19\u0E02\u0E49\u0E32\u0E21\u0E01\u0E31\u0E1A\u0E02\u0E49\u0E2D\u0E15\ + \u0E01\u0E25\u0E07\u0E19\u0E35\u0E49 \u0E23\u0E31\u0E10\u0E2A\u0E20\u0E32\u0E2D\ + \u0E31\u0E07\u0E01\u0E24\u0E29, \u0E1E\u0E23\u0E30\u0E23\u0E32\u0E0A\u0E1A\u0E31\ + \u0E0D\u0E0D\u0E31\u0E15\u0E34\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E2A\u0E39\u0E07\ + \u0E2A\u0E38\u0E14, \u0E04.\u0E28. 1534 \u0E08\u0E32\u0E01\u0E02\u0E49\u0E2D\ + \u0E04\u0E27\u0E32\u0E21\u0E19\u0E35\u0E49 \u0E2D\u0E32\u0E08\u0E2D\u0E19\u0E38\ + \u0E21\u0E32\u0E19\u0E44\u0E14\u0E49\u0E27\u0E48\u0E32\u0E23\u0E31\u0E10\u0E2A\ + \u0E20\u0E32\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E15\u0E49\u0E2D\u0E07\u0E01\ + \u0E32\u0E23\u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\u0E27\u0E48\u0E32\u0E1E\ + \u0E23\u0E30\u0E23\u0E32\u0E0A\u0E1A\u0E31\u0E0D\u0E0D\u0E31\u0E15\u0E34\u0E2D\ + \u0E33\u0E19\u0E32\u0E08\u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14\u0E08\u0E30" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_european_history +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_high_school_european_history +task_alias: high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e3ab0fee61da46860bce3d200fdda315dc43e61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_geography.yaml @@ -0,0 +1,112 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E15\u0E32\u0E22\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E2B\u0E22\u0E32\u0E1A\u0E08\u0E32\u0E01\u0E27\u0E31\u0E19\u0E40\u0E01\ + \u0E34\u0E14\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2B\u0E22\u0E32\u0E1A" + B: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2B\u0E22\u0E32\u0E1A\u0E08\u0E32\u0E01\u0E2D\ + \u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E15\u0E32\u0E22\u0E2D\u0E22\u0E48\ + \u0E32\u0E07\u0E2B\u0E22\u0E32\u0E1A" + C: "\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E2A\u0E2D\u0E07\ + \u0E40\u0E17\u0E48\u0E32\u0E08\u0E32\u0E01\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\ + \u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2B\u0E22\ + \u0E32\u0E1A" + D: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E08\u0E23\u0E34\ + \u0E0D\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E4C\u0E08\u0E32\u0E01\u0E2D\u0E31\u0E15\ + \u0E23\u0E32\u0E01\u0E32\u0E23\u0E15\u0E32\u0E22\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E2B\u0E22\u0E32\u0E1A" + input_correct_responses: + - A + input_question: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\ + \u0E48\u0E21\u0E15\u0E32\u0E21\u0E18\u0E23\u0E23\u0E21\u0E0A\u0E32\u0E15\u0E34\ + \u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E2B\u0E32\u0E44\ + \u0E14\u0E49\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E25\u0E1A" + - input_choice_list: + A: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\ + \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E41\u0E25\u0E30\u0E2D\ + \u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15\ + \u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E25\u0E14\u0E25\ + \u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E23\u0E27\u0E14\u0E40\u0E23\u0E47\u0E27" + B: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\ + \u0E25\u0E14\u0E25\u0E07\u0E41\u0E25\u0E30\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\ + \u0E32\u0E23\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15\u0E02\u0E2D\u0E07\u0E1B\u0E23\ + \u0E30\u0E0A\u0E32\u0E01\u0E23\u0E25\u0E14\u0E25\u0E07\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E23\u0E27\u0E14\u0E40\u0E23\u0E47\u0E27" + C: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\ + \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E41\u0E25\u0E30\u0E2D\ + \u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\ + \u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E40\u0E1E\u0E34\u0E48\ + \u0E21\u0E02\u0E36\u0E49\u0E19" + D: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\ + \u0E25\u0E14\u0E25\u0E07\u0E41\u0E25\u0E30\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\ + \u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\ + \u0E0A\u0E32\u0E01\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + input_correct_responses: + - B + input_question: "\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E32\ + \u0E21\u0E02\u0E2D\u0E07\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E01\u0E32\u0E23\ + \u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E41\u0E1B\u0E25\u0E07\u0E17\u0E32\ + \u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23 \u0E02\u0E49\u0E2D\u0E43\u0E14\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E08\ + \u0E23\u0E34\u0E07" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E22\u0E32\u0E22\u0E32\u0E21\u0E0B\u0E49\ + \u0E33\u0E0B\u0E49\u0E2D\u0E19\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\ + \u0E1A\u0E48\u0E2D\u0E22\u0E04\u0E23\u0E31\u0E49\u0E07" + B: "\u0E1B\u0E31\u0E0D\u0E2B\u0E32\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E43\u0E08\ + \u0E01\u0E25\u0E32\u0E07\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E17\u0E30\u0E25\u0E31\ + \u0E01\u0E2A\u0E39\u0E48\u0E17\u0E35\u0E48\u0E2D\u0E22\u0E39\u0E48\u0E2D\u0E32\ + \u0E28\u0E31\u0E22\u0E23\u0E2D\u0E1A\u0E19\u0E2D\u0E01" + C: "\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E21\u0E35\u0E1B\u0E23\u0E30\ + \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E20\u0E32\u0E1E\u0E43\u0E19\u0E01\u0E32\u0E23\ + \u0E43\u0E2B\u0E49\u0E1A\u0E23\u0E34\u0E01\u0E32\u0E23\u0E40\u0E01\u0E34\u0E14\ + \u0E02\u0E36\u0E49\u0E19\u0E1A\u0E48\u0E2D\u0E22\u0E04\u0E23\u0E31\u0E49\u0E07" + D: "\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E22\u0E32\u0E22\u0E32\u0E21\u0E02\u0E2D\ + \u0E07\u0E22\u0E48\u0E32\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E43\u0E19\u0E01\ + \u0E32\u0E23\u0E25\u0E14\u0E21\u0E25\u0E1E\u0E34\u0E29\u0E44\u0E14\u0E49\u0E23\ + \u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2A\u0E19\u0E31\u0E1A\u0E2A\u0E19\u0E38\u0E19\ + \u0E08\u0E32\u0E01\u0E0A\u0E38\u0E21\u0E0A\u0E19\u0E43\u0E01\u0E25\u0E49\u0E40\ + \u0E04\u0E35\u0E22\u0E07\u0E40\u0E2A\u0E21\u0E2D" + input_correct_responses: + - D + input_question: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E43\u0E14\u0E15\u0E48\ + \u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E44\u0E21\u0E48\u0E16\u0E39\u0E01\u0E15\ + \u0E49\u0E2D\u0E07\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E1A\ + \u0E23\u0E34\u0E01\u0E32\u0E23\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\ + \u0E25\u0E17\u0E49\u0E2D\u0E07\u0E16\u0E34\u0E48\u0E19\u0E43\u0E19\u0E2A\u0E2B\ + \u0E23\u0E31\u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32" + - input_choice_list: + A: "\u0E40\u0E2D\u0E32\u0E17\u0E4C\u0E0B\u0E2D\u0E23\u0E4C\u0E2A" + B: "\u0E15\u0E48\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28" + C: "\u0E21\u0E32\u0E01\u0E34\u0E25\u0E32\u0E42\u0E14\u0E23\u0E32\u0E2A" + D: "\u0E01\u0E32\u0E23\u0E1E\u0E36\u0E48\u0E07\u0E1E\u0E32\u0E2D\u0E32\u0E28\ + \u0E31\u0E22\u0E01\u0E31\u0E19\u0E17\u0E32\u0E07\u0E2A\u0E16\u0E32\u0E19\u0E17\ + \u0E35\u0E48" + input_correct_responses: + - B + input_question: "\u0E01\u0E32\u0E23\u0E08\u0E49\u0E32\u0E07\u0E1C\u0E39\u0E49\u0E43\ + \u0E2B\u0E49\u0E1A\u0E23\u0E34\u0E01\u0E32\u0E23\u0E1A\u0E38\u0E04\u0E04\u0E25\ + \u0E17\u0E35\u0E48\u0E2A\u0E32\u0E21\u0E08\u0E32\u0E01\u0E15\u0E48\u0E32\u0E07\ + \u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E14\u0E33\ + \u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E40\u0E23\u0E35\u0E22\u0E01\u0E27\ + \u0E48\u0E32" + - input_choice_list: + A: "\u0E2A\u0E1A\u0E39\u0E48\u0E42\u0E14\u0E1F" + B: "\u0E41\u0E04\u0E19\u0E14\u0E35\u0E49\u0E1A\u0E32\u0E23\u0E4C\u0E42\u0E14\ + \u0E1F" + C: "\u0E2A\u0E31\u0E0D\u0E25\u0E31\u0E01\u0E29\u0E13\u0E4C\u0E19\u0E01\u0E1E\ + \u0E34\u0E23\u0E32\u0E1A" + D: "\u0E19\u0E01\u0E1E\u0E34\u0E23\u0E32\u0E1A (\u0E19\u0E01)" + input_correct_responses: + - C + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E02\u0E2D\u0E07\u0E27\u0E31\u0E12\u0E19\u0E18\u0E23\u0E23\u0E21\u0E17\ + \u0E35\u0E48\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E27\u0E31\u0E15\u0E16\u0E38" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_geography +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_high_school_geography +task_alias: high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86bf0a195c60d7d3759f66c6e885266cb59735ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_government_and_politics.yaml @@ -0,0 +1,140 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E04\u0E33\u0E08\u0E33\u0E01\u0E31\u0E14\u0E04\u0E27\u0E32\u0E21\u0E15\ + \u0E32\u0E21\u0E23\u0E31\u0E10\u0E18\u0E23\u0E23\u0E21\u0E19\u0E39\u0E0D\u0E02\ + \u0E2D\u0E07\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\ + \u0E31\u0E49\u0E19\u0E01\u0E27\u0E49\u0E32\u0E07\u0E41\u0E25\u0E30\u0E44\u0E21\ + \u0E48\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E40\u0E08\u0E32\u0E30\u0E08\u0E07" + B: "\u0E04\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E43\u0E2B\u0E0D\u0E48\u0E40\u0E2B\ + \u0E47\u0E19\u0E27\u0E48\u0E32\u0E23\u0E31\u0E10\u0E18\u0E23\u0E23\u0E21\u0E19\ + \u0E39\u0E0D\u0E08\u0E33\u0E01\u0E31\u0E14\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E1B\ + \u0E23\u0E30\u0E18\u0E32\u0E19\u0E32\u0E18\u0E34\u0E1A\u0E14\u0E35\u0E21\u0E32\ + \u0E01\u0E40\u0E01\u0E34\u0E19\u0E44\u0E1B" + C: "\u0E28\u0E32\u0E25\u0E0E\u0E35\u0E01\u0E32\u0E1B\u0E0F\u0E34\u0E40\u0E2A\ + \u0E18\u0E17\u0E35\u0E48\u0E08\u0E30\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E04\ + \u0E14\u0E35\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2D\u0E33\ + \u0E19\u0E32\u0E08\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E18\u0E32\u0E19\u0E32\ + \u0E18\u0E34\u0E1A\u0E14\u0E35\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E15\u0E48\u0E2D\ + \u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07" + D: "\u0E01\u0E32\u0E23\u0E41\u0E01\u0E49\u0E44\u0E02\u0E23\u0E31\u0E10\u0E18\ + \u0E23\u0E23\u0E21\u0E19\u0E39\u0E0D\u0E44\u0E14\u0E49\u0E40\u0E1E\u0E34\u0E48\ + \u0E21\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E18\ + \u0E32\u0E19\u0E32\u0E18\u0E34\u0E1A\u0E14\u0E35\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E21\u0E32\u0E01" + input_correct_responses: + - A + input_question: "\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E41\u0E19\u0E48\u0E19\ + \u0E2D\u0E19\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E02\u0E2D\ + \u0E1A\u0E40\u0E02\u0E15\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E02\u0E2D\u0E07\u0E1B\ + \u0E23\u0E30\u0E18\u0E32\u0E19\u0E32\u0E18\u0E34\u0E1A\u0E14\u0E35\u0E21\u0E35\ + \u0E2A\u0E32\u0E40\u0E2B\u0E15\u0E38\u0E2B\u0E25\u0E31\u0E01\u0E21\u0E32\u0E08\ + \u0E32\u0E01\u0E02\u0E49\u0E2D\u0E40\u0E17\u0E47\u0E08\u0E08\u0E23\u0E34\u0E07\ + \u0E17\u0E35\u0E48\u0E27\u0E48\u0E32" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E17\u0E35\ + \u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E17\u0E38\u0E01\ + \u0E1B\u0E35\u0E43\u0E19\u0E01\u0E32\u0E23\u0E17\u0E2B\u0E32\u0E23" + B: "\u0E08\u0E33\u0E19\u0E27\u0E19\u0E14\u0E2D\u0E01\u0E40\u0E1A\u0E35\u0E49\ + \u0E22\u0E2B\u0E19\u0E35\u0E49\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\ + \u0E28" + C: "\u0E04\u0E27\u0E32\u0E21\u0E41\u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E23\ + \u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E02\u0E49\u0E2D\u0E40\u0E2A\u0E19\u0E2D\ + \u0E07\u0E1A\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13\u0E40\u0E1A\u0E37\u0E49\u0E2D\ + \u0E07\u0E15\u0E49\u0E19\u0E17\u0E35\u0E48\u0E40\u0E2A\u0E19\u0E2D\u0E42\u0E14\ + \u0E22\u0E1B\u0E23\u0E30\u0E18\u0E32\u0E19\u0E32\u0E18\u0E34\u0E1A\u0E14\u0E35\ + \u0E41\u0E25\u0E30\u0E2A\u0E20\u0E32\u0E04\u0E2D\u0E07\u0E40\u0E01\u0E23\u0E2A" + D: "\u0E08\u0E33\u0E19\u0E27\u0E19\u0E40\u0E07\u0E34\u0E19\u0E17\u0E35\u0E48\ + \u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22\ + \u0E40\u0E01\u0E34\u0E19\u0E23\u0E32\u0E22\u0E44\u0E14\u0E49" + input_correct_responses: + - D + input_question: "\u0E04\u0E33\u0E27\u0E48\u0E32 "\u0E07\u0E1A\u0E1B\u0E23\ + \u0E30\u0E21\u0E32\u0E13\u0E02\u0E32\u0E14\u0E14\u0E38\u0E25" \u0E2B\u0E21\ + \u0E32\u0E22\u0E16\u0E36\u0E07" + - input_choice_list: + A: "\u0E2A\u0E31\u0E1B\u0E14\u0E32\u0E2B\u0E4C v. \u0E2A\u0E2B\u0E23\u0E31\u0E10\ + \u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32" + B: "\u0E40\u0E1A\u0E15\u0E15\u0E4C\u0E01\u0E31\u0E1A\u0E40\u0E1A\u0E23\u0E14\ + \u0E35\u0E49" + C: "\u0E41\u0E1C\u0E19\u0E17\u0E35\u0E48 v. \u0E42\u0E2D\u0E44\u0E2E\u0E42\u0E2D" + D: "\u0E21\u0E34\u0E41\u0E23\u0E19\u0E14\u0E32\u0E01\u0E31\u0E1A\u0E41\u0E2D\ + \u0E23\u0E34\u0E42\u0E0B\u0E19\u0E32" + input_correct_responses: + - D + input_question: "\u0E01\u0E23\u0E13\u0E35\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\ + \u0E19\u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E23\u0E13\u0E35\u0E15\u0E31\ + \u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E17\u0E35\u0E48\u0E08\u0E33\u0E40\u0E25\ + \u0E22\u0E15\u0E49\u0E2D\u0E07\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\ + \u0E23\u0E41\u0E08\u0E49\u0E07\u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E2A\u0E34\ + \u0E17\u0E18\u0E34\u0E43\u0E19\u0E01\u0E32\u0E23\u0E19\u0E34\u0E48\u0E07\u0E40\ + \u0E09\u0E22 \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E43\u0E19\u0E01\u0E32\u0E23\u0E21\ + \u0E35\u0E17\u0E19\u0E32\u0E22\u0E04\u0E27\u0E32\u0E21 \u0E41\u0E25\u0E30\u0E01\ + \u0E32\u0E23\u0E04\u0E38\u0E49\u0E21\u0E04\u0E23\u0E2D\u0E07\u0E08\u0E32\u0E01\ + \u0E01\u0E32\u0E23\u0E01\u0E25\u0E48\u0E32\u0E27\u0E2B\u0E32\u0E15\u0E19\u0E40\ + \u0E2D\u0E07" + - input_choice_list: + A: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E08\u0E31\u0E14\u0E15\u0E31\u0E49\ + \u0E07\u0E02\u0E36\u0E49\u0E19\u0E42\u0E14\u0E22\u0E1D\u0E48\u0E32\u0E22\u0E19\ + \u0E34\u0E15\u0E34\u0E1A\u0E31\u0E0D\u0E0D\u0E31\u0E15\u0E34" + B: "\u0E2A\u0E21\u0E32\u0E0A\u0E34\u0E01\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\ + \u0E40\u0E02\u0E32\u0E21\u0E31\u0E01\u0E44\u0E21\u0E48\u0E21\u0E35\u0E2D\u0E34\ + \u0E17\u0E18\u0E34\u0E1E\u0E25\u0E15\u0E48\u0E2D\u0E01\u0E32\u0E23\u0E15\u0E31\ + \u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E18\ + \u0E32\u0E19\u0E32\u0E18\u0E34\u0E1A\u0E14\u0E35\u0E21\u0E32\u0E01\u0E19\u0E31\ + \u0E01" + C: "\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\ + \u0E14\u0E44\u0E21\u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E14\u0E33\u0E40\ + \u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E42\u0E14\u0E22\u0E1C\u0E39\u0E49\u0E19\ + \u0E33\u0E17\u0E35\u0E48\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E1E\u0E23\u0E23\ + \u0E04\u0E01\u0E32\u0E23\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E40\u0E14\u0E35\u0E22\ + \u0E27\u0E01\u0E31\u0E19\u0E01\u0E31\u0E1A\u0E1B\u0E23\u0E30\u0E18\u0E32\u0E19\ + \u0E32\u0E18\u0E34\u0E1A\u0E14\u0E35\u0E44\u0E14\u0E49" + D: "\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E17\u0E38\u0E01\u0E2B\u0E19\u0E48\ + \u0E27\u0E22\u0E07\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\ + \u0E25\u0E01\u0E25\u0E32\u0E07\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E41\ + \u0E1C\u0E19\u0E01\u0E04\u0E13\u0E30\u0E23\u0E31\u0E10\u0E21\u0E19\u0E15\u0E23\ + \u0E35" + input_correct_responses: + - C + input_question: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E43\u0E14\u0E15\u0E48\ + \u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\ + \u0E31\u0E1A\u0E41\u0E1C\u0E19\u0E01\u0E04\u0E13\u0E30\u0E23\u0E31\u0E10\u0E21\ + \u0E19\u0E15\u0E23\u0E35\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E17\u0E47\u0E08" + - input_choice_list: + A: "\u0E19\u0E31\u0E01\u0E01\u0E32\u0E23\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E17\ + \u0E35\u0E48\u0E0B\u0E37\u0E48\u0E2D\u0E2A\u0E31\u0E15\u0E22\u0E4C\u0E2A\u0E32\ + \u0E21\u0E32\u0E23\u0E16\u0E02\u0E31\u0E14\u0E02\u0E27\u0E32\u0E07\u0E44\u0E21\ + \u0E48\u0E43\u0E2B\u0E49\u0E01\u0E25\u0E38\u0E48\u0E21\u0E15\u0E48\u0E32\u0E07\ + \u0E46 \u0E1E\u0E31\u0E12\u0E19\u0E32\u0E44\u0E14\u0E49" + B: "\u0E01\u0E25\u0E38\u0E48\u0E21\u0E21\u0E35\u0E41\u0E19\u0E27\u0E42\u0E19\ + \u0E49\u0E21\u0E17\u0E35\u0E48\u0E08\u0E30\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\ + \u0E49\u0E19\u0E43\u0E19\u0E2A\u0E32\u0E18\u0E32\u0E23\u0E13\u0E23\u0E31\u0E10\ + \u0E02\u0E19\u0E32\u0E14\u0E43\u0E2B\u0E0D\u0E48\u0E21\u0E32\u0E01\u0E01\u0E27\ + \u0E48\u0E32\u0E43\u0E19\u0E2A\u0E32\u0E18\u0E32\u0E23\u0E13\u0E23\u0E31\u0E10\ + \u0E02\u0E19\u0E32\u0E14\u0E40\u0E25\u0E47\u0E01" + C: "\u0E1C\u0E25\u0E01\u0E23\u0E30\u0E17\u0E1A\u0E14\u0E49\u0E32\u0E19\u0E25\ + \u0E1A\u0E02\u0E2D\u0E07\u0E25\u0E31\u0E17\u0E18\u0E34\u0E1D\u0E31\u0E01\u0E1D\ + \u0E48\u0E32\u0E22\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E25\u0E14\u0E25\u0E07\ + \u0E44\u0E14\u0E49\u0E42\u0E14\u0E22\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E2A\ + \u0E32\u0E18\u0E32\u0E23\u0E13\u0E23\u0E31\u0E10" + D: "\u0E01\u0E32\u0E23\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E15\u0E31\u0E49\u0E07\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E40\u0E2A\u0E23\u0E35\u0E04\u0E37\u0E2D\u0E01\ + \u0E32\u0E23\u0E1B\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E19\u0E17\u0E35\u0E48\u0E14\ + \u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\ + \u0E0A\u0E32\u0E0A\u0E19\u0E08\u0E32\u0E01\u0E25\u0E31\u0E17\u0E18\u0E34\u0E1D\ + \u0E31\u0E01\u0E1D\u0E48\u0E32\u0E22" + input_correct_responses: + - C + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E01\u0E25\u0E48\u0E32\u0E27\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\ + \u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\u0E02\u0E2D\u0E07 James Madison \u0E43\ + \u0E19 The Federalist \u0E2B\u0E21\u0E32\u0E22\u0E40\u0E25\u0E02 10 \u0E44\u0E14\ + \u0E49\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_government_and_politics +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_high_school_government_and_politics +task_alias: high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6321a804682b623c8870f8296684a6715d1f4c28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_macroeconomics.yaml @@ -0,0 +1,113 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E01\u0E2D\u0E07\u0E17\u0E31\u0E1E\u0E2A\u0E2B\u0E23\u0E31\u0E10\u0E40\ + \u0E1B\u0E34\u0E14\u0E10\u0E32\u0E19\u0E17\u0E31\u0E1E\u0E43\u0E2B\u0E21\u0E48\ + \u0E43\u0E19\u0E15\u0E48\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E1E\ + \u0E23\u0E49\u0E2D\u0E21\u0E1A\u0E38\u0E04\u0E25\u0E32\u0E01\u0E23\u0E2A\u0E2B\ + \u0E23\u0E31\u0E10 1,000 \u0E19\u0E32\u0E22" + B: "\u0E1C\u0E39\u0E49\u0E1A\u0E23\u0E34\u0E42\u0E20\u0E04\u0E0A\u0E32\u0E27\ + \u0E0D\u0E35\u0E48\u0E1B\u0E38\u0E48\u0E19\u0E0B\u0E37\u0E49\u0E2D\u0E0B\u0E35\ + \u0E14\u0E35\u0E17\u0E35\u0E48\u0E1C\u0E25\u0E34\u0E15\u0E43\u0E19\u0E2A\u0E2B\ + \u0E23\u0E31\u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E2B\u0E25\u0E32\ + \u0E22\u0E1E\u0E31\u0E19\u0E41\u0E1C\u0E48\u0E19" + C: "\u0E19\u0E31\u0E01\u0E23\u0E49\u0E2D\u0E07\u0E1B\u0E4A\u0E2D\u0E1B\u0E0A\ + \u0E32\u0E27\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E31\u0E19\u0E41\u0E2A\u0E14\ + \u0E07\u0E04\u0E2D\u0E19\u0E40\u0E2A\u0E34\u0E23\u0E4C\u0E15\u0E17\u0E35\u0E48\ + \u0E1A\u0E31\u0E15\u0E23\u0E2B\u0E21\u0E14\u0E43\u0E19\u0E1B\u0E32\u0E23\u0E35\ + \u0E2A" + D: "\u0E01\u0E32\u0E23\u0E1C\u0E25\u0E34\u0E15\u0E25\u0E30\u0E04\u0E23\u0E02\ + \u0E2D\u0E07\u0E1D\u0E23\u0E31\u0E48\u0E07\u0E40\u0E28\u0E2A\u0E2D\u0E2D\u0E01\ + \u0E17\u0E31\u0E27\u0E23\u0E4C\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E15\u0E48\u0E32\ + \u0E07\u0E46 \u0E02\u0E2D\u0E07\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E2B\ + \u0E25\u0E32\u0E22\u0E2A\u0E34\u0E1A\u0E41\u0E2B\u0E48\u0E07" + input_correct_responses: + - C + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E44\u0E21\u0E48\u0E23\u0E27\u0E21\u0E2D\u0E22\u0E39\u0E48\u0E43\ + \u0E19 GDP \u0E02\u0E2D\u0E07\u0E2A\u0E2B\u0E23\u0E31\u0E10\u0E2D\u0E40\u0E21\ + \u0E23\u0E34\u0E01\u0E32" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\ + \u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\ + \u0E01\u0E32\u0E23\u0E27\u0E48\u0E32\u0E07\u0E07\u0E32\u0E19\u0E41\u0E25\u0E30\ + \u0E2D\u0E31\u0E15\u0E23\u0E32\u0E40\u0E07\u0E34\u0E19\u0E40\u0E1F\u0E49\u0E2D" + B: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\ + \u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\ + \u0E23\u0E32\u0E04\u0E32\u0E41\u0E25\u0E30\u0E1B\u0E23\u0E34\u0E21\u0E32\u0E13\ + \u0E17\u0E35\u0E48\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23" + C: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\ + \u0E1C\u0E01\u0E1C\u0E31\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E23\ + \u0E32\u0E04\u0E32\u0E41\u0E25\u0E30\u0E1B\u0E23\u0E34\u0E21\u0E32\u0E13\u0E17\ + \u0E35\u0E48\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23" + D: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\ + \u0E1C\u0E01\u0E1C\u0E31\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E01\ + \u0E32\u0E23\u0E27\u0E48\u0E32\u0E07\u0E07\u0E32\u0E19\u0E41\u0E25\u0E30\u0E2D\ + \u0E31\u0E15\u0E23\u0E32\u0E40\u0E07\u0E34\u0E19\u0E40\u0E1F\u0E49\u0E2D" + input_correct_responses: + - D + input_question: "\u0E40\u0E2A\u0E49\u0E19\u0E42\u0E04\u0E49\u0E07 Phillips \u0E23\ + \u0E30\u0E22\u0E30\u0E2A\u0E31\u0E49\u0E19\u0E1A\u0E48\u0E07\u0E0A\u0E35\u0E49\ + \u0E27\u0E48\u0E32" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E2A\u0E48\u0E07\u0E2D\u0E2D\u0E01\u0E21\u0E32\u0E01\ + \u0E01\u0E27\u0E48\u0E32\u0E01\u0E32\u0E23\u0E19\u0E33\u0E40\u0E02\u0E49\u0E32" + B: "\u0E01\u0E32\u0E23\u0E19\u0E33\u0E40\u0E02\u0E49\u0E32\u0E21\u0E32\u0E01\ + \u0E01\u0E27\u0E48\u0E32\u0E01\u0E32\u0E23\u0E2A\u0E48\u0E07\u0E2D\u0E2D\u0E01" + C: "\u0E01\u0E32\u0E23\u0E40\u0E01\u0E47\u0E1A\u0E20\u0E32\u0E29\u0E35\u0E02\ + \u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E40\ + \u0E01\u0E34\u0E19\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22" + D: "\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E40\u0E01\ + \u0E34\u0E19\u0E01\u0E27\u0E48\u0E32\u0E23\u0E32\u0E22\u0E44\u0E14\u0E49\u0E20\ + \u0E32\u0E29\u0E35\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\ + \u0E25\u0E32\u0E07" + input_correct_responses: + - D + input_question: "\u0E01\u0E32\u0E23\u0E02\u0E32\u0E14\u0E14\u0E38\u0E25\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E40\u0E01\ + \u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E40\u0E21\u0E37\u0E48\u0E2D" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E2D\u0E31\u0E15\u0E23\ + \u0E32\u0E04\u0E34\u0E14\u0E25\u0E14" + B: "\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E2A\u0E31\u0E14\u0E2A\ + \u0E48\u0E27\u0E19\u0E2A\u0E33\u0E23\u0E2D\u0E07" + C: "\u0E01\u0E32\u0E23\u0E0B\u0E37\u0E49\u0E2D\u0E2B\u0E25\u0E31\u0E01\u0E17\ + \u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\ + \u0E25" + D: "\u0E01\u0E32\u0E23\u0E25\u0E14\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E20\u0E32\ + \u0E29\u0E35" + input_correct_responses: + - C + input_question: "\u0E01\u0E32\u0E23\u0E16\u0E37\u0E2D\u0E04\u0E23\u0E2D\u0E07\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E2D\u0E37\u0E48\u0E19\u0E40\u0E17\u0E48\u0E32\u0E01\ + \u0E31\u0E19\u0E19\u0E42\u0E22\u0E1A\u0E32\u0E22\u0E01\u0E32\u0E23\u0E40\u0E07\ + \u0E34\u0E19\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E17\ + \u0E35\u0E48\u0E08\u0E30\u0E43\u0E0A\u0E49\u0E01\u0E23\u0E30\u0E15\u0E38\u0E49\ + \u0E19\u0E01\u0E32\u0E23\u0E2A\u0E48\u0E07\u0E2D\u0E2D\u0E01\u0E02\u0E2D\u0E07\ + \u0E2A\u0E2B\u0E23\u0E31\u0E10\u0E2F" + - input_choice_list: + A: "\u0E1B\u0E23\u0E34\u0E21\u0E32\u0E13\u0E40\u0E07\u0E34\u0E19\u0E17\u0E35\ + \u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + B: "\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22\u0E20\u0E32\ + \u0E04\u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\ + \u0E36\u0E49\u0E19" + C: "\u0E25\u0E14\u0E20\u0E32\u0E29\u0E35\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\ + \u0E27\u0E34\u0E08\u0E31\u0E22\u0E41\u0E25\u0E30\u0E1E\u0E31\u0E12\u0E19\u0E32\ + \u0E40\u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35\u0E43\u0E2B\u0E21\u0E48" + D: "\u0E20\u0E32\u0E29\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E39\u0E07\u0E02\u0E36\ + \u0E49\u0E19\u0E08\u0E32\u0E01\u0E23\u0E32\u0E22\u0E44\u0E14\u0E49\u0E04\u0E23\ + \u0E31\u0E27\u0E40\u0E23\u0E37\u0E2D\u0E19" + input_correct_responses: + - C + input_question: "\u0E19\u0E42\u0E22\u0E1A\u0E32\u0E22\u0E02\u0E49\u0E2D\u0E43\u0E14\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E2D\u0E18\u0E34\u0E1A\u0E32\ + \u0E22\u0E19\u0E42\u0E22\u0E1A\u0E32\u0E22\u0E01\u0E32\u0E23\u0E04\u0E25\u0E31\ + \u0E07\u0E14\u0E49\u0E32\u0E19\u0E2D\u0E38\u0E1B\u0E17\u0E32\u0E19\u0E44\u0E14\ + \u0E49\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_macroeconomics +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_high_school_macroeconomics +task_alias: high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..623e6fc5086c51cf44b99660068dcb6f509b200e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_mathematics.yaml @@ -0,0 +1,98 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: '3' + B: '15' + C: '6' + D: '5' + input_correct_responses: + - B + input_question: "\u0E42\u0E08\u0E23\u0E31\u0E1A\u0E1C\u0E34\u0E14\u0E0A\u0E2D\u0E1A\ + \u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E41\u0E2A\u0E07\u0E2A\u0E33\u0E2B\u0E23\ + \u0E31\u0E1A\u0E01\u0E32\u0E23\u0E40\u0E15\u0E49\u0E19\u0E23\u0E33 \u0E44\u0E1F\ + \u0E2A\u0E35\u0E41\u0E14\u0E07\u0E01\u0E30\u0E1E\u0E23\u0E34\u0E1A\u0E17\u0E38\ + \u0E01\u0E46 2 \u0E27\u0E34\u0E19\u0E32\u0E17\u0E35 \u0E44\u0E1F\u0E2A\u0E35\ + \u0E40\u0E2B\u0E25\u0E37\u0E2D\u0E07\u0E17\u0E38\u0E01\u0E46 3 \u0E27\u0E34\u0E19\ + \u0E32\u0E17\u0E35 \u0E41\u0E25\u0E30\u0E44\u0E1F\u0E2A\u0E35\u0E19\u0E49\u0E33\ + \u0E40\u0E07\u0E34\u0E19\u0E17\u0E38\u0E01\u0E46 5 \u0E27\u0E34\u0E19\u0E32\u0E17\ + \u0E35 \u0E2B\u0E32\u0E01\u0E23\u0E27\u0E21\u0E08\u0E38\u0E14\u0E40\u0E23\u0E34\ + \u0E48\u0E21\u0E15\u0E49\u0E19\u0E41\u0E25\u0E30\u0E08\u0E38\u0E14\u0E2A\u0E34\ + \u0E49\u0E19\u0E2A\u0E38\u0E14\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E40\u0E15\ + \u0E49\u0E19\u0E23\u0E33 \u0E01\u0E35\u0E48\u0E04\u0E23\u0E31\u0E49\u0E07\u0E23\ + \u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E01\u0E32\u0E23\u0E40\u0E15\u0E49\u0E19\ + \u0E23\u0E33\u0E40\u0E08\u0E47\u0E14\u0E19\u0E32\u0E17\u0E35 \u0E44\u0E1F\u0E17\ + \u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\u0E08\u0E30\u0E2A\u0E27\u0E48\u0E32\u0E07\ + \u0E02\u0E36\u0E49\u0E19\u0E1E\u0E23\u0E49\u0E2D\u0E21\u0E01\u0E31\u0E19 (\u0E2A\ + \u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32\u0E44\u0E1F\u0E17\u0E31\u0E49\u0E07\ + \u0E2A\u0E32\u0E21\u0E14\u0E27\u0E07\u0E01\u0E30\u0E1E\u0E23\u0E34\u0E1A\u0E1E\ + \u0E23\u0E49\u0E2D\u0E21\u0E01\u0E31\u0E19\u0E17\u0E35\u0E48\u0E08\u0E38\u0E14\ + \u0E40\u0E23\u0E34\u0E48\u0E21\u0E15\u0E49\u0E19\u0E02\u0E2D\u0E07\u0E01\u0E32\ + \u0E23\u0E40\u0E15\u0E49\u0E19\u0E23\u0E33)" + - input_choice_list: + A: '12' + B: '1' + C: '30' + D: '5' + input_correct_responses: + - C + input_question: "\u0E2B\u0E49\u0E32\u0E1E\u0E31\u0E19\u0E14\u0E2D\u0E25\u0E25\u0E32\ + \u0E23\u0E4C\u0E17\u0E1A\u0E15\u0E49\u0E19\u0E15\u0E48\u0E2D\u0E1B\u0E35\u0E17\ + \u0E35\u0E48\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E14\u0E2D\u0E01\u0E40\u0E1A\u0E35\ + \u0E49\u0E22 $x\\%$ \u0E43\u0E0A\u0E49\u0E40\u0E27\u0E25\u0E32\u0E2B\u0E01\u0E1B\ + \u0E35\u0E43\u0E19\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E40\u0E1B\ + \u0E47\u0E19\u0E2A\u0E2D\u0E07\u0E40\u0E17\u0E48\u0E32 \u0E17\u0E35\u0E48\u0E2D\ + \u0E31\u0E15\u0E23\u0E32\u0E14\u0E2D\u0E01\u0E40\u0E1A\u0E35\u0E49\u0E22\u0E40\ + \u0E14\u0E35\u0E22\u0E27\u0E01\u0E31\u0E19 $\\$300$ \u0E08\u0E30\u0E43\u0E0A\ + \u0E49\u0E40\u0E27\u0E25\u0E32\u0E01\u0E35\u0E48\u0E1B\u0E35\u0E08\u0E36\u0E07\ + \u0E08\u0E30\u0E40\u0E15\u0E34\u0E1A\u0E42\u0E15\u0E40\u0E1B\u0E47\u0E19 $\\\ + $9600$" + - input_choice_list: + A: '-1' + B: '16' + C: -\frac{1}{256} + D: \frac{1}{16} + input_correct_responses: + - C + input_question: "\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23 $x$ \u0E41\u0E1B\u0E23\u0E1C\ + \u0E31\u0E19\u0E15\u0E23\u0E07\u0E43\u0E19\u0E23\u0E39\u0E1B\u0E01\u0E33\u0E25\ + \u0E31\u0E07\u0E2A\u0E2D\u0E07\u0E02\u0E2D\u0E07 $y$ \u0E41\u0E25\u0E30 $y$\ + \ \u0E41\u0E1B\u0E23\u0E1C\u0E31\u0E19\u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E40\ + \u0E1B\u0E47\u0E19\u0E23\u0E39\u0E1B\u0E25\u0E39\u0E01\u0E1A\u0E32\u0E28\u0E01\ + \u0E4C\u0E02\u0E2D\u0E07 $z$ \u0E16\u0E49\u0E32 $x$ \u0E40\u0E17\u0E48\u0E32\ + \u0E01\u0E31\u0E1A $-16$ \u0E40\u0E21\u0E37\u0E48\u0E2D $z$ \u0E40\u0E17\u0E48\ + \u0E32\u0E01\u0E31\u0E1A 2 \u0E04\u0E48\u0E32\u0E02\u0E2D\u0E07 $x$ \u0E40\u0E21\ + \u0E37\u0E48\u0E2D $z$ \u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A $\\frac{1}{2}$\ + \ \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23" + - input_choice_list: + A: \frac{3\sqrt{3}}{3} + B: \frac{1}{3} + C: \sqrt{3} + D: \frac{\sqrt{3}}{3} + input_correct_responses: + - D + input_question: "\u0E25\u0E14\u0E04\u0E27\u0E32\u0E21\u0E0B\u0E31\u0E1A\u0E0B\u0E49\ + \u0E2D\u0E19\u0E41\u0E25\u0E30\u0E40\u0E02\u0E35\u0E22\u0E19\u0E1C\u0E25\u0E25\ + \u0E31\u0E1E\u0E18\u0E4C\u0E14\u0E49\u0E27\u0E22\u0E15\u0E31\u0E27\u0E2A\u0E48\ + \u0E27\u0E19\u0E17\u0E35\u0E48\u0E21\u0E35\u0E40\u0E2B\u0E15\u0E38\u0E1C\u0E25\ + : $$\\sqrt{\\sqrt[3]{\\sqrt{\\frac{1}{729}}}}$$" + - input_choice_list: + A: '55' + B: '60' + C: '62' + D: '65' + input_correct_responses: + - D + input_question: "\u0E19\u0E31\u0E01\u0E40\u0E23\u0E35\u0E22\u0E19 10 \u0E04\u0E19\ + \u0E17\u0E33\u0E41\u0E1A\u0E1A\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E27\u0E34\u0E0A\ + \u0E32\u0E0A\u0E35\u0E27\u0E27\u0E34\u0E17\u0E22\u0E32\u0E41\u0E25\u0E30\u0E44\ + \u0E14\u0E49\u0E04\u0E30\u0E41\u0E19\u0E19\u0E14\u0E31\u0E07\u0E19\u0E35\u0E49\ + \ 45, 55, 50, 70, 65, 80, 40, 90, 70, 85 \u0E04\u0E30\u0E41\u0E19\u0E19\u0E2A\ + \u0E2D\u0E1A\u0E02\u0E2D\u0E07\u0E19\u0E31\u0E01\u0E40\u0E23\u0E35\u0E22\u0E19\ + \u0E21\u0E35\u0E04\u0E48\u0E32\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22\u0E40\u0E17\ + \u0E48\u0E32\u0E43\u0E14" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_mathematics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_high_school_mathematics +task_alias: high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d79dae44d6b11f14928f896291715a49b569b540 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_microeconomics.yaml @@ -0,0 +1,107 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E04\u0E48\u0E32\u0E08\u0E49\u0E32\u0E07\u0E02\u0E31\u0E49\u0E19\u0E15\ + \u0E48\u0E33\u0E17\u0E35\u0E48\u0E21\u0E35\u0E1B\u0E23\u0E30\u0E2A\u0E34\u0E17\ + \u0E18\u0E34\u0E20\u0E32\u0E1E\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E43\u0E19\u0E15\ + \u0E25\u0E32\u0E14\u0E41\u0E23\u0E07\u0E07\u0E32\u0E19\u0E19\u0E35\u0E49" + B: "\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\ + \u0E02\u0E2D\u0E07\u0E23\u0E32\u0E04\u0E32\u0E2A\u0E35\u0E41\u0E01\u0E25\u0E25\ + \u0E2D\u0E19" + C: "\u0E01\u0E32\u0E23\u0E01\u0E48\u0E2D\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E1A\ + \u0E49\u0E32\u0E19\u0E43\u0E2B\u0E21\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\ + \u0E36\u0E49\u0E19" + D: "\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\ + \u0E02\u0E2D\u0E07\u0E23\u0E32\u0E04\u0E32\u0E02\u0E2D\u0E07\u0E0A\u0E48\u0E32\ + \u0E07\u0E2A\u0E35\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E01\u0E25\u0E15\ + \u0E23\u0E32\u0E1A\u0E40\u0E17\u0E48\u0E32\u0E17\u0E35\u0E48\u0E1C\u0E25\u0E25\ + \u0E31\u0E1E\u0E18\u0E4C\u0E17\u0E35\u0E48\u0E2D\u0E2D\u0E01\u0E21\u0E32\u0E40\ + \u0E01\u0E34\u0E19\u0E01\u0E27\u0E48\u0E32\u0E40\u0E2D\u0E1F\u0E40\u0E1F\u0E01\ + \u0E15\u0E4C\u0E01\u0E32\u0E23\u0E17\u0E14\u0E41\u0E17\u0E19" + input_correct_responses: + - C + input_question: "\u0E43\u0E19\u0E15\u0E25\u0E32\u0E14\u0E41\u0E23\u0E07\u0E07\u0E32\ + \u0E19\u0E17\u0E35\u0E48\u0E21\u0E35\u0E01\u0E32\u0E23\u0E41\u0E02\u0E48\u0E07\ + \u0E02\u0E31\u0E19\u0E2A\u0E39\u0E07\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E0A\ + \u0E48\u0E32\u0E07\u0E17\u0E32\u0E2A\u0E35\u0E1A\u0E49\u0E32\u0E19 \u0E02\u0E49\ + \u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E08\u0E30\ + \u0E40\u0E1E\u0E34\u0E48\u0E21\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\ + \u0E01\u0E32\u0E23\u0E0A\u0E48\u0E32\u0E07\u0E17\u0E32\u0E2A\u0E35\u0E1A\u0E49\ + \u0E32\u0E19" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E2A\ + \u0E34\u0E19\u0E04\u0E49\u0E32\u0E08\u0E30\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\ + \u0E36\u0E49\u0E19" + B: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E2A\ + \u0E34\u0E19\u0E04\u0E49\u0E32\u0E08\u0E30\u0E25\u0E14\u0E25\u0E07" + C: "\u0E2A\u0E48\u0E27\u0E19\u0E40\u0E01\u0E34\u0E19\u0E02\u0E2D\u0E07\u0E1C\ + \u0E39\u0E49\u0E1A\u0E23\u0E34\u0E42\u0E20\u0E04\u0E08\u0E30\u0E40\u0E1E\u0E34\ + \u0E48\u0E21\u0E02\u0E36\u0E49\u0E19" + D: "\u0E2A\u0E48\u0E27\u0E19\u0E40\u0E01\u0E34\u0E19\u0E02\u0E2D\u0E07\u0E1C\ + \u0E39\u0E49\u0E1A\u0E23\u0E34\u0E42\u0E20\u0E04\u0E08\u0E30\u0E25\u0E14\u0E25\ + \u0E07" + input_correct_responses: + - C + input_question: "\u0E16\u0E49\u0E32\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E2D\u0E38\ + \u0E14\u0E2B\u0E19\u0E38\u0E19\u0E1C\u0E39\u0E49\u0E1C\u0E25\u0E34\u0E15\u0E43\ + \u0E19\u0E15\u0E25\u0E32\u0E14\u0E17\u0E35\u0E48\u0E21\u0E35\u0E01\u0E32\u0E23\ + \u0E41\u0E02\u0E48\u0E07\u0E02\u0E31\u0E19\u0E2A\u0E21\u0E1A\u0E39\u0E23\u0E13\ + \u0E4C\u0E25\u0E48\u0E30\u0E01\u0E47" + - input_choice_list: + A: '0' + B: '5' + C: '10' + D: '100' + input_correct_responses: + - D + input_question: "\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E2A\u0E48\u0E27\u0E19\u0E04\u0E27\ + \u0E32\u0E21\u0E40\u0E02\u0E49\u0E21\u0E02\u0E49\u0E19\u0E2A\u0E33\u0E2B\u0E23\ + \u0E31\u0E1A\u0E01\u0E32\u0E23\u0E1C\u0E39\u0E01\u0E02\u0E32\u0E14\u0E04\u0E37\ + \u0E2D" + - input_choice_list: + A: "\u0E23\u0E32\u0E04\u0E32\u0E1E\u0E37\u0E49\u0E19\u0E40\u0E25\u0E37\u0E48\ + \u0E2D\u0E19\u0E40\u0E2A\u0E49\u0E19\u0E2D\u0E38\u0E1B\u0E2A\u0E07\u0E04\u0E4C\ + \u0E44\u0E1B\u0E17\u0E32\u0E07\u0E0B\u0E49\u0E32\u0E22" + B: "\u0E1E\u0E37\u0E49\u0E19\u0E17\u0E35\u0E48\u0E21\u0E35\u0E1B\u0E23\u0E30\ + \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E20\u0E32\u0E1E\u0E17\u0E33\u0E43\u0E2B\u0E49\ + \u0E40\u0E01\u0E34\u0E14\u0E01\u0E32\u0E23\u0E02\u0E32\u0E14\u0E41\u0E04\u0E25\ + \u0E19\u0E17\u0E35\u0E48\u0E14\u0E35" + C: "\u0E23\u0E32\u0E04\u0E32\u0E1E\u0E37\u0E49\u0E19\u0E40\u0E25\u0E37\u0E48\ + \u0E2D\u0E19\u0E40\u0E2A\u0E49\u0E19\u0E2D\u0E38\u0E1B\u0E17\u0E32\u0E19\u0E02\ + \u0E2D\u0E07\u0E2A\u0E34\u0E19\u0E04\u0E49\u0E32\u0E44\u0E1B\u0E17\u0E32\u0E07\ + \u0E02\u0E27\u0E32" + D: "\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\u0E19\ + \u0E23\u0E32\u0E04\u0E32\u0E02\u0E31\u0E49\u0E19\u0E15\u0E48\u0E33\u0E17\u0E35\ + \u0E48\u0E21\u0E35\u0E1B\u0E23\u0E30\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E20\u0E32\ + \u0E1E \u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E15\u0E31\u0E49\u0E07\u0E44\u0E27\ + \u0E49\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E23\u0E32\u0E04\u0E32\u0E14\u0E38\u0E25\ + \u0E22\u0E20\u0E32\u0E1E" + input_correct_responses: + - D + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07\u0E2A\u0E33\u0E2B\ + \u0E23\u0E31\u0E1A\u0E23\u0E32\u0E04\u0E32\u0E1E\u0E37\u0E49\u0E19" + - input_choice_list: + A: "\u0E40\u0E02\u0E49\u0E32\u0E41\u0E25\u0E30\u0E2D\u0E2D\u0E01\u0E08\u0E32\ + \u0E01\u0E15\u0E25\u0E32\u0E14\u0E44\u0E14\u0E49\u0E1F\u0E23\u0E35" + B: "\u0E1C\u0E39\u0E49\u0E1C\u0E25\u0E34\u0E15\u0E23\u0E32\u0E22\u0E43\u0E2B\ + \u0E0D\u0E48\u0E44\u0E21\u0E48\u0E01\u0E35\u0E48\u0E23\u0E32\u0E22" + C: "\u0E1C\u0E39\u0E49\u0E1C\u0E25\u0E34\u0E15\u0E2A\u0E34\u0E19\u0E04\u0E49\ + \u0E32\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E23\u0E32\u0E22\u0E17\u0E35\u0E48\u0E44\ + \u0E21\u0E48\u0E21\u0E35\u0E1C\u0E39\u0E49\u0E17\u0E14\u0E41\u0E17\u0E19\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E43\u0E01\u0E25\u0E49\u0E0A\u0E34\u0E14" + D: "\u0E1C\u0E25\u0E34\u0E15\u0E20\u0E31\u0E13\u0E11\u0E4C\u0E17\u0E35\u0E48\ + \u0E40\u0E1B\u0E47\u0E19\u0E40\u0E19\u0E37\u0E49\u0E2D\u0E40\u0E14\u0E35\u0E22\ + \u0E27\u0E01\u0E31\u0E19" + input_correct_responses: + - B + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\u0E02\ + \u0E2D\u0E07\u0E1C\u0E39\u0E49\u0E02\u0E32\u0E22\u0E19\u0E49\u0E2D\u0E22\u0E23\ + \u0E32\u0E22" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_microeconomics +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_high_school_microeconomics +task_alias: high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97e8f92a78995b749afdb68ee3f430972eb87d79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_physics.yaml @@ -0,0 +1,111 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E09\u0E31\u0E19\u0E41\u0E25\u0E30 II \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\ + \u0E49\u0E19" + B: "I \u0E41\u0E25\u0E30 III \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + C: "II \u0E41\u0E25\u0E30 III \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + D: "III \u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + input_correct_responses: + - D + input_question: "\u0E40\u0E07\u0E37\u0E48\u0E2D\u0E19\u0E44\u0E02\u0E43\u0E14\u0E15\ + \u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E08\u0E30\u0E0A\u0E48\u0E27\u0E22\ + \u0E43\u0E2B\u0E49\u0E21\u0E31\u0E48\u0E19\u0E43\u0E08\u0E44\u0E14\u0E49\u0E27\ + \u0E48\u0E32\u0E42\u0E21\u0E40\u0E21\u0E19\u0E15\u0E31\u0E21\u0E40\u0E0A\u0E34\ + \u0E07\u0E21\u0E38\u0E21\u0E08\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\ + \u0E32\u0E23\u0E2D\u0E19\u0E38\u0E23\u0E31\u0E01\u0E29\u0E4C\u0E44\u0E27\u0E49\ + \ I. \u0E01\u0E32\u0E23\u0E2D\u0E19\u0E38\u0E23\u0E31\u0E01\u0E29\u0E4C\u0E42\ + \u0E21\u0E40\u0E21\u0E19\u0E15\u0E31\u0E21\u0E40\u0E0A\u0E34\u0E07\u0E40\u0E2A\ + \u0E49\u0E19 II. \u0E41\u0E23\u0E07\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01\u0E2A\ + \u0E38\u0E17\u0E18\u0E34\u0E40\u0E1B\u0E47\u0E19\u0E28\u0E39\u0E19\u0E22\u0E4C\ + \ III \u0E41\u0E23\u0E07\u0E1A\u0E34\u0E14\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01\ + \u0E2A\u0E38\u0E17\u0E18\u0E34\u0E40\u0E1B\u0E47\u0E19\u0E28\u0E39\u0E19\u0E22\ + \u0E4C" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E14\u0E31\u0E19\u0E2D\u0E22\u0E39\u0E48\u0E17\ + \u0E35\u0E48\u0E42\u0E2B\u0E19\u0E14 \u0E41\u0E15\u0E48\u0E01\u0E32\u0E23\u0E01\ + \u0E23\u0E30\u0E08\u0E31\u0E14\u0E02\u0E2D\u0E07\u0E2D\u0E19\u0E38\u0E20\u0E32\ + \u0E04\u0E2D\u0E22\u0E39\u0E48\u0E17\u0E35\u0E48\u0E41\u0E2D\u0E19\u0E15\u0E34\ + \u0E42\u0E19\u0E14" + B: "\u0E04\u0E27\u0E32\u0E21\u0E14\u0E31\u0E19\u0E2D\u0E22\u0E39\u0E48\u0E17\ + \u0E35\u0E48\u0E41\u0E2D\u0E19\u0E15\u0E34\u0E42\u0E19\u0E14 \u0E41\u0E15\u0E48\ + \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E08\u0E31\u0E14\u0E02\u0E2D\u0E07\u0E2D\ + \u0E19\u0E38\u0E20\u0E32\u0E04\u0E2D\u0E22\u0E39\u0E48\u0E17\u0E35\u0E48\u0E42\ + \u0E2B\u0E19\u0E14" + C: "\u0E04\u0E27\u0E32\u0E21\u0E14\u0E31\u0E19\u0E41\u0E25\u0E30\u0E01\u0E32\ + \u0E23\u0E01\u0E23\u0E30\u0E08\u0E31\u0E14\u0E02\u0E2D\u0E07\u0E2D\u0E19\u0E38\ + \u0E20\u0E32\u0E04\u0E2D\u0E22\u0E39\u0E48\u0E17\u0E35\u0E48\u0E42\u0E2B\u0E19\ + \u0E14" + D: "\u0E04\u0E27\u0E32\u0E21\u0E14\u0E31\u0E19\u0E41\u0E25\u0E30\u0E01\u0E32\ + \u0E23\u0E01\u0E23\u0E30\u0E08\u0E31\u0E14\u0E02\u0E2D\u0E07\u0E2D\u0E19\u0E38\ + \u0E20\u0E32\u0E04\u0E2D\u0E22\u0E39\u0E48\u0E17\u0E35\u0E48\u0E41\u0E2D\u0E19\ + \u0E15\u0E34\u0E42\u0E19\u0E14" + input_correct_responses: + - B + input_question: "\u0E17\u0E48\u0E2D\u0E17\u0E35\u0E48\u0E40\u0E15\u0E47\u0E21\u0E44\ + \u0E1B\u0E14\u0E49\u0E27\u0E22\u0E2D\u0E32\u0E01\u0E32\u0E28\u0E16\u0E39\u0E01\ + \u0E1B\u0E34\u0E14\u0E17\u0E35\u0E48\u0E1B\u0E25\u0E32\u0E22\u0E14\u0E49\u0E32\ + \u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07 \u0E08\u0E30\u0E40\u0E01\u0E34\u0E14\u0E04\ + \u0E25\u0E37\u0E48\u0E19\u0E19\u0E34\u0E48\u0E07\u0E02\u0E36\u0E49\u0E19\u0E43\ + \u0E19\u0E17\u0E48\u0E2D \u0E17\u0E33\u0E43\u0E2B\u0E49\u0E17\u0E48\u0E2D\u0E2A\ + \u0E48\u0E07\u0E40\u0E2A\u0E35\u0E22\u0E07\u0E42\u0E19\u0E49\u0E15 \u0E02\u0E49\ + \u0E2D\u0E43\u0E14\u0E01\u0E25\u0E48\u0E32\u0E27\u0E16\u0E39\u0E01\u0E15\u0E49\ + \u0E2D\u0E07\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E04\u0E38\ + \u0E13\u0E2A\u0E21\u0E1A\u0E31\u0E15\u0E34\u0E02\u0E2D\u0E07\u0E04\u0E25\u0E37\ + \u0E48\u0E19\u0E17\u0E35\u0E48\u0E1B\u0E25\u0E32\u0E22\u0E17\u0E48\u0E2D\u0E1B\ + \u0E34\u0E14" + - input_choice_list: + A: "02:00 \u0E19" + B: 06:00 + C: "12:00 \u0E19" + D: "24 \u0E01" + input_correct_responses: + - D + input_question: "\u0E15\u0E32\u0E41\u0E21\u0E27\u0E02\u0E2D\u0E07\u0E1F\u0E31\u0E07\ + \u0E01\u0E4C\u0E0A\u0E31\u0E19\u0E01\u0E32\u0E23\u0E17\u0E33\u0E07\u0E32\u0E19\ + \ \u03D5 = 2eV \u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E21\u0E15\u0E48\u0E2D\u0E01\u0E31\ + \u0E1A\u0E15\u0E31\u0E27\u0E15\u0E49\u0E32\u0E19\u0E17\u0E32\u0E19\u0E41\u0E1A\ + \u0E1A\u0E2D\u0E19\u0E38\u0E01\u0E23\u0E21 \u0E41\u0E2A\u0E07\u0E04\u0E27\u0E32\ + \u0E21\u0E16\u0E35\u0E48 f = 1 \xD7 10^15 Hz \u0E15\u0E01\u0E01\u0E23\u0E30\u0E17\ + \u0E1A\u0E41\u0E1C\u0E48\u0E19\u0E42\u0E25\u0E2B\u0E30\u0E02\u0E2D\u0E07\u0E15\ + \u0E32\u0E41\u0E21\u0E27 \u0E16\u0E49\u0E32\u0E01\u0E33\u0E25\u0E31\u0E07\u0E02\ + \u0E2D\u0E07\u0E41\u0E2A\u0E07\u0E04\u0E37\u0E2D P = 100 W \u0E01\u0E23\u0E30\ + \u0E41\u0E2A\u0E17\u0E35\u0E48\u0E44\u0E2B\u0E25\u0E1C\u0E48\u0E32\u0E19\u0E15\ + \u0E31\u0E27\u0E15\u0E49\u0E32\u0E19\u0E17\u0E32\u0E19\u0E08\u0E30\u0E40\u0E1B\ + \u0E47\u0E19\u0E40\u0E17\u0E48\u0E32\u0E43\u0E14" + - input_choice_list: + A: "10 \u0E27" + B: "30 \u0E27" + C: "60 \u0E27" + D: "240 \u0E27" + input_correct_responses: + - D + input_question: "\u0E40\u0E15\u0E32\u0E44\u0E21\u0E42\u0E04\u0E23\u0E40\u0E27\u0E1F\ + \u0E15\u0E48\u0E2D\u0E40\u0E02\u0E49\u0E32\u0E01\u0E31\u0E1A\u0E40\u0E15\u0E49\ + \u0E32\u0E23\u0E31\u0E1A 120 V \u0E41\u0E25\u0E30\u0E43\u0E0A\u0E49\u0E01\u0E23\ + \u0E30\u0E41\u0E2A\u0E44\u0E1F 2 \u0E41\u0E2D\u0E21\u0E1B\u0E4C \u0E40\u0E15\ + \u0E32\u0E44\u0E21\u0E42\u0E04\u0E23\u0E40\u0E27\u0E1F\u0E43\u0E0A\u0E49\u0E1E\ + \u0E25\u0E31\u0E07\u0E07\u0E32\u0E19\u0E43\u0E19\u0E2D\u0E31\u0E15\u0E23\u0E32\ + \u0E40\u0E17\u0E48\u0E32\u0E43\u0E14" + - input_choice_list: + A: "3.5 \u0E40\u0E08" + B: "6.0 \u0E40\u0E08" + C: "22.5 \u0E40\u0E08" + D: "40 \u0E40\u0E08" + input_correct_responses: + - B + input_question: "\u0E08\u0E38\u0E14\u0E0A\u0E32\u0E23\u0E4C\u0E08 Q = +1 mC \u0E44\ + \u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E41\u0E01\u0E49\u0E44\u0E02\ + \u0E17\u0E35\u0E48\u0E08\u0E38\u0E14\u0E01\u0E33\u0E40\u0E19\u0E34\u0E14 \u0E15\ + \u0E49\u0E2D\u0E07\u0E43\u0E0A\u0E49\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E22\u0E32\ + \u0E22\u0E32\u0E21\u0E40\u0E17\u0E48\u0E32\u0E43\u0E14\u0E43\u0E19\u0E01\u0E32\ + \u0E23\u0E40\u0E04\u0E25\u0E37\u0E48\u0E2D\u0E19\u0E22\u0E49\u0E32\u0E22\u0E1B\ + \u0E23\u0E30\u0E08\u0E38 Q = +8 \xB5C \u0E08\u0E32\u0E01\u0E08\u0E38\u0E14 (0,\ + \ 4 \u0E40\u0E21\u0E15\u0E23) \u0E44\u0E1B\u0E22\u0E31\u0E07\u0E08\u0E38\u0E14\ + \ (3 \u0E40\u0E21\u0E15\u0E23, 0)" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_physics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_high_school_physics +task_alias: high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..427fa5a1fb03b9c6eb43488a71ae9fd420b1c178 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_psychology.yaml @@ -0,0 +1,140 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2B\u0E34\u0E23\u0E34\u0E42\u0E2D\u0E15\u0E15\u0E31\u0E1B\u0E1B\u0E30\ + \u0E17\u0E35\u0E48\u0E41\u0E02\u0E47\u0E07\u0E41\u0E01\u0E23\u0E48\u0E07" + B: "\u0E04\u0E27\u0E32\u0E21\u0E19\u0E31\u0E1A\u0E16\u0E37\u0E2D\u0E15\u0E19\ + \u0E40\u0E2D\u0E07\u0E15\u0E48\u0E33" + C: "\u0E01\u0E32\u0E23\u0E23\u0E31\u0E1A\u0E23\u0E39\u0E49\u0E04\u0E27\u0E32\ + \u0E21\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E02\u0E2D\u0E07\u0E15\u0E19\u0E40\ + \u0E2D\u0E07\u0E15\u0E48\u0E33" + D: "\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E20\u0E32\u0E22\u0E43\u0E19\u0E02\u0E2D\ + \u0E07\u0E01\u0E32\u0E23\u0E04\u0E27\u0E1A\u0E04\u0E38\u0E21" + input_correct_responses: + - D + input_question: "Ani \u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E27\u0E48\u0E32\u0E17\u0E31\ + \u0E28\u0E19\u0E04\u0E15\u0E34\u0E41\u0E25\u0E30\u0E1E\u0E24\u0E15\u0E34\u0E01\ + \u0E23\u0E23\u0E21\u0E02\u0E2D\u0E07\u0E40\u0E18\u0E2D\u0E21\u0E35\u0E1A\u0E17\ + \u0E1A\u0E32\u0E17\u0E2A\u0E33\u0E04\u0E31\u0E0D\u0E43\u0E19\u0E2A\u0E34\u0E48\ + \u0E07\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E01\ + \u0E31\u0E1A\u0E40\u0E18\u0E2D \u0E04\u0E27\u0E32\u0E21\u0E40\u0E0A\u0E37\u0E48\ + \u0E2D\u0E14\u0E31\u0E07\u0E01\u0E25\u0E48\u0E32\u0E27\u0E19\u0E48\u0E32\u0E08\ + \u0E30\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\ + \u0E1A" + - input_choice_list: + A: "\u0E23\u0E30\u0E1A\u0E38\u0E2A\u0E32\u0E40\u0E2B\u0E15\u0E38\u0E41\u0E25\ + \u0E30\u0E41\u0E19\u0E27\u0E17\u0E32\u0E07\u0E41\u0E01\u0E49\u0E44\u0E02\u0E1B\ + \u0E31\u0E0D\u0E2B\u0E32\u0E17\u0E35\u0E48\u0E25\u0E39\u0E01\u0E04\u0E49\u0E32\ + \u0E19\u0E33\u0E40\u0E2A\u0E19\u0E2D" + B: "\u0E23\u0E30\u0E1A\u0E38\u0E41\u0E25\u0E30\u0E02\u0E08\u0E31\u0E14\u0E2A\ + \u0E32\u0E40\u0E2B\u0E15\u0E38\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\u0E22\ + \u0E32\u0E01\u0E25\u0E33\u0E1A\u0E32\u0E01\u0E43\u0E19\u0E01\u0E32\u0E23\u0E08\ + \u0E31\u0E14\u0E01\u0E32\u0E23\u0E1B\u0E31\u0E0D\u0E2B\u0E32\u0E02\u0E2D\u0E07\ + \u0E1C\u0E39\u0E49\u0E43\u0E2B\u0E49\u0E04\u0E33\u0E1B\u0E23\u0E36\u0E01\u0E29\ + \u0E32" + C: "\u0E01\u0E32\u0E23\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E25\u0E33\u0E14\u0E31\ + \u0E1A\u0E0A\u0E31\u0E49\u0E19\u0E02\u0E2D\u0E07\u0E2D\u0E33\u0E19\u0E32\u0E08\ + \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E2A\u0E32\u0E21\u0E32\u0E23\ + \u0E16\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08\u0E44\u0E14\u0E49\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E21\u0E35\u0E1B\u0E23\u0E30\u0E2A\u0E34\u0E17\u0E18\ + \u0E34\u0E20\u0E32\u0E1E" + D: "\u0E19\u0E33\u0E40\u0E2A\u0E19\u0E2D\u0E41\u0E19\u0E27\u0E17\u0E32\u0E07\ + \u0E01\u0E32\u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\u0E01\u0E32\u0E23\u0E40\ + \u0E14\u0E35\u0E22\u0E27\u0E17\u0E35\u0E48\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19\ + \u0E41\u0E25\u0E30\u0E44\u0E21\u0E48\u0E04\u0E25\u0E38\u0E21\u0E40\u0E04\u0E23\ + \u0E37\u0E2D\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E17\u0E35\u0E48\u0E1B\u0E23\ + \u0E36\u0E01\u0E29\u0E32\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E40\u0E2D\u0E32\u0E0A\ + \u0E19\u0E30\u0E01\u0E32\u0E23\u0E02\u0E32\u0E14\u0E17\u0E31\u0E01\u0E29\u0E30" + input_correct_responses: + - B + input_question: "\u0E15\u0E32\u0E21\u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\u0E01\u0E32\ + \u0E23\u0E43\u0E2B\u0E49\u0E04\u0E33\u0E1B\u0E23\u0E36\u0E01\u0E29\u0E32\u0E01\ + \u0E23\u0E13\u0E35\u0E17\u0E35\u0E48\u0E1B\u0E23\u0E36\u0E01\u0E29\u0E32\u0E40\ + \u0E1B\u0E47\u0E19\u0E28\u0E39\u0E19\u0E22\u0E4C\u0E01\u0E25\u0E32\u0E07\u0E02\ + \u0E2D\u0E07 Caplan \u0E17\u0E35\u0E48\u0E1B\u0E23\u0E36\u0E01\u0E29\u0E32\u0E2A\ + \u0E19\u0E43\u0E08\u0E40\u0E1B\u0E47\u0E19\u0E2B\u0E25\u0E31\u0E01" + - input_choice_list: + A: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E08\u0E30\u0E16\u0E39\u0E01\ + \u0E2A\u0E48\u0E07\u0E08\u0E32\u0E01\u0E17\u0E32\u0E25\u0E32\u0E21\u0E31\u0E2A\ + \u0E44\u0E1B\u0E22\u0E31\u0E07\u0E2D\u0E21\u0E34\u0E01\u0E14\u0E32\u0E25\u0E32\ + \u0E42\u0E14\u0E22\u0E15\u0E23\u0E07" + B: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E08\u0E30\u0E16\u0E39\u0E01\ + \u0E2A\u0E48\u0E07\u0E08\u0E32\u0E01\u0E10\u0E32\u0E19\u0E14\u0E2D\u0E01\u0E44\ + \u0E1B\u0E22\u0E31\u0E07\u0E17\u0E32\u0E07\u0E40\u0E14\u0E34\u0E19 "\u0E2D\ + \u0E30\u0E44\u0E23" \u0E41\u0E25\u0E30 "\u0E17\u0E35\u0E48\u0E44\ + \u0E2B\u0E19"" + C: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E16\u0E39\u0E01\u0E2A\u0E48\ + \u0E07\u0E08\u0E32\u0E01\u0E23\u0E30\u0E1A\u0E1A\u0E1B\u0E23\u0E30\u0E2A\u0E32\ + \u0E17\u0E01\u0E23\u0E30\u0E0B\u0E34\u0E01\u0E44\u0E1B\u0E22\u0E31\u0E07\u0E40\ + \u0E1B\u0E25\u0E37\u0E2D\u0E01\u0E2A\u0E21\u0E2D\u0E07" + D: "\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E16\u0E39\u0E01\u0E2A\u0E48\ + \u0E07\u0E08\u0E32\u0E01\u0E2A\u0E21\u0E2D\u0E07\u0E2A\u0E48\u0E27\u0E19\u0E2B\ + \u0E19\u0E49\u0E32\u0E44\u0E1B\u0E22\u0E31\u0E07\u0E15\u0E48\u0E2D\u0E21\u0E43\ + \u0E15\u0E49\u0E2A\u0E21\u0E2D\u0E07" + input_correct_responses: + - A + input_question: "\u0E02\u0E13\u0E30\u0E17\u0E35\u0E48\u0E01\u0E33\u0E25\u0E31\u0E07\ + \u0E27\u0E48\u0E32\u0E22\u0E19\u0E49\u0E33\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\ + \u0E21\u0E2B\u0E32\u0E2A\u0E21\u0E38\u0E17\u0E23 \u0E2D\u0E35\u0E27\u0E32\u0E19\ + \u0E23\u0E39\u0E49\u0E2A\u0E36\u0E01\u0E2B\u0E27\u0E32\u0E14\u0E01\u0E25\u0E31\ + \u0E27\u0E01\u0E31\u0E1A\u0E40\u0E07\u0E32\u0E14\u0E33\u0E43\u0E19\u0E19\u0E49\ + \u0E33 \u0E01\u0E48\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E40\u0E02\u0E32\u0E08\u0E30\ + \u0E21\u0E35\u0E42\u0E2D\u0E01\u0E32\u0E2A\u0E23\u0E30\u0E1A\u0E38\u0E44\u0E14\ + \u0E49\u0E27\u0E48\u0E32\u0E40\u0E07\u0E32\u0E19\u0E31\u0E49\u0E19\u0E04\u0E37\ + \u0E2D\u0E2D\u0E30\u0E44\u0E23 \u0E01\u0E32\u0E23\u0E40\u0E0A\u0E37\u0E48\u0E2D\ + \u0E21\u0E15\u0E48\u0E2D synaptic \u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\ + \u0E02\u0E36\u0E49\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E40\u0E2B\ + \u0E15\u0E38\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E19\u0E48\u0E32\ + \u0E2A\u0E30\u0E1E\u0E23\u0E36\u0E07\u0E01\u0E25\u0E31\u0E27\u0E19\u0E35\u0E49\ + \u0E2D\u0E18\u0E34\u0E1A\u0E32\u0E22\u0E44\u0E14\u0E49\u0E14\u0E35\u0E17\u0E35\ + \u0E48\u0E2A\u0E38\u0E14\u0E42\u0E14\u0E22\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\ + \u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49" + - input_choice_list: + A: "\u0E43\u0E2B\u0E49\u0E40\u0E14\u0E47\u0E01\u0E44\u0E14\u0E49\u0E17\u0E14\ + \u0E25\u0E2D\u0E07\u0E43\u0E19\u0E2A\u0E20\u0E32\u0E1E\u0E41\u0E27\u0E14\u0E25\ + \u0E49\u0E2D\u0E21\u0E43\u0E2B\u0E21\u0E48" + B: "\u0E41\u0E08\u0E49\u0E07\u0E43\u0E2B\u0E49\u0E1C\u0E39\u0E49\u0E1B\u0E01\ + \u0E04\u0E23\u0E2D\u0E07\u0E17\u0E23\u0E32\u0E1A\u0E40\u0E1B\u0E47\u0E19\u0E25\ + \u0E32\u0E22\u0E25\u0E31\u0E01\u0E29\u0E13\u0E4C\u0E2D\u0E31\u0E01\u0E29\u0E23" + C: "\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2D\u0E19\u0E38\ + \u0E21\u0E31\u0E15\u0E34\u0E08\u0E32\u0E01\u0E04\u0E13\u0E30\u0E01\u0E23\u0E23\ + \u0E21\u0E01\u0E32\u0E23\u0E42\u0E23\u0E07\u0E40\u0E23\u0E35\u0E22\u0E19" + D: "\u0E02\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\ + \u0E08\u0E32\u0E01\u0E1C\u0E39\u0E49\u0E1B\u0E01\u0E04\u0E23\u0E2D\u0E07" + input_correct_responses: + - B + input_question: "\u0E15\u0E32\u0E21\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E01\u0E32\ + \u0E23\u0E1B\u0E23\u0E31\u0E1A\u0E1B\u0E23\u0E38\u0E07\u0E01\u0E32\u0E23\u0E28\ + \u0E36\u0E01\u0E29\u0E32\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E1A\u0E38\u0E04\ + \u0E04\u0E25\u0E17\u0E38\u0E1E\u0E1E\u0E25\u0E20\u0E32\u0E1E \u0E02\u0E49\u0E2D\ + \u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E17\u0E35\u0E48\ + \u0E2B\u0E19\u0E48\u0E27\u0E22\u0E07\u0E32\u0E19\u0E01\u0E32\u0E23\u0E28\u0E36\ + \u0E01\u0E29\u0E32\u0E15\u0E49\u0E2D\u0E07\u0E17\u0E33\u0E01\u0E48\u0E2D\u0E19\ + \u0E17\u0E35\u0E48\u0E08\u0E30\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\u0E19\u0E15\ + \u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E01\u0E32\u0E23\u0E28\u0E36\u0E01\u0E29\ + \u0E32\u0E02\u0E2D\u0E07\u0E19\u0E31\u0E01\u0E40\u0E23\u0E35\u0E22\u0E19\u0E17\ + \u0E35\u0E48\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E34\u0E01\u0E32\u0E23" + - input_choice_list: + A: "\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E27\u0E31\u0E12\u0E19\u0E18\u0E23\u0E23\ + \u0E21" + B: "\u0E17\u0E32\u0E07\u0E04\u0E25\u0E34\u0E19\u0E34\u0E01" + C: "\u0E04\u0E27\u0E32\u0E21\u0E23\u0E39\u0E49\u0E04\u0E27\u0E32\u0E21\u0E40\ + \u0E02\u0E49\u0E32\u0E43\u0E08" + D: "\u0E19\u0E31\u0E01\u0E1E\u0E24\u0E15\u0E34\u0E01\u0E23\u0E23\u0E21" + input_correct_responses: + - C + input_question: "Pascale \u0E2A\u0E19\u0E43\u0E08\u0E43\u0E19\u0E01\u0E25\u0E22\ + \u0E38\u0E17\u0E18\u0E4C\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E21\u0E27\u0E25\ + \u0E1C\u0E25\u0E17\u0E35\u0E48\u0E40\u0E14\u0E47\u0E01\u0E46 \u0E43\u0E0A\u0E49\ + \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E40\u0E23\u0E35\u0E22\u0E19\u0E23\u0E39\u0E49\ + \u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E43\u0E2B\u0E21\u0E48\u0E46 Pascale \u0E08\ + \u0E30\u0E16\u0E39\u0E01\u0E08\u0E31\u0E14\u0E27\u0E48\u0E32\u0E40\u0E1B\u0E47\ + \u0E19\u0E19\u0E31\u0E01\u0E08\u0E34\u0E15\u0E27\u0E34\u0E17\u0E22\u0E32\u0E1B\ + \u0E23\u0E30\u0E40\u0E20\u0E17\u0E43\u0E14\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\ + \u0E38\u0E14?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_psychology +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_high_school_psychology +task_alias: high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8de42d8d64f94b7d1e4f26b9245fb6d3668fe9a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_statistics.yaml @@ -0,0 +1,167 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E16\u0E49\u0E32\u0E04\u0E27\u0E32\u0E21\u0E0A\u0E31\u0E19\u0E02\u0E2D\ + \u0E07\u0E40\u0E2A\u0E49\u0E19\u0E16\u0E14\u0E16\u0E2D\u0E22\u0E40\u0E17\u0E48\ + \u0E32\u0E01\u0E31\u0E1A 1 \u0E40\u0E1B\u0E4A\u0E30\u0E46 \u0E04\u0E27\u0E32\ + \u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\u0E01\u0E47\u0E08\u0E30\ + \u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A 1 \u0E40\u0E1B\u0E4A\u0E30\u0E46" + B: "\u0E2B\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\ + \u0E19\u0E18\u0E4C\u0E40\u0E1B\u0E47\u0E19 0 \u0E04\u0E27\u0E32\u0E21\u0E0A\ + \u0E31\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E2A\u0E49\u0E19\u0E16\u0E14\u0E16\u0E2D\ + \u0E22\u0E08\u0E30\u0E44\u0E21\u0E48\u0E16\u0E39\u0E01\u0E01\u0E33\u0E2B\u0E19\ + \u0E14" + C: "\u0E01\u0E32\u0E23\u0E2A\u0E25\u0E31\u0E1A\u0E15\u0E31\u0E27\u0E41\u0E1B\ + \u0E23\u0E17\u0E35\u0E48\u0E40\u0E23\u0E35\u0E22\u0E01\u0E27\u0E48\u0E32 x\ + \ \u0E41\u0E25\u0E30\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23\u0E17\u0E35\u0E48\ + \u0E40\u0E23\u0E35\u0E22\u0E01\u0E27\u0E48\u0E32 y \u0E08\u0E30\u0E40\u0E1B\ + \u0E25\u0E35\u0E48\u0E22\u0E19\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E2B\ + \u0E21\u0E32\u0E22\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\ + \u0E1E\u0E31\u0E19\u0E18\u0E4C" + D: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\ + \ r \u0E40\u0E17\u0E48\u0E32\u0E01\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E0A\ + \u0E31\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E2A\u0E49\u0E19\u0E16\u0E14\u0E16\u0E2D\ + \u0E22\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E04\u0E30\u0E41\u0E19\u0E19 z \u0E2A\ + \u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23 y \u0E16\ + \u0E39\u0E01\u0E1E\u0E25\u0E47\u0E2D\u0E15\u0E40\u0E17\u0E35\u0E22\u0E1A\u0E01\ + \u0E31\u0E1A\u0E04\u0E30\u0E41\u0E19\u0E19 z \u0E2A\u0E33\u0E2B\u0E23\u0E31\ + \u0E1A\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23 x" + input_correct_responses: + - D + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E01\u0E25\u0E48\u0E32\u0E27\u0E16\ + \u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\ + \u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\ + \u0E4C" + - input_choice_list: + A: E(X + Y) = 99, var(X + Y) = 8.5 + B: E(X + Y) = 99, var(X + Y) = 13 + C: E(X + Y) = 99, var(X + Y) = 17 + D: "\u0E21\u0E35\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E44\u0E21\u0E48\u0E40\ + \u0E1E\u0E35\u0E22\u0E07\u0E1E\u0E2D\u0E17\u0E35\u0E48\u0E08\u0E30\u0E15\u0E2D\ + \u0E1A\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49" + input_correct_responses: + - D + input_question: "\u0E2A\u0E21\u0E21\u0E15\u0E34\u0E27\u0E48\u0E32 X \u0E41\u0E25\ + \u0E30 Y \u0E40\u0E1B\u0E47\u0E19\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23\u0E2A\u0E38\ + \u0E48\u0E21\u0E17\u0E35\u0E48\u0E21\u0E35 E(X) = 37, var(X) = 5, E(Y) = 62\ + \ \u0E41\u0E25\u0E30 var(Y) = 12 \u0E04\u0E48\u0E32\u0E17\u0E35\u0E48\u0E04\u0E32\ + \u0E14\u0E2B\u0E27\u0E31\u0E07\u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E41\ + \u0E1B\u0E23\u0E1B\u0E23\u0E27\u0E19\u0E02\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E41\ + \u0E1B\u0E23\u0E2A\u0E38\u0E48\u0E21 X + \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\ + \u0E23 \u0E27\u0E32\u0E22?" + - input_choice_list: + A: "\u0E2A\u0E31\u0E14\u0E2A\u0E48\u0E27\u0E19\u0E02\u0E2D\u0E07\u0E15\u0E49\ + \u0E19\u0E44\u0E21\u0E49\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E2A\u0E35\u0E22\u0E2B\u0E32\u0E22\u0E21\u0E32\u0E01\u0E01\u0E27\ + \u0E48\u0E32\u0E23\u0E49\u0E2D\u0E22\u0E25\u0E30 50 \u0E40\u0E19\u0E37\u0E48\ + \u0E2D\u0E07\u0E08\u0E32\u0E01\u0E19\u0E49\u0E33\u0E04\u0E49\u0E32\u0E07\u0E41\ + \u0E02\u0E47\u0E07" + B: "\u0E08\u0E33\u0E19\u0E27\u0E19\u0E15\u0E49\u0E19\u0E44\u0E21\u0E49\u0E17\ + \u0E35\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E1C\u0E25\u0E01\u0E23\u0E30\ + \u0E17\u0E1A\u0E08\u0E32\u0E01\u0E19\u0E49\u0E33\u0E04\u0E49\u0E32\u0E07\u0E41\ + \u0E02\u0E47\u0E07" + C: "\u0E08\u0E33\u0E19\u0E27\u0E19\u0E15\u0E49\u0E19\u0E44\u0E21\u0E49\u0E17\ + \u0E35\u0E48\u0E2A\u0E38\u0E48\u0E21\u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E08\u0E32\u0E01\u0E1B\u0E48\u0E32\u0E25\u0E30\u0E40\u0E21\u0E32\u0E30" + D: "\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E15\u0E49\u0E19\u0E44\u0E21\u0E49\ + \u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E41\u0E15\u0E48\u0E25\u0E30\ + \u0E15\u0E49\u0E19\u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\u0E08\u0E30\u0E44\u0E14\ + \u0E49\u0E23\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E40\u0E2A\u0E35\u0E22\u0E2B\ + \u0E32\u0E22\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32\u0E23\u0E49\u0E2D\u0E22\ + \u0E25\u0E30 50 \u0E2B\u0E23\u0E37\u0E2D\u0E40\u0E2A\u0E35\u0E22\u0E2B\u0E32\ + \u0E22\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E23\u0E49\u0E2D\ + \u0E22\u0E25\u0E30 50" + input_correct_responses: + - D + input_question: "\u0E2B\u0E25\u0E31\u0E07\u0E08\u0E32\u0E01\u0E1B\u0E23\u0E30\u0E01\ + \u0E32\u0E28\u0E40\u0E15\u0E37\u0E2D\u0E19\u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\ + \u0E19\u0E49\u0E33\u0E04\u0E49\u0E32\u0E07\u0E41\u0E02\u0E47\u0E07 \u0E40\u0E08\ + \u0E49\u0E32\u0E02\u0E2D\u0E07\u0E2A\u0E27\u0E19\u0E2A\u0E49\u0E21\u0E02\u0E19\ + \u0E32\u0E14\u0E43\u0E2B\u0E0D\u0E48\u0E02\u0E2D\u0E43\u0E2B\u0E49\u0E04\u0E19\ + \u0E07\u0E32\u0E19\u0E09\u0E35\u0E14\u0E19\u0E49\u0E33\u0E23\u0E14\u0E15\u0E49\ + \u0E19\u0E44\u0E21\u0E49\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 \u0E19\u0E49\ + \u0E33\u0E04\u0E27\u0E23\u0E08\u0E30\u0E01\u0E25\u0E32\u0E22\u0E40\u0E1B\u0E47\ + \u0E19\u0E19\u0E49\u0E33\u0E41\u0E02\u0E47\u0E07\u0E41\u0E25\u0E30\u0E01\u0E48\ + \u0E2D\u0E15\u0E31\u0E27\u0E40\u0E1B\u0E47\u0E19\u0E19\u0E49\u0E33\u0E41\u0E02\ + \u0E47\u0E07\u0E1B\u0E01\u0E04\u0E25\u0E38\u0E21\u0E23\u0E2D\u0E1A\u0E14\u0E2D\ + \u0E01\u0E2A\u0E49\u0E21 \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E23\u0E01\u0E47\ + \u0E15\u0E32\u0E21 \u0E40\u0E08\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E2A\u0E07\u0E2A\ + \u0E31\u0E22\u0E27\u0E48\u0E32\u0E15\u0E49\u0E19\u0E44\u0E21\u0E49\u0E1A\u0E32\ + \u0E07\u0E15\u0E49\u0E19\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E2A\u0E35\u0E22\u0E2B\u0E32\u0E22\u0E21\u0E32\u0E01\u0E40\u0E19\ + \u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E19\u0E49\u0E33\u0E04\u0E49\u0E32\ + \u0E07\u0E41\u0E02\u0E47\u0E07 \u0E43\u0E19\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\ + \u0E40\u0E21\u0E34\u0E19\u0E2A\u0E31\u0E14\u0E2A\u0E48\u0E27\u0E19\u0E02\u0E2D\ + \u0E07\u0E15\u0E49\u0E19\u0E44\u0E21\u0E49\u0E17\u0E35\u0E48\u0E44\u0E14\u0E49\ + \u0E23\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E40\u0E2A\u0E35\u0E22\u0E2B\u0E32\ + \u0E22\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32 50 \u0E40\u0E1B\u0E2D\u0E23\ + \u0E4C\u0E40\u0E0B\u0E47\u0E19\u0E15\u0E4C\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\ + \u0E08\u0E32\u0E01\u0E19\u0E49\u0E33\u0E04\u0E49\u0E32\u0E07\u0E41\u0E02\u0E47\ + \u0E07 \u0E40\u0E02\u0E32\u0E2A\u0E38\u0E48\u0E21\u0E15\u0E31\u0E27\u0E2D\u0E22\ + \u0E48\u0E32\u0E07\u0E15\u0E49\u0E19\u0E44\u0E21\u0E49 100 \u0E15\u0E49\u0E19\ + \u0E08\u0E32\u0E01\u0E1B\u0E48\u0E32\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32 \u0E15\ + \u0E31\u0E27\u0E41\u0E1B\u0E23\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E43\ + \u0E19\u0E01\u0E32\u0E23\u0E17\u0E14\u0E25\u0E2D\u0E07\u0E19\u0E35\u0E49\u0E04\ + \u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23?" + - input_choice_list: + A: "\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22 518 \u0E01\u0E23\u0E31\u0E21; \u0E2A\ + \u0E48\u0E27\u0E19\u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\u0E19\u0E21\ + \u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 7.0 \u0E01\u0E23\u0E31\u0E21" + B: "\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22 518 \u0E01\u0E23\u0E31\u0E21; \u0E2A\ + \u0E48\u0E27\u0E19\u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\u0E19\u0E21\ + \u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 3.5 \u0E01\u0E23\u0E31\u0E21" + C: "\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22 518 \u0E01\u0E23\u0E31\u0E21; \u0E2A\ + \u0E48\u0E27\u0E19\u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\u0E19\u0E21\ + \u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 6.1 \u0E01\u0E23\u0E31\u0E21" + D: "\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22 394 \u0E01\u0E23\u0E31\u0E21; \u0E2A\ + \u0E48\u0E27\u0E19\u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\u0E19\u0E21\ + \u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 6.1 \u0E01\u0E23\u0E31\u0E21" + input_correct_responses: + - C + input_question: "\u0E2A\u0E21\u0E32\u0E23\u0E4C\u0E17\u0E27\u0E2D\u0E17\u0E0A\u0E4C\ + \u0E43\u0E2B\u0E21\u0E48\u0E1C\u0E25\u0E34\u0E15\u0E02\u0E36\u0E49\u0E19\u0E43\ + \u0E19\u0E2A\u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E02\u0E2D\u0E07\ + \u0E42\u0E23\u0E07\u0E07\u0E32\u0E19 \u0E08\u0E32\u0E01\u0E19\u0E31\u0E49\u0E19\ + \u0E08\u0E36\u0E07\u0E08\u0E31\u0E14\u0E2A\u0E48\u0E07\u0E44\u0E1B\u0E22\u0E31\ + \u0E07\u0E2D\u0E35\u0E01\u0E2A\u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\ + \u0E02\u0E2D\u0E07\u0E42\u0E23\u0E07\u0E07\u0E32\u0E19 \u0E19\u0E49\u0E33\u0E2B\ + \u0E19\u0E31\u0E01\u0E02\u0E2D\u0E07\u0E2A\u0E21\u0E32\u0E23\u0E4C\u0E17\u0E27\ + \u0E2D\u0E17\u0E0A\u0E4C\u0E21\u0E35\u0E04\u0E48\u0E32\u0E40\u0E09\u0E25\u0E35\ + \u0E48\u0E22 62 \u0E01\u0E23\u0E31\u0E21 \u0E41\u0E25\u0E30\u0E04\u0E48\u0E32\ + \u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\u0E19\u0E21\u0E32\u0E15\u0E23\ + \u0E10\u0E32\u0E19 1.0 \u0E01\u0E23\u0E31\u0E21 \u0E19\u0E49\u0E33\u0E2B\u0E19\ + \u0E31\u0E01\u0E02\u0E2D\u0E07\u0E1A\u0E23\u0E23\u0E08\u0E38\u0E20\u0E31\u0E13\ + \u0E11\u0E4C (\u0E01\u0E25\u0E48\u0E2D\u0E07 \u0E04\u0E39\u0E48\u0E21\u0E37\u0E2D\ + \u0E1C\u0E39\u0E49\u0E43\u0E0A\u0E49 \u0E41\u0E1C\u0E48\u0E19\u0E01\u0E31\u0E19\ + \u0E01\u0E23\u0E30\u0E41\u0E17\u0E01 \u0E2F\u0E25\u0E2F) \u0E21\u0E35\u0E04\u0E48\ + \u0E32\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22 456 \u0E01\u0E23\u0E31\u0E21 \u0E41\ + \u0E25\u0E30\u0E04\u0E48\u0E32\u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\ + \u0E19\u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 6 \u0E01\u0E23\u0E31\u0E21\ + \ \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E23\u0E27\u0E21\u0E01\u0E31\u0E19\u0E41\u0E25\ + \u0E49\u0E27 \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E08\u0E32\u0E22\u0E19\u0E49\ + \u0E33\u0E2B\u0E19\u0E31\u0E01\u0E02\u0E2D\u0E07\u0E2A\u0E21\u0E32\u0E23\u0E4C\ + \u0E17\u0E27\u0E2D\u0E17\u0E0A\u0E4C\u0E41\u0E25\u0E30\u0E1A\u0E23\u0E23\u0E08\ + \u0E38\u0E20\u0E31\u0E13\u0E11\u0E4C\u0E08\u0E30\u0E21\u0E35\u0E04\u0E48\u0E32\ + \u0E40\u0E09\u0E25\u0E35\u0E48\u0E22\u0E41\u0E25\u0E30\u0E2A\u0E48\u0E27\u0E19\ + \u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\u0E40\u0E1A\u0E19\u0E21\u0E32\u0E15\u0E23\ + \u0E10\u0E32\u0E19\u0E14\u0E31\u0E07\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\ + \u0E49:" + - input_choice_list: + A: "\u0E2A\u0E32\u0E21" + B: II, III + C: III, I + D: "III, \u0E04\u0E23\u0E31\u0E49\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07" + input_correct_responses: + - D + input_question: "\u0E0A\u0E38\u0E14\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E21\u0E35\u0E04\u0E48\u0E32\u0E40\u0E1A\u0E35\u0E48\u0E22\u0E07\ + \u0E40\u0E1A\u0E19\u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19\u0E19\u0E49\u0E2D\ + \u0E22\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14 \u0E0B\u0E36\u0E48\u0E07\u0E21\u0E35\ + \u0E02\u0E19\u0E32\u0E14\u0E43\u0E2B\u0E0D\u0E48\u0E17\u0E35\u0E48\u0E2A\u0E38\ + \u0E14? I: {1,2,3} II: {-10,10} III: {100}" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_statistics +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_high_school_statistics +task_alias: high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9f95fe19edc2ee590d411e361cb3cce1072f234 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_us_history.yaml @@ -0,0 +1,583 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E08\u0E31\u0E14\u0E01\u0E32\u0E23\u0E15\u0E2D\u0E1A\u0E42\u0E15\u0E49\ + \u0E01\u0E32\u0E23\u0E08\u0E25\u0E32\u0E08\u0E25\u0E02\u0E2D\u0E07\u0E40\u0E1A\ + \u0E04\u0E2D\u0E19" + B: "\u0E01\u0E32\u0E23\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E15\u0E48\ + \u0E2D\u0E01\u0E32\u0E23\u0E08\u0E25\u0E32\u0E08\u0E25\u0E02\u0E2D\u0E07 Shays" + C: "\u0E01\u0E32\u0E23\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E15\u0E48\ + \u0E2D\u0E01\u0E32\u0E23\u0E08\u0E25\u0E32\u0E08\u0E25\u0E27\u0E34\u0E2A\u0E01\ + \u0E35\u0E49" + D: "\u0E01\u0E32\u0E23\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E15\u0E48\ + \u0E2D\u0E01\u0E32\u0E23\u0E08\u0E25\u0E32\u0E08\u0E25\u0E02\u0E2D\u0E07\u0E1B\ + \u0E2D\u0E19\u0E40\u0E15\u0E35\u0E4A\u0E22\u0E01" + input_correct_responses: + - C + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 "\u0E2A\u0E31\u0E07\u0E04\ + \u0E21\u0E43\u0E19\u0E17\u0E38\u0E01\u0E23\u0E31\u0E10\u0E40\u0E1B\u0E47\u0E19\ + \u0E1E\u0E23\u0E30\u0E1E\u0E23 \u0E41\u0E15\u0E48\u0E23\u0E31\u0E10\u0E1A\u0E32\ + \u0E25\u0E41\u0E21\u0E49\u0E43\u0E19\u0E2A\u0E16\u0E32\u0E19\u0E30\u0E17\u0E35\ + \u0E48\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E01\u0E47\u0E40\u0E1B\ + \u0E47\u0E19\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E04\u0E27\u0E32\u0E21\u0E0A\u0E31\ + \u0E48\u0E27\u0E23\u0E49\u0E32\u0E22\u0E17\u0E35\u0E48\u0E08\u0E33\u0E40\u0E1B\ + \u0E47\u0E19 \u0E43\u0E19\u0E2A\u0E20\u0E32\u0E1E\u0E17\u0E35\u0E48\u0E40\u0E25\ + \u0E27\u0E23\u0E49\u0E32\u0E22\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E01\u0E47\ + \u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E17\u0E19\ + \u0E44\u0E21\u0E48\u0E44\u0E14\u0E49 \u0E40\u0E1E\u0E23\u0E32\u0E30\u0E40\u0E21\ + \u0E37\u0E48\u0E2D\u0E40\u0E23\u0E32\u0E1B\u0E23\u0E30\u0E2A\u0E1A\u0E04\u0E27\ + \u0E32\u0E21\u0E17\u0E38\u0E01\u0E02\u0E4C\u0E22\u0E32\u0E01\u0E2B\u0E23\u0E37\ + \u0E2D\u0E40\u0E1C\u0E0A\u0E34\u0E0D\u0E01\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\ + \u0E17\u0E38\u0E01\u0E02\u0E4C\u0E22\u0E32\u0E01\u0E41\u0E1A\u0E1A\u0E40\u0E14\ + \u0E35\u0E22\u0E27\u0E01\u0E31\u0E19\u0E42\u0E14\u0E22\u0E23\u0E31\u0E10\u0E1A\ + \u0E32\u0E25 \u0E0B\u0E36\u0E48\u0E07\u0E40\u0E23\u0E32\u0E2D\u0E32\u0E08\u0E04\ + \u0E32\u0E14\u0E2B\u0E27\u0E31\u0E07\u0E44\u0E14\u0E49 \u0E43\u0E19\u0E1B\u0E23\ + \u0E30\u0E40\u0E17\u0E28\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E21\u0E35\u0E23\ + \u0E31\u0E10\u0E1A\u0E32\u0E25 \u0E04\u0E27\u0E32\u0E21\u0E2B\u0E32\u0E22\u0E19\ + \u0E30\u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\ + \u0E36\u0E49\u0E19\u0E42\u0E14\u0E22\u0E2A\u0E30\u0E17\u0E49\u0E2D\u0E19\u0E43\ + \u0E2B\u0E49\u0E40\u0E2B\u0E47\u0E19\u0E27\u0E48\u0E32\u0E40\u0E23\u0E32\u0E08\ + \u0E31\u0E14\u0E40\u0E15\u0E23\u0E35\u0E22\u0E21\u0E27\u0E34\u0E18\u0E35\u0E01\ + \u0E32\u0E23\u0E17\u0E35\u0E48\u0E40\u0E23\u0E32\u0E1B\u0E23\u0E30\u0E2A\u0E1A\ + \u0E20\u0E31\u0E22 \u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E47\u0E40\u0E2B\ + \u0E21\u0E37\u0E2D\u0E19\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E41\u0E15\u0E48\ + \u0E07\u0E01\u0E32\u0E22 \u0E40\u0E1B\u0E47\u0E19\u0E15\u0E23\u0E32\u0E2A\u0E31\ + \u0E0D\u0E25\u0E31\u0E01\u0E29\u0E13\u0E4C\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\ + \u0E21\u0E44\u0E23\u0E49\u0E40\u0E14\u0E35\u0E22\u0E07\u0E2A\u0E32\u0E17\u0E35\ + \u0E48\u0E2B\u0E32\u0E22\u0E44\u0E1B \u0E1E\u0E23\u0E30\u0E23\u0E32\u0E0A\u0E27\ + \u0E31\u0E07\u0E02\u0E2D\u0E07\u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\ + \u0E16\u0E39\u0E01\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E02\u0E36\u0E49\u0E19\u0E1A\ + \u0E19\u0E0B\u0E32\u0E01\u0E1B\u0E23\u0E31\u0E01\u0E2B\u0E31\u0E01\u0E1E\u0E31\ + \u0E07\u0E02\u0E2D\u0E07\u0E14\u0E34\u0E19\u0E41\u0E14\u0E19\u0E41\u0E2B\u0E48\ + \u0E07\u0E2A\u0E23\u0E27\u0E07\u0E2A\u0E27\u0E23\u0E23\u0E04\u0E4C \u0E2A\u0E33\ + \u0E2B\u0E23\u0E31\u0E1A \u0E2B\u0E32\u0E01\u0E41\u0E23\u0E07\u0E01\u0E23\u0E30\ + \u0E15\u0E38\u0E49\u0E19\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\u0E23\u0E39\ + \u0E49\u0E2A\u0E36\u0E01\u0E1C\u0E34\u0E14\u0E0A\u0E2D\u0E1A\u0E0A\u0E31\u0E48\ + \u0E27\u0E14\u0E35\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19 \u0E2A\u0E21\u0E48\u0E33\ + \u0E40\u0E2A\u0E21\u0E2D \u0E41\u0E25\u0E30\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E1F\ + \u0E31\u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E21\u0E48\u0E2D\u0E32\u0E08\ + \u0E15\u0E49\u0E32\u0E19\u0E17\u0E32\u0E19\u0E44\u0E14\u0E49 \u0E21\u0E19\u0E38\ + \u0E29\u0E22\u0E4C\u0E01\u0E47\u0E44\u0E21\u0E48\u0E15\u0E49\u0E2D\u0E07\u0E01\ + \u0E32\u0E23\u0E1C\u0E39\u0E49\u0E23\u0E31\u0E01\u0E29\u0E32\u0E01\u0E0E\u0E2B\ + \u0E21\u0E32\u0E22\u0E04\u0E19\u0E2D\u0E37\u0E48\u0E19 \u0E41\u0E15\u0E48\u0E19\ + \u0E31\u0E48\u0E19\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E43\u0E19\u0E01\u0E23\ + \u0E13\u0E35\u0E19\u0E35\u0E49 \u0E40\u0E02\u0E32\u0E1E\u0E1A\u0E27\u0E48\u0E32\ + \u0E08\u0E33\u0E40\u0E1B\u0E47\u0E19\u0E15\u0E49\u0E2D\u0E07\u0E22\u0E2D\u0E21\ + \u0E2A\u0E25\u0E30\u0E17\u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E2A\u0E34\u0E19\u0E2A\ + \u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E02\u0E2D\u0E07\u0E40\u0E02\ + \u0E32\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E08\u0E31\u0E14\u0E2B\u0E32\u0E27\u0E34\ + \u0E18\u0E35\u0E01\u0E32\u0E23\u0E43\u0E19\u0E01\u0E32\u0E23\u0E04\u0E38\u0E49\ + \u0E21\u0E04\u0E23\u0E2D\u0E07\u0E2A\u0E48\u0E27\u0E19\u0E17\u0E35\u0E48\u0E40\ + \u0E2B\u0E25\u0E37\u0E2D \u0E41\u0E25\u0E30 \u0E2A\u0E34\u0E48\u0E07\u0E19\u0E35\ + \u0E49\u0E0A\u0E31\u0E01\u0E19\u0E33\u0E43\u0E2B\u0E49\u0E40\u0E02\u0E32\u0E17\ + \u0E33\u0E14\u0E49\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E40\u0E09\u0E25\u0E35\ + \u0E22\u0E27\u0E09\u0E25\u0E32\u0E14\u0E41\u0E1A\u0E1A\u0E40\u0E14\u0E35\u0E22\ + \u0E27\u0E01\u0E31\u0E19\u0E0B\u0E36\u0E48\u0E07\u0E43\u0E19\u0E17\u0E38\u0E01\ + \ \u0E46 \u0E01\u0E23\u0E13\u0E35\u0E08\u0E30\u0E41\u0E19\u0E30\u0E19\u0E33\u0E43\ + \u0E2B\u0E49\u0E40\u0E02\u0E32\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E04\u0E27\u0E32\ + \u0E21\u0E0A\u0E31\u0E48\u0E27\u0E23\u0E49\u0E32\u0E22\u0E19\u0E49\u0E2D\u0E22\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E08\u0E32\u0E01\u0E2A\u0E2D\u0E07\u0E04\ + \u0E27\u0E32\u0E21\u0E0A\u0E31\u0E48\u0E27\u0E23\u0E49\u0E32\u0E22 \u0E14\u0E31\ + \u0E07\u0E19\u0E31\u0E49\u0E19 \u0E04\u0E27\u0E32\u0E21\u0E1B\u0E25\u0E2D\u0E14\ + \u0E20\u0E31\u0E22\u0E04\u0E37\u0E2D\u0E01\u0E32\u0E23\u0E2D\u0E2D\u0E01\u0E41\ + \u0E1A\u0E1A\u0E17\u0E35\u0E48\u0E41\u0E17\u0E49\u0E08\u0E23\u0E34\u0E07\u0E41\ + \u0E25\u0E30\u0E08\u0E38\u0E14\u0E08\u0E1A\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\ + \u0E1A\u0E32\u0E25 \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E40\u0E23\u0E32\u0E41\ + \u0E25\u0E49\u0E27\u0E21\u0E35\u0E04\u0E48\u0E32\u0E43\u0E0A\u0E49\u0E08\u0E48\ + \u0E32\u0E22\u0E19\u0E49\u0E2D\u0E22\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E41\ + \u0E25\u0E30\u0E43\u0E2B\u0E49\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\u0E19\u0E4C\ + \u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14 \u0E14\u0E35\u0E01\u0E27\ + \u0E48\u0E32\u0E04\u0E19\u0E2D\u0E37\u0E48\u0E19\u0E46 \u0E17\u0E31\u0E49\u0E07\ + \u0E2B\u0E21\u0E14" Thomas Paine, Common Sense, 1776 "\u0E04\u0E27\ + \u0E32\u0E21\u0E17\u0E38\u0E01\u0E02\u0E4C\u0E22\u0E32\u0E01" \u0E02\u0E49\ + \u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E17\u0E35\ + \u0E48\u0E01\u0E25\u0E48\u0E32\u0E27\u0E16\u0E36\u0E07\u0E02\u0E49\u0E32\u0E07\ + \u0E15\u0E49\u0E19\u0E16\u0E39\u0E01\u0E01\u0E25\u0E38\u0E48\u0E21\u0E15\u0E48\ + \u0E2D\u0E15\u0E49\u0E32\u0E19\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\ + \u0E32\u0E07\u0E43\u0E19\u0E22\u0E38\u0E04\u0E2B\u0E25\u0E31\u0E07\u0E01\u0E32\ + \u0E23\u0E1B\u0E0F\u0E34\u0E27\u0E31\u0E15\u0E34\u0E1B\u0E23\u0E30\u0E13\u0E32\ + \u0E21\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E36\u0E07\u0E40\u0E04\u0E23\u0E35\u0E22\ + \u0E14\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E19\u0E42\u0E22\u0E1A\u0E32\ + \u0E22\u0E02\u0E2D\u0E07\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29\u0E41\u0E25\u0E30\ + \u0E41\u0E23\u0E07\u0E1A\u0E31\u0E19\u0E14\u0E32\u0E25\u0E43\u0E08\u0E02\u0E2D\ + \u0E07\u0E0A\u0E32\u0E27\u0E2D\u0E32\u0E13\u0E32\u0E19\u0E34\u0E04\u0E21\u0E43\ + \u0E19\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E40\u0E2B\u0E19\u0E37\u0E2D" + B: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E36\u0E07\u0E40\u0E04\u0E23\u0E35\u0E22\ + \u0E14\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E0A\u0E32\u0E27\u0E2D\u0E40\ + \u0E21\u0E23\u0E34\u0E01\u0E31\u0E19\u0E2D\u0E34\u0E19\u0E40\u0E14\u0E35\u0E22\ + \u0E19\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E31\u0E19\u0E18\u0E21\ + \u0E34\u0E15\u0E23\u0E01\u0E31\u0E1A\u0E1D\u0E23\u0E31\u0E48\u0E07\u0E40\u0E28\ + \u0E2A\u0E41\u0E25\u0E30\u0E1E\u0E31\u0E19\u0E18\u0E21\u0E34\u0E15\u0E23\u0E01\ + \u0E31\u0E1A\u0E2D\u0E31\u0E07\u0E01\u0E24\u0E29" + C: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E36\u0E07\u0E40\u0E04\u0E23\u0E35\u0E22\ + \u0E14\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E0A\u0E32\u0E27\u0E41\u0E2D\ + \u0E1F\u0E23\u0E34\u0E01\u0E31\u0E19\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E31\ + \u0E19\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E34\u0E2A\u0E23\u0E30\ + \u0E01\u0E31\u0E1A\u0E0A\u0E32\u0E27\u0E2A\u0E27\u0E19\u0E1C\u0E34\u0E27\u0E02\ + \u0E32\u0E27" + D: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E36\u0E07\u0E40\u0E04\u0E23\u0E35\u0E22\ + \u0E14\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E1C\u0E39\u0E49\u0E15\u0E31\ + \u0E49\u0E07\u0E16\u0E34\u0E48\u0E19\u0E10\u0E32\u0E19\u0E43\u0E19\u0E40\u0E02\ + \u0E15\u0E17\u0E38\u0E23\u0E01\u0E31\u0E19\u0E14\u0E32\u0E23\u0E41\u0E25\u0E30\ + \u0E0A\u0E19\u0E0A\u0E31\u0E49\u0E19\u0E2A\u0E39\u0E07\u0E43\u0E19\u0E2D\u0E32\ + \u0E13\u0E32\u0E19\u0E34\u0E04\u0E21\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32" + input_correct_responses: + - D + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 "\u0E43\u0E19\u0E02\u0E13\ + \u0E30\u0E17\u0E35\u0E48\u0E01\u0E32\u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\u0E19\ + \u0E01\u0E32\u0E23\u0E0A\u0E48\u0E27\u0E07\u0E1B\u0E25\u0E32\u0E22\u0E02\u0E2D\ + \u0E07\u0E40\u0E23\u0E32\u0E17\u0E35\u0E48 Conestoga Manor \u0E41\u0E25\u0E30\ + \ Lancaster \u0E44\u0E14\u0E49\u0E01\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E40\u0E01\ + \u0E34\u0E14\u0E01\u0E32\u0E23\u0E40\u0E01\u0E47\u0E07\u0E01\u0E33\u0E44\u0E23\ + \u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E23\u0E39\u0E49\u0E2A\u0E36\u0E01\ + \u0E17\u0E35\u0E48\u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\u0E2D\u0E22\ + \u0E48\u0E32\u0E07\u0E21\u0E32\u0E01\u0E43\u0E19\u0E23\u0E31\u0E10\u0E1A\u0E32\ + \u0E25\u0E19\u0E35\u0E49\u0E41\u0E25\u0E30\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\ + \u0E43\u0E01\u0E25\u0E49\u0E40\u0E04\u0E35\u0E22\u0E07 \u0E1A\u0E32\u0E07\u0E04\ + \u0E19\u0E41\u0E01\u0E49\u0E15\u0E31\u0E27\u0E41\u0E25\u0E30\u0E04\u0E19\u0E2D\ + \u0E37\u0E48\u0E19 \u0E46 \u0E1B\u0E23\u0E30\u0E13\u0E32\u0E21\u0E21\u0E31\u0E19\ + \ \u0E1A\u0E32\u0E07\u0E04\u0E19\u0E0A\u0E48\u0E27\u0E22\u0E1A\u0E23\u0E23\u0E40\ + \u0E17\u0E32\u0E2D\u0E32\u0E0A\u0E0D\u0E32\u0E01\u0E23\u0E23\u0E21 & \u0E04\ + \u0E19\u0E2D\u0E37\u0E48\u0E19 \u0E46 \u0E27\u0E32\u0E14\u0E20\u0E32\u0E1E\u0E19\ + \u0E35\u0E49\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E21\u0E38\u0E48\u0E07\u0E23\u0E49\ + \u0E32\u0E22 & \u0E2A\u0E35\u0E17\u0E35\u0E48\u0E19\u0E48\u0E32\u0E23\u0E31\ + \u0E07\u0E40\u0E01\u0E35\u0E22\u0E08, \u0E40\u0E23\u0E32\u0E04\u0E34\u0E14\u0E27\ + \u0E48\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\ + \u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E17\u0E35\u0E48\u0E08\u0E30\u0E15\u0E49\ + \u0E2D\u0E07\u0E27\u0E32\u0E07\u0E15\u0E48\u0E2D\u0E2B\u0E19\u0E49\u0E32 Publick,\ + \ \u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\ + \u0E15\u0E32\u0E21\u0E17\u0E35\u0E48\u0E1B\u0E23\u0E32\u0E01\u0E0F, & \u0E22\ + \u0E31\u0E07\u0E04\u0E07\u0E1B\u0E23\u0E32\u0E01\u0E0F, \u0E2A\u0E33\u0E2B\u0E23\ + \u0E31\u0E1A\u0E40\u0E23\u0E32 . . "\u0E2B\u0E32\u0E01\u0E2A\u0E34\u0E48\ + \u0E07\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E35\u0E49\u0E44\u0E21\u0E48\u0E40\ + \u0E1E\u0E35\u0E22\u0E07\u0E1E\u0E2D\u0E17\u0E35\u0E48\u0E08\u0E30\u0E1E\u0E34\ + \u0E2A\u0E39\u0E08\u0E19\u0E4C\u0E2A\u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E41\ + \u0E19\u0E1A\u0E21\u0E32\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E2A\u0E21\u0E40\ + \u0E2B\u0E15\u0E38\u0E2A\u0E21\u0E1C\u0E25\u0E43\u0E19 Quakers \u0E15\u0E48\u0E2D\ + \ Indians Savages , \u0E21\u0E15\u0E34\u0E17\u0E35\u0E48\u0E41\u0E19\u0E48\u0E19\ + \u0E2D\u0E19\u0E17\u0E35\u0E48\u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E1E\ + \u0E37\u0E48\u0E2D\u0E19\u0E01\u0E31\u0E1A\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\ + \ & \u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E23\u0E39\u0E49\u0E2A\u0E36\ + \u0E01\u0E16\u0E36\u0E07\u0E04\u0E27\u0E32\u0E21\u0E17\u0E38\u0E01\u0E02\u0E4C\ + \u0E22\u0E32\u0E01\u0E02\u0E2D\u0E07\u0E21\u0E19\u0E38\u0E29\u0E22\u0E4C \u0E43\ + \u0E2B\u0E49\u0E40\u0E23\u0E32\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\u0E02\ + \u0E49\u0E2D\u0E40\u0E17\u0E47\u0E08\u0E08\u0E23\u0E34\u0E07\u0E25\u0E48\u0E32\ + \u0E2A\u0E38\u0E14\u0E2D\u0E35\u0E01\u0E2A\u0E2D\u0E07\u0E2A\u0E32\u0E21\u0E02\ + \u0E49\u0E2D \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E23\u0E32\u0E1E\u0E1A\u0E27\ + \u0E48\u0E32\u0E24\u0E14\u0E39\u0E23\u0E49\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E41\ + \u0E25\u0E49\u0E27\u0E21\u0E35\u0E41\u0E19\u0E27\u0E42\u0E19\u0E49\u0E21\u0E27\ + \u0E48\u0E32\u0E08\u0E30\u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\ + \u0E04\u0E27\u0E32\u0E21\u0E0A\u0E48\u0E27\u0E22\u0E40\u0E2B\u0E25\u0E37\u0E2D\ + \u0E08\u0E32\u0E01\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25 \u0E2D\u0E32\u0E2A\u0E32\ + \u0E2A\u0E21\u0E31\u0E04\u0E23\u0E1A\u0E32\u0E07\u0E04\u0E19\u0E2D\u0E2D\u0E01\ + \u0E44\u0E1B\u0E42\u0E14\u0E22\u0E2D\u0E2D\u0E01\u0E04\u0E48\u0E32\u0E43\u0E0A\ + \u0E49\u0E08\u0E48\u0E32\u0E22\u0E40\u0E2D\u0E07 \u0E15\u0E31\u0E49\u0E07\u0E43\ + \u0E08\u0E41\u0E19\u0E48\u0E27\u0E41\u0E19\u0E48\u0E27\u0E48\u0E32\u0E08\u0E30\ + \u0E02\u0E31\u0E1A\u0E44\u0E25\u0E48\u0E28\u0E31\u0E15\u0E23\u0E39\u0E2D\u0E2D\ + \u0E01\u0E08\u0E32\u0E01\u0E1E\u0E23\u0E21\u0E41\u0E14\u0E19 \u0E41\u0E25\u0E30\ + \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E23\u0E32\u0E40\u0E02\u0E49\u0E32\u0E43\ + \u0E01\u0E25\u0E49\u0E40\u0E01\u0E32\u0E30\u0E2D\u0E31\u0E19\u0E22\u0E34\u0E48\ + \u0E07\u0E43\u0E2B\u0E0D\u0E48 \u0E40\u0E23\u0E32\u0E40\u0E02\u0E49\u0E32\u0E43\ + \u0E08\u0E27\u0E48\u0E32\u0E19\u0E31\u0E01\u0E23\u0E1A\u0E02\u0E2D\u0E07\u0E1E\ + \u0E27\u0E01\u0E40\u0E02\u0E32\u0E08\u0E33\u0E19\u0E27\u0E19\u0E2B\u0E19\u0E36\ + \u0E48\u0E07\u0E2D\u0E2D\u0E01\u0E44\u0E1B\u0E15\u0E48\u0E2D\u0E2A\u0E39\u0E49\ + \u0E01\u0E31\u0E1A\u0E0A\u0E32\u0E22\u0E41\u0E14\u0E19\u0E02\u0E2D\u0E07\u0E40\ + \u0E23\u0E32 \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E2A\u0E34\u0E48\u0E07\u0E19\u0E35\ + \u0E49\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19 \u0E40\u0E23\u0E32\u0E01\ + \u0E25\u0E31\u0E1A\u0E21\u0E32\u0E1E\u0E23\u0E49\u0E2D\u0E21\u0E01\u0E31\u0E1A\ + \u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E41\u0E25\u0E30\u0E15\u0E48\u0E2D\u0E2A\ + \u0E39\u0E49\u0E01\u0E31\u0E1A\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E17\u0E35\ + \u0E48 Munfey Hill \u0E0B\u0E36\u0E48\u0E07\u0E40\u0E23\u0E32\u0E2A\u0E39\u0E0D\ + \u0E40\u0E2A\u0E35\u0E22\u0E04\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E44\ + \u0E1B\u0E1A\u0E32\u0E07\u0E2A\u0E48\u0E27\u0E19 & \u0E06\u0E48\u0E32\u0E19\ + \u0E31\u0E01\u0E23\u0E1A\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\ + \u0E44\u0E1B\u0E1A\u0E32\u0E07\u0E2A\u0E48\u0E27\u0E19 \u0E41\u0E25\u0E30\u0E14\ + \u0E49\u0E27\u0E22\u0E40\u0E2B\u0E15\u0E38\u0E19\u0E35\u0E49\u0E08\u0E36\u0E07\ + \u0E0A\u0E48\u0E27\u0E22 Frontiers \u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E08\ + \u0E32\u0E01\u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E23\u0E32\u0E27\u0E19\u0E35\ + \u0E49\u0E43\u0E19 Expedition \u0E2D\u0E37\u0E48\u0E19 \u0E41\u0E15\u0E48\u0E44\ + \u0E21\u0E48\u0E17\u0E31\u0E19\u0E44\u0E23\u0E40\u0E23\u0E32\u0E01\u0E47\u0E17\ + \u0E33\u0E25\u0E32\u0E22\u0E40\u0E2A\u0E1A\u0E35\u0E22\u0E07\u0E2D\u0E32\u0E2B\ + \u0E32\u0E23\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E1A\u0E19\ + \u0E40\u0E01\u0E32\u0E30\u0E43\u0E2B\u0E0D\u0E48\u0E41\u0E25\u0E30\u0E17\u0E33\ + \u0E25\u0E32\u0E22\u0E01\u0E32\u0E23\u0E04\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E1E\ + \u0E27\u0E01\u0E40\u0E02\u0E32\u0E01\u0E31\u0E1A\u0E04\u0E19\u0E14\u0E35\u0E17\ + \u0E35\u0E48\u0E40\u0E1A\u0E18\u0E40\u0E25\u0E40\u0E2E\u0E21 \u0E41\u0E15\u0E48\ + \u0E0A\u0E32\u0E27\u0E2D\u0E34\u0E19\u0E40\u0E14\u0E35\u0E22\u0E19\u0E41\u0E14\ + \u0E07\u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E35\u0E49\u0E0B\u0E36\u0E48\u0E07\ + \u0E16\u0E39\u0E01\u0E2A\u0E07\u0E2A\u0E31\u0E22\u0E27\u0E48\u0E32\u0E40\u0E1B\ + \u0E47\u0E19\u0E04\u0E19\u0E06\u0E48\u0E32\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E19\ + \u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E43\u0E19\u0E40\u0E17\u0E28\u0E21\u0E13\ + \u0E11\u0E25\u0E19\u0E2D\u0E23\u0E4C\u0E17\u0E41\u0E18\u0E21\u0E1B\u0E4C\u0E15\ + \u0E31\u0E19 \u0E16\u0E39\u0E01\u0E2D\u0E34\u0E17\u0E18\u0E34\u0E1E\u0E25\u0E02\ + \u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E04\u0E27\u0E01\u0E40\u0E01\u0E2D\u0E23\ + \u0E4C\u0E1A\u0E32\u0E07\u0E04\u0E19\u0E22\u0E36\u0E14\u0E04\u0E23\u0E2D\u0E07\ + \ \u0E20\u0E32\u0E22\u0E43\u0E15\u0E49\u0E01\u0E32\u0E23\u0E04\u0E38\u0E49\u0E21\ + \u0E04\u0E23\u0E2D\u0E07\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\ + \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E04\u0E31\u0E14\u0E01\u0E23\u0E2D\u0E07\u0E1E\ + \u0E27\u0E01\u0E40\u0E02\u0E32\u0E08\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E41\ + \u0E04\u0E49\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E19\u0E41\ + \u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\ + \u0E4C\u0E02\u0E2D\u0E07\u0E1C\u0E39\u0E49\u0E16\u0E39\u0E01\u0E06\u0E32\u0E15\ + \u0E01\u0E23\u0E23\u0E21 & \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E2A\u0E19\u0E31\ + \u0E1A\u0E2A\u0E19\u0E38\u0E19\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E15\u0E25\ + \u0E2D\u0E14\u0E24\u0E14\u0E39\u0E2B\u0E19\u0E32\u0E27\u201D \u2014"\u0E04\ + \u0E33\u0E02\u0E2D\u0E42\u0E17\u0E29\u0E02\u0E2D\u0E07\u0E40\u0E14\u0E47\u0E01\ + \u0E0A\u0E32\u0E22\u0E41\u0E1E\u0E01\u0E0B\u0E4C\u0E15\u0E31\u0E19" (\u0E08\ + \u0E38\u0E25\u0E2A\u0E32\u0E23), 2307 (\u0E2B\u0E21\u0E32\u0E22\u0E40\u0E2B\u0E15\ + \u0E38: "\u0E04\u0E33\u0E02\u0E2D\u0E42\u0E17\u0E29" \u0E43\u0E19\ + \ \u0E04\u0E27\u0E23\u0E2D\u0E48\u0E32\u0E19\u0E1A\u0E23\u0E34\u0E1A\u0E17\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E32\u0E23\u0E2D\u0E18\u0E34\u0E1A\ + \u0E32\u0E22 \u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E01\u0E32\u0E23\u0E22\u0E2D\ + \u0E21\u0E23\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E2B\u0E23\ + \u0E37\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E40\u0E2A\u0E35\u0E22\u0E43\u0E08) \u0E04\ + \u0E27\u0E32\u0E21\u0E23\u0E39\u0E49\u0E2A\u0E36\u0E01\u0E17\u0E35\u0E48\u0E41\ + \u0E2A\u0E14\u0E07\u0E43\u0E19\u0E04\u0E33\u0E2D\u0E18\u0E34\u0E1A\u0E32\u0E22\ + \u0E02\u0E49\u0E32\u0E07\u0E15\u0E49\u0E19\u0E2A\u0E30\u0E17\u0E49\u0E2D\u0E19\ + \u0E16\u0E36\u0E07\u0E04\u0E27\u0E32\u0E21\u0E15\u0E36\u0E07\u0E40\u0E04\u0E23\ + \u0E35\u0E22\u0E14\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\ + \u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E15\u0E48\u0E2D\u0E40\u0E19\u0E37\u0E48\ + \u0E2D\u0E07\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E22\u0E38\u0E04\u0E2D\u0E32\ + \u0E13\u0E32\u0E19\u0E34\u0E04\u0E21\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E27\ + \u0E31\u0E15\u0E34\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C\u0E2D\u0E40\u0E21\u0E23\ + \u0E34\u0E01\u0E32?" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E41\u0E01\u0E49\u0E44\u0E02\u0E2A\u0E34\u0E17\u0E18\ + \u0E34\u0E17\u0E35\u0E48\u0E40\u0E17\u0E48\u0E32\u0E40\u0E17\u0E35\u0E22\u0E21\ + \u0E01\u0E31\u0E19" + B: "\u0E01\u0E32\u0E23\u0E25\u0E07\u0E04\u0E30\u0E41\u0E19\u0E19\u0E40\u0E2A\ + \u0E35\u0E22\u0E07\u0E41\u0E1A\u0E1A\u0E2A\u0E32\u0E01\u0E25" + C: "\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10" + D: "\u0E02\u0E49\u0E2D\u0E2B\u0E49\u0E32\u0E21" + input_correct_responses: + - B + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 "\u0E43\u0E19\u0E1B\u0E23\ + \u0E30\u0E21\u0E27\u0E25\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E43\u0E2B\u0E21\ + \u0E48\u0E0B\u0E36\u0E48\u0E07\u0E09\u0E31\u0E19\u0E04\u0E34\u0E14\u0E27\u0E48\ + \u0E32\u0E08\u0E33\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\ + \u0E04\u0E38\u0E13\u0E17\u0E35\u0E48\u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E17\ + \u0E33\u0E43\u0E2B\u0E49 \u0E09\u0E31\u0E19\u0E1B\u0E23\u0E32\u0E23\u0E16\u0E19\ + \u0E32\u0E43\u0E2B\u0E49\u0E04\u0E38\u0E13\u0E08\u0E33\u0E2A\u0E38\u0E20\u0E32\ + \u0E1E\u0E2A\u0E15\u0E23\u0E35\u0E41\u0E25\u0E30\u0E21\u0E35\u0E19\u0E49\u0E33\ + \u0E43\u0E08\u0E41\u0E25\u0E30\u0E40\u0E2D\u0E37\u0E49\u0E2D\u0E40\u0E1F\u0E37\ + \u0E49\u0E2D\u0E40\u0E1C\u0E37\u0E48\u0E2D\u0E41\u0E1C\u0E48\u0E15\u0E48\u0E2D\ + \u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32\ + \u0E1A\u0E23\u0E23\u0E1E\u0E1A\u0E38\u0E23\u0E38\u0E29\u0E02\u0E2D\u0E07\u0E04\ + \u0E38\u0E13 \u0E2D\u0E22\u0E48\u0E32\u0E21\u0E2D\u0E1A\u0E2D\u0E33\u0E19\u0E32\ + \u0E08\u0E17\u0E35\u0E48\u0E44\u0E23\u0E49\u0E02\u0E2D\u0E1A\u0E40\u0E02\u0E15\ + \u0E40\u0E0A\u0E48\u0E19\u0E19\u0E35\u0E49\u0E44\u0E27\u0E49\u0E43\u0E19\u0E21\ + \u0E37\u0E2D\u0E02\u0E2D\u0E07\u0E2A\u0E32\u0E21\u0E35 \u0E08\u0E33\u0E44\u0E27\ + \u0E49\u0E27\u0E48\u0E32\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22\u0E17\u0E38\u0E01\ + \u0E04\u0E19\u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E23\u0E23\u0E32\u0E0A\ + \u0E16\u0E49\u0E32\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E17\u0E33\u0E44\u0E14\ + \u0E49 \u0E2B\u0E32\u0E01\u0E44\u0E21\u0E48\u0E14\u0E39\u0E41\u0E25\u0E41\u0E25\ + \u0E30\u0E40\u0E2D\u0E32\u0E43\u0E08\u0E43\u0E2A\u0E48\u0E40\u0E1B\u0E47\u0E19\ + \u0E1E\u0E34\u0E40\u0E28\u0E29\u0E01\u0E31\u0E1A\u0E1C\u0E39\u0E49\u0E2B\u0E0D\ + \u0E34\u0E07 \u0E40\u0E23\u0E32\u0E15\u0E31\u0E49\u0E07\u0E43\u0E08\u0E41\u0E19\ + \u0E48\u0E27\u0E41\u0E19\u0E48\u0E17\u0E35\u0E48\u0E08\u0E30\u0E1B\u0E25\u0E38\ + \u0E01\u0E1B\u0E31\u0E48\u0E19\u0E43\u0E2B\u0E49\u0E40\u0E01\u0E34\u0E14\u0E01\ + \u0E32\u0E23\u0E01\u0E1A\u0E0F \u0E41\u0E25\u0E30\u0E08\u0E30\u0E44\u0E21\u0E48\ + \u0E1C\u0E39\u0E01\u0E21\u0E31\u0E14\u0E15\u0E31\u0E27\u0E40\u0E2D\u0E07\u0E14\ + \u0E49\u0E27\u0E22\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E43\u0E14 \u0E46 \u0E17\ + \u0E35\u0E48\u0E40\u0E23\u0E32\u0E44\u0E21\u0E48\u0E21\u0E35\u0E2A\u0E34\u0E17\ + \u0E18\u0E34\u0E4C\u0E21\u0E35\u0E40\u0E2A\u0E35\u0E22\u0E07 Abigail Adams \u0E43\ + \u0E19\u0E08\u0E14\u0E2B\u0E21\u0E32\u0E22\u0E16\u0E36\u0E07 John Adams, 1776\ + \ "\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E1E\u0E34\u0E40\u0E28\u0E29\u0E2A\ + \u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E17\ + \u0E33\u0E43\u0E2B\u0E49\u0E40\u0E23\u0E32\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\ + \u0E2A\u0E16\u0E32\u0E19\u0E30\u0E17\u0E35\u0E48\u0E1C\u0E34\u0E14\u0E1B\u0E01\ + \u0E15\u0E34\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14 \u0E1C\u0E39\ + \u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E25\u0E07\u0E17\u0E38\u0E19\u0E01\u0E31\u0E1A\ + \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E02\u0E2D\u0E07\u0E1E\u0E25\u0E40\u0E21\u0E37\ + \u0E2D\u0E07\u0E43\u0E19\u0E2B\u0E21\u0E27\u0E14\u0E2B\u0E19\u0E36\u0E48\u0E07\ + \ - \u0E1C\u0E39\u0E49\u0E21\u0E35\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E40\u0E25\ + \u0E37\u0E2D\u0E01\u0E15\u0E31\u0E49\u0E07 \u0E04\u0E13\u0E30\u0E25\u0E39\u0E01\ + \u0E02\u0E38\u0E19 \u0E1C\u0E39\u0E49\u0E16\u0E37\u0E2D\u0E15\u0E33\u0E41\u0E2B\ + \u0E19\u0E48\u0E07 - \u0E02\u0E49\u0E32\u0E21\u0E40\u0E2A\u0E49\u0E19\u0E08\u0E34\ + \u0E19\u0E15\u0E20\u0E32\u0E1E\u0E04\u0E37\u0E2D \u0E43\u0E19\u0E40\u0E23\u0E37\ + \u0E48\u0E2D\u0E07\u0E15\u0E48\u0E2D\u0E44\u0E1B \u0E43\u0E19\u0E1A\u0E32\u0E07\ + \u0E23\u0E31\u0E10 \u0E2A\u0E15\u0E23\u0E35\u0E17\u0E35\u0E48\u0E41\u0E15\u0E48\ + \u0E07\u0E07\u0E32\u0E19\u0E41\u0E25\u0E49\u0E27\u0E2D\u0E32\u0E08\u0E16\u0E37\ + \u0E2D\u0E04\u0E23\u0E2D\u0E07\u0E17\u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E2A\u0E34\ + \u0E19\u0E41\u0E25\u0E30\u0E17\u0E33\u0E18\u0E38\u0E23\u0E01\u0E23\u0E23\u0E21\ + \u0E43\u0E19\u0E19\u0E32\u0E21\u0E02\u0E2D\u0E07\u0E15\u0E19\u0E40\u0E2D\u0E07\ + \ \u0E43\u0E19\u0E1A\u0E32\u0E07\u0E23\u0E31\u0E10 \u0E23\u0E32\u0E22\u0E44\u0E14\ + \u0E49\u0E02\u0E2D\u0E07\u0E40\u0E18\u0E2D\u0E40\u0E1B\u0E47\u0E19\u0E02\u0E2D\ + \u0E07\u0E2A\u0E32\u0E21\u0E35 \u0E43\u0E19\u0E1A\u0E32\u0E07\u0E23\u0E31\u0E10\ + \ \u0E2A\u0E15\u0E23\u0E35\u0E2D\u0E32\u0E08\u0E43\u0E2B\u0E49\u0E01\u0E32\u0E23\ + \u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E22\u0E32\u0E19\u0E1B\u0E23\u0E31\u0E01\u0E1B\ + \u0E23\u0E33\u0E2A\u0E32\u0E21\u0E35 \u0E1F\u0E49\u0E2D\u0E07\u0E23\u0E49\u0E2D\ + \u0E07 \u0E41\u0E25\u0E30\u0E16\u0E39\u0E01\u0E1F\u0E49\u0E2D\u0E07\u0E43\u0E19\ + \ \u0E43\u0E19\u0E0A\u0E31\u0E49\u0E19\u0E28\u0E32\u0E25 \u0E2D\u0E37\u0E48\u0E19\ + \ \u0E46 \u0E40\u0E18\u0E2D\u0E44\u0E21\u0E48\u0E21\u0E35\u0E04\u0E48\u0E32\u0E0A\ + \u0E14\u0E40\u0E0A\u0E22\u0E43\u0E19\u0E01\u0E23\u0E13\u0E35\u0E40\u0E2A\u0E35\ + \u0E22\u0E2B\u0E32\u0E22\u0E15\u0E48\u0E2D\u0E1A\u0E38\u0E04\u0E04\u0E25 \u0E17\ + \u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E2A\u0E34\u0E19 \u0E2B\u0E23\u0E37\u0E2D\u0E25\ + \u0E31\u0E01\u0E29\u0E13\u0E30. \u0E43\u0E19\u0E01\u0E23\u0E13\u0E35\u0E2B\u0E22\ + \u0E48\u0E32\u0E40\u0E1E\u0E23\u0E32\u0E30\u0E2A\u0E32\u0E21\u0E35\u0E40\u0E1B\ + \u0E47\u0E19\u0E0A\u0E39\u0E49, \u0E20\u0E23\u0E23\u0E22\u0E32\u0E1C\u0E39\u0E49\ + \u0E1A\u0E23\u0E34\u0E2A\u0E38\u0E17\u0E18\u0E34\u0E4C\u0E44\u0E21\u0E48\u0E21\ + \u0E35\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C\u0E43\u0E19\u0E1A\u0E38\u0E15\u0E23\ + \u0E2B\u0E23\u0E37\u0E2D\u0E17\u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E2A\u0E34\u0E19\ + , \u0E40\u0E27\u0E49\u0E19\u0E41\u0E15\u0E48\u0E42\u0E14\u0E22 \u0E04\u0E33\u0E2A\ + \u0E31\u0E48\u0E07\u0E1E\u0E34\u0E40\u0E28\u0E29\u0E02\u0E2D\u0E07\u0E28\u0E32\ + \u0E25 \u0E41\u0E15\u0E48\u0E44\u0E21\u0E48\u0E21\u0E35\u0E43\u0E19\u0E23\u0E31\ + \u0E10\u0E43\u0E14\u0E02\u0E2D\u0E07\u0E2A\u0E2B\u0E20\u0E32\u0E1E \u0E20\u0E23\ + \u0E23\u0E22\u0E32\u0E21\u0E35\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E43\u0E19\u0E15\ + \u0E31\u0E27\u0E40\u0E18\u0E2D\u0E40\u0E2D\u0E07 \u0E2B\u0E23\u0E37\u0E2D\u0E2A\ + \u0E48\u0E27\u0E19\u0E43\u0E14\u0E2A\u0E48\u0E27\u0E19\u0E2B\u0E19\u0E36\u0E48\ + \u0E07\u0E02\u0E2D\u0E07\u0E23\u0E32\u0E22\u0E44\u0E14\u0E49\u0E23\u0E48\u0E27\ + \u0E21\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E47\u0E19\u0E2B\u0E38\ + \u0E49\u0E19\u0E2A\u0E48\u0E27\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\ + \u0E0A\u0E35\u0E27\u0E34\u0E15\u0E02\u0E2D\u0E07\u0E2A\u0E32\u0E21\u0E35 \u0E43\ + \u0E19\u0E1A\u0E32\u0E07\u0E23\u0E31\u0E10 \u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\ + \u0E07\u0E2D\u0E32\u0E08\u0E40\u0E02\u0E49\u0E32\u0E2A\u0E39\u0E48 \u0E42\u0E23\ + \u0E07\u0E40\u0E23\u0E35\u0E22\u0E19\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E41\ + \u0E25\u0E30\u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E43\ + \u0E19\u0E28\u0E32\u0E25 \u0E17\u0E35\u0E48\u0E2D\u0E37\u0E48\u0E19 \u0E46 \u0E1E\ + \u0E27\u0E01\u0E40\u0E02\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E34\u0E48\u0E07\ + \u0E15\u0E49\u0E2D\u0E07\u0E2B\u0E49\u0E32\u0E21 \u0E43\u0E19\u0E21\u0E2B\u0E32\ + \u0E27\u0E34\u0E17\u0E22\u0E32\u0E25\u0E31\u0E22\u0E1A\u0E32\u0E07\u0E41\u0E2B\ + \u0E48\u0E07 \u0E40\u0E14\u0E47\u0E01\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\u0E07\ + \u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E44\u0E14\u0E49\u0E40\u0E1B\u0E23\u0E35\ + \u0E22\u0E1A\u0E14\u0E49\u0E32\u0E19\u0E01\u0E32\u0E23\u0E28\u0E36\u0E01\u0E29\ + \u0E32\u0E40\u0E17\u0E48\u0E32\u0E40\u0E17\u0E35\u0E22\u0E21\u0E01\u0E31\u0E1A\ + \u0E40\u0E14\u0E47\u0E01\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22 \u0E43\u0E19\u0E02\ + \u0E13\u0E30\u0E17\u0E35\u0E48\u0E2A\u0E16\u0E32\u0E1A\u0E31\u0E19\u0E17\u0E35\ + \u0E48\u0E19\u0E48\u0E32\u0E20\u0E32\u0E04\u0E20\u0E39\u0E21\u0E34\u0E43\u0E08\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E2B\u0E25\u0E32\u0E22\u0E41\u0E2B\u0E48\ + \u0E07\u0E43\u0E19\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E1B\u0E0F\u0E34\u0E40\ + \u0E2A\u0E18\u0E44\u0E21\u0E48\u0E43\u0E2B\u0E49\u0E23\u0E31\u0E1A\u0E40\u0E02\ + \u0E49\u0E32 \u0E41\u0E21\u0E49\u0E27\u0E48\u0E32\u0E1A\u0E38\u0E15\u0E23\u0E0A\ + \u0E32\u0E22\u0E02\u0E2D\u0E07\u0E08\u0E35\u0E19 \u0E0D\u0E35\u0E48\u0E1B\u0E38\ + \u0E48\u0E19 \u0E41\u0E25\u0E30\u0E41\u0E2D\u0E1F\u0E23\u0E34\u0E01\u0E32\u0E08\ + \u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E15\u0E49\u0E2D\ + \u0E19\u0E23\u0E31\u0E1A\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E41\u0E15\u0E48\u0E2A\ + \u0E34\u0E17\u0E18\u0E34\u0E1E\u0E34\u0E40\u0E28\u0E29\u0E17\u0E35\u0E48\u0E44\ + \u0E14\u0E49\u0E23\u0E31\u0E1A\u0E41\u0E25\u0E49\u0E27\u0E43\u0E19\u0E2B\u0E25\ + \u0E32\u0E22\u0E23\u0E31\u0E10\u0E19\u0E31\u0E49\u0E19\u0E44\u0E21\u0E48\u0E1B\ + \u0E25\u0E2D\u0E14\u0E20\u0E31\u0E22\u0E40\u0E25\u0E22" \u0E0B\u0E39\u0E0B\ + \u0E32\u0E19 \u0E1A\u0E35. \u0E41\u0E2D\u0E19\u0E42\u0E18\u0E19\u0E35, "\u0E04\ + \u0E33\u0E1B\u0E23\u0E30\u0E01\u0E32\u0E28\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E2A\ + \u0E15\u0E23\u0E35" 4 \u0E01\u0E23\u0E01\u0E0E\u0E32\u0E04\u0E21 \u0E1E\ + .\u0E28. 2419 \u0E04\u0E27\u0E32\u0E21\u0E23\u0E39\u0E49\u0E2A\u0E36\u0E01\u0E17\ + \u0E35\u0E48\u0E41\u0E2A\u0E14\u0E07\u0E2D\u0E2D\u0E01\u0E21\u0E32\u0E43\u0E19\ + \u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E17\u0E35\u0E48\u0E15\u0E31\u0E14\ + \u0E15\u0E2D\u0E19\u0E21\u0E32\u0E04\u0E23\u0E31\u0E49\u0E07\u0E17\u0E35\u0E48\ + \u0E2A\u0E2D\u0E07\u0E42\u0E14\u0E22\u0E0B\u0E39\u0E0B\u0E32\u0E19 \u0E1A\u0E35\ + . \u0E41\u0E2D\u0E19\u0E42\u0E18\u0E19\u0E35\u0E21\u0E35\u0E41\u0E19\u0E27\u0E42\ + \u0E19\u0E49\u0E21\u0E17\u0E35\u0E48\u0E08\u0E30\u0E2A\u0E19\u0E31\u0E1A\u0E2A\ + \u0E19\u0E38\u0E19\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14 \u0E02\ + \u0E2D\u0E07" + - input_choice_list: + A: "\u0E0A\u0E32\u0E27\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E31\u0E19\u0E15\ + \u0E49\u0E2D\u0E07\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1E\u0E39\u0E19\u0E04\u0E27\ + \u0E32\u0E21\u0E44\u0E14\u0E49\u0E40\u0E1B\u0E23\u0E35\u0E22\u0E1A\u0E17\u0E32\ + \u0E07\u0E40\u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35\u0E43\u0E19\u0E40\ + \u0E27\u0E35\u0E22\u0E14\u0E19\u0E32\u0E21" + B: "\u0E01\u0E32\u0E23\u0E17\u0E34\u0E49\u0E07\u0E23\u0E30\u0E40\u0E1A\u0E34\ + \u0E14\u0E02\u0E2D\u0E07\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\ + \u0E40\u0E27\u0E35\u0E22\u0E14\u0E19\u0E32\u0E21\u0E40\u0E1B\u0E47\u0E19\u0E02\ + \u0E31\u0E49\u0E19\u0E15\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E19\u0E33\u0E44\u0E1B\ + \u0E2A\u0E39\u0E48\u0E04\u0E27\u0E32\u0E21\u0E04\u0E37\u0E1A\u0E2B\u0E19\u0E49\ + \u0E32\u0E43\u0E19\u0E2A\u0E07\u0E04\u0E23\u0E32\u0E21" + C: "\u0E01\u0E32\u0E23\u0E17\u0E34\u0E49\u0E07\u0E23\u0E30\u0E40\u0E1A\u0E34\ + \u0E14\u0E02\u0E2D\u0E07\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\ + \u0E40\u0E27\u0E35\u0E22\u0E14\u0E19\u0E32\u0E21\u0E25\u0E49\u0E21\u0E40\u0E2B\ + \u0E25\u0E27" + D: "\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E08\u0E30\u0E15\u0E49\u0E2D\ + \u0E07\u0E44\u0E21\u0E48\u0E22\u0E2D\u0E21\u0E41\u0E1E\u0E49\u0E15\u0E48\u0E2D\ + \u0E04\u0E27\u0E32\u0E21\u0E1E\u0E48\u0E32\u0E22\u0E41\u0E1E\u0E49\u0E15\u0E48\ + \u0E2D\u0E2A\u0E07\u0E04\u0E23\u0E32\u0E21\u0E43\u0E19\u0E40\u0E27\u0E35\u0E22\ + \u0E14\u0E19\u0E32\u0E21" + input_correct_responses: + - C + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E1C\u0E39\u0E49\u0E19\u0E33\ + \u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E1E\u0E39\u0E14\u0E16\u0E36\u0E07\u0E01\ + \u0E32\u0E23\u0E2B\u0E22\u0E38\u0E14\u0E01\u0E32\u0E23\u0E23\u0E38\u0E01\u0E23\ + \u0E32\u0E19\u0E08\u0E32\u0E01\u0E17\u0E32\u0E07\u0E40\u0E2B\u0E19\u0E37\u0E2D\ + \ \u0E41\u0E15\u0E48\u0E19\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E32\u0E23\ + \u0E15\u0E48\u0E2D\u0E2A\u0E39\u0E49\u0E02\u0E2D\u0E07\u0E01\u0E25\u0E38\u0E48\ + \u0E21\u0E0A\u0E32\u0E27\u0E40\u0E27\u0E35\u0E22\u0E14\u0E19\u0E32\u0E21\u0E08\ + \u0E19\u0E01\u0E23\u0E30\u0E17\u0E31\u0E48\u0E07\u0E40\u0E23\u0E32\u0E40\u0E02\ + \u0E49\u0E32\u0E41\u0E17\u0E23\u0E01\u0E41\u0E0B\u0E07 \u0E14\u0E39\u0E40\u0E2B\ + \u0E21\u0E37\u0E2D\u0E19\u0E27\u0E48\u0E32\u0E40\u0E23\u0E32\u0E08\u0E30\u0E1E\ + \u0E22\u0E32\u0E22\u0E32\u0E21\u0E0A\u0E48\u0E27\u0E22\u0E0A\u0E32\u0E27\u0E40\ + \u0E27\u0E35\u0E22\u0E14\u0E19\u0E32\u0E21\u0E08\u0E32\u0E01\u0E42\u0E2E\u0E08\ + \u0E34\u0E21\u0E34\u0E19\u0E2B\u0E4C\u0E41\u0E21\u0E49\u0E27\u0E48\u0E32\u0E40\ + \u0E23\u0E32\u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E06\u0E48\u0E32\u0E1E\u0E27\ + \u0E01\u0E40\u0E02\u0E32\u0E41\u0E25\u0E30\u0E17\u0E33\u0E25\u0E32\u0E22\u0E1B\ + \u0E23\u0E30\u0E40\u0E17\u0E28\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\ + \u0E32\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E17\u0E33\u0E40\u0E0A\u0E48\u0E19\u0E19\ + \u0E31\u0E49\u0E19 \u0E02\u0E13\u0E30\u0E17\u0E35\u0E48\u0E0A\u0E32\u0E27\u0E1E\ + \u0E37\u0E49\u0E19\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E2A\u0E33\u0E23\u0E27\u0E08\ + \u0E2B\u0E21\u0E39\u0E48\u0E1A\u0E49\u0E32\u0E19\u0E17\u0E35\u0E48\u0E16\u0E39\ + \u0E01\u0E17\u0E34\u0E49\u0E07\u0E23\u0E30\u0E40\u0E1A\u0E34\u0E14 \u0E1C\u0E39\ + \u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E41\u0E25\u0E30\u0E40\u0E14\u0E47\u0E01\u0E16\ + \u0E39\u0E01\u0E44\u0E1F\u0E40\u0E1C\u0E32\u0E17\u0E33\u0E25\u0E32\u0E22 \u0E15\ + \u0E49\u0E19\u0E02\u0E49\u0E32\u0E27\u0E16\u0E39\u0E01\u0E17\u0E33\u0E25\u0E32\ + \u0E22 \u0E41\u0E25\u0E30\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E15\u0E48\u0E32\u0E07\ + \u0E46 \u0E40\u0E15\u0E47\u0E21\u0E44\u0E1B\u0E14\u0E49\u0E27\u0E22\u0E40\u0E08\ + \u0E49\u0E32\u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\u0E17\u0E2B\u0E32\u0E23\ + \u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32 \u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E01\ + \u0E33\u0E25\u0E31\u0E07\u0E1E\u0E39\u0E14\u0E16\u0E36\u0E07\u0E01\u0E2D\u0E07\ + \u0E42\u0E08\u0E23\u0E40\u0E27\u0E35\u0E22\u0E14\u0E01\u0E07\u0E41\u0E25\u0E30\ + \u0E01\u0E2D\u0E07\u0E01\u0E33\u0E25\u0E31\u0E07\u0E2D\u0E40\u0E21\u0E23\u0E34\ + \u0E01\u0E31\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E25\u0E31\u0E1A\u0E46 \u0E27\ + \u0E48\u0E32 "\u0E42\u0E23\u0E04\u0E23\u0E30\u0E1A\u0E32\u0E14\u0E43\u0E19\ + \u0E1A\u0E49\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13\u0E17\u0E31\u0E49\ + \u0E07\u0E04\u0E39\u0E48 " \u2026 \u0E2B\u0E22\u0E38\u0E14\u0E01\u0E32\u0E23\ + \u0E17\u0E34\u0E49\u0E07\u0E23\u0E30\u0E40\u0E1A\u0E34\u0E14\u0E17\u0E32\u0E07\ + \u0E17\u0E34\u0E28\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E41\u0E25\u0E30\u0E17\u0E34\ + \u0E28\u0E43\u0E15\u0E49 \u0E22\u0E38\u0E15\u0E34\u0E01\u0E32\u0E23\u0E04\u0E49\ + \u0E19\u0E2B\u0E32\u0E41\u0E25\u0E30\u0E17\u0E33\u0E25\u0E32\u0E22\u0E01\u0E32\ + \u0E23\u0E01\u0E27\u0E32\u0E14\u0E25\u0E49\u0E32\u0E07\u0E17\u0E35\u0E48\u0E19\ + \u0E48\u0E32\u0E23\u0E31\u0E07\u0E40\u0E01\u0E35\u0E22\u0E08 \u0E41\u0E25\u0E30\ + \u0E08\u0E33\u0E01\u0E31\u0E14\u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E01\ + \u0E32\u0E23\u0E17\u0E32\u0E07\u0E17\u0E2B\u0E32\u0E23\u0E02\u0E2D\u0E07\u0E40\ + \u0E23\u0E32\u0E43\u0E2B\u0E49\u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E01\ + \u0E32\u0E23\u0E1A\u0E19\u0E20\u0E32\u0E04\u0E1E\u0E37\u0E49\u0E19\u0E14\u0E34\ + \u0E19\u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19 \u0E01\u0E32\u0E23\u0E17\ + \u0E34\u0E49\u0E07\u0E23\u0E30\u0E40\u0E1A\u0E34\u0E14\u0E17\u0E32\u0E07\u0E15\ + \u0E2D\u0E19\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E44\u0E21\u0E48\u0E2A\u0E32\u0E21\ + \u0E32\u0E23\u0E16\u0E2B\u0E22\u0E38\u0E14\u0E2B\u0E23\u0E37\u0E2D\u0E15\u0E23\ + \u0E27\u0E08\u0E2A\u0E2D\u0E1A\u0E01\u0E32\u0E23\u0E44\u0E2B\u0E25\u0E40\u0E27\ + \u0E35\u0E22\u0E19\u0E02\u0E2D\u0E07\u0E17\u0E2B\u0E32\u0E23\u0E17\u0E32\u0E07\ + \u0E15\u0E2D\u0E19\u0E43\u0E15\u0E49\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E08\u0E23\ + \u0E34\u0E07\u0E08\u0E31\u0E07 \u0E41\u0E25\u0E30\u0E43\u0E19\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07 \u0E2D\u0E32\u0E08\u0E01\ + \u0E23\u0E30\u0E15\u0E38\u0E49\u0E19\u0E43\u0E2B\u0E49\u0E2E\u0E32\u0E19\u0E2D\ + \u0E22\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E22\u0E32\u0E22\u0E32\u0E21\ + \u0E17\u0E33\u0E2A\u0E07\u0E04\u0E23\u0E32\u0E21\u0E21\u0E32\u0E01\u0E02\u0E36\ + \u0E49\u0E19 \u2014Senator George McGovern, "The Lessons of Vietnam,"\ + \ 25 \u0E40\u0E21\u0E29\u0E32\u0E22\u0E19 2510 \u0E04\u0E27\u0E32\u0E21\u0E04\ + \u0E34\u0E14\u0E40\u0E2B\u0E47\u0E19\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\ + \u0E19\u0E35\u0E49\u0E08\u0E32\u0E01\u0E17\u0E28\u0E27\u0E23\u0E23\u0E29 1960\ + \ \u0E17\u0E35\u0E48\u0E2A\u0E30\u0E17\u0E49\u0E2D\u0E19\u0E21\u0E38\u0E21\u0E21\ + \u0E2D\u0E07\u0E02\u0E2D\u0E07\u0E2A\u0E38\u0E19\u0E17\u0E23\u0E1E\u0E08\u0E19\ + \u0E4C\u0E02\u0E2D\u0E07 George McGovern \u0E44\u0E14\u0E49\u0E42\u0E14\u0E22\ + \u0E15\u0E23\u0E07\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" + - input_choice_list: + A: "\u0E2D\u0E1A\u0E34\u0E40\u0E01\u0E25 \u0E2D\u0E14\u0E31\u0E21\u0E2A\u0E4C" + B: "\u0E04\u0E25\u0E32\u0E23\u0E32 \u0E1A\u0E32\u0E23\u0E4C\u0E15\u0E31\u0E19" + C: "\u0E40\u0E0A\u0E2D\u0E23\u0E4C\u0E25\u0E35\u0E48\u0E22\u0E4C \u0E40\u0E17\ + \u0E21\u0E40\u0E1E\u0E34\u0E25" + D: "\u0E2E\u0E34\u0E25\u0E25\u0E32\u0E23\u0E35 \u0E04\u0E25\u0E34\u0E19\u0E15\ + \u0E31\u0E19" + input_correct_responses: + - B + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E02\u0E49\u0E32\u0E1E\u0E40\ + \u0E08\u0E49\u0E32\u0E44\u0E21\u0E48\u0E40\u0E23\u0E35\u0E22\u0E01\u0E23\u0E49\ + \u0E2D\u0E07\u0E2B\u0E32\u0E1C\u0E25\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\u0E19\ + \u0E4C\u0E2A\u0E48\u0E27\u0E19\u0E15\u0E31\u0E27\u0E2B\u0E23\u0E37\u0E2D\u0E2B\ + \u0E32\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\u0E19\u0E4C\u0E2A\u0E48\u0E27\u0E19\ + \u0E15\u0E19 \u0E09\u0E31\u0E19\u0E1B\u0E23\u0E32\u0E01\u0E0F\u0E15\u0E31\u0E27\ + \u0E43\u0E19\u0E10\u0E32\u0E19\u0E30\u0E1C\u0E39\u0E49\u0E2A\u0E19\u0E31\u0E1A\ + \u0E2A\u0E19\u0E38\u0E19\u0E1C\u0E39\u0E49\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\ + \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\ + \u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E02\u0E2D\u0E07\u0E15\u0E31\u0E27\u0E40\ + \u0E2D\u0E07\u0E44\u0E14\u0E49 \u0E09\u0E31\u0E19\u0E21\u0E32\u0E40\u0E1B\u0E47\ + \u0E19\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E19\u0E02\u0E2D\u0E07\u0E04\u0E19\u0E17\ + \u0E35\u0E48\u0E16\u0E39\u0E01\u0E17\u0E34\u0E49\u0E07\u0E23\u0E49\u0E32\u0E07\ + \ \u0E16\u0E39\u0E01\u0E01\u0E14\u0E02\u0E35\u0E48 \u0E41\u0E25\u0E30\u0E2D\u0E49\ + \u0E32\u0E07\u0E27\u0E49\u0E32\u0E07 \u0E43\u0E19\u0E01\u0E32\u0E23\u0E08\u0E31\ + \u0E14\u0E40\u0E15\u0E23\u0E35\u0E22\u0E21\u0E02\u0E2D\u0E07\u0E1E\u0E23\u0E30\ + \u0E40\u0E08\u0E49\u0E32 \u0E09\u0E31\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E2A\ + \u0E35\u0E22\u0E07\u0E02\u0E2D\u0E07\u0E04\u0E19\u0E1A\u0E49\u0E32\u0E17\u0E35\ + \u0E48\u0E21\u0E35\u0E40\u0E2A\u0E35\u0E22\u0E07\u0E23\u0E49\u0E2D\u0E07\u0E40\ + \u0E2A\u0E35\u0E22\u0E14\u0E41\u0E17\u0E07\u0E08\u0E32\u0E01\u0E04\u0E38\u0E01\ + \u0E43\u0E15\u0E49\u0E14\u0E34\u0E19\u0E2D\u0E31\u0E19\u0E19\u0E48\u0E32\u0E2A\ + \u0E22\u0E14\u0E2A\u0E22\u0E2D\u0E07\u0E43\u0E19\u0E04\u0E38\u0E01\u0E02\u0E2D\ + \u0E07\u0E04\u0E38\u0E13 \u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E40\u0E2A\u0E35\ + \u0E22\u0E07\u0E02\u0E2D\u0E07\u0E2B\u0E2D\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\ + \u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13 \u0E09\u0E31\u0E19\u0E04\u0E37\u0E2D\u0E04\ + \u0E27\u0E32\u0E21\u0E2B\u0E27\u0E31\u0E07\u0E02\u0E2D\u0E07\u0E2A\u0E34\u0E48\ + \u0E07\u0E21\u0E35\u0E0A\u0E35\u0E27\u0E34\u0E15\u0E17\u0E35\u0E48\u0E04\u0E25\ + \u0E31\u0E48\u0E07\u0E44\u0E04\u0E25\u0E49\u0E22\u0E32\u0E01\u0E08\u0E19\u0E17\ + \u0E35\u0E48\u0E2D\u0E32\u0E28\u0E31\u0E22\u0E2D\u0E22\u0E39\u0E48\u0E15\u0E32\ + \u0E21\u0E2B\u0E49\u0E2D\u0E07\u0E02\u0E31\u0E07 \u0E41\u0E1C\u0E07\u0E25\u0E2D\ + \u0E22 \u0E01\u0E23\u0E07 \u0E41\u0E25\u0E30\u0E2B\u0E49\u0E2D\u0E07\u0E23\u0E01\ + \u0E23\u0E49\u0E32\u0E07\u0E43\u0E19\u0E1A\u0E49\u0E32\u0E19\u0E17\u0E35\u0E48\ + \u0E22\u0E32\u0E01\u0E08\u0E19\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13 \u0E09\u0E31\ + \u0E19\u0E04\u0E37\u0E2D\u0E01\u0E32\u0E23\u0E40\u0E1B\u0E34\u0E14\u0E40\u0E1C\ + \u0E22\u0E02\u0E2D\u0E07\u0E2A\u0E34\u0E48\u0E07\u0E21\u0E35\u0E0A\u0E35\u0E27\ + \u0E34\u0E15\u0E17\u0E35\u0E48\u0E23\u0E48\u0E33\u0E44\u0E2B\u0E49\u0E41\u0E25\ + \u0E30\u0E17\u0E19\u0E17\u0E38\u0E01\u0E02\u0E4C\u0E17\u0E23\u0E21\u0E32\u0E19\ + \u0E2B\u0E25\u0E32\u0E22\u0E23\u0E49\u0E2D\u0E22\u0E15\u0E31\u0E27\u0E17\u0E35\ + \u0E48\u0E0B\u0E48\u0E2D\u0E19\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E17\u0E35\ + \u0E48\u0E1E\u0E31\u0E01\u0E2D\u0E32\u0E28\u0E31\u0E22\u0E2A\u0E48\u0E27\u0E19\ + \u0E15\u0E31\u0E27\u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13 \u0E43\u0E19\u0E04\u0E2D\ + \u0E01\u0E41\u0E25\u0E30\u0E01\u0E23\u0E30\u0E17\u0E48\u0E2D\u0E21 - \u0E1B\u0E34\ + \u0E14\u0E15\u0E31\u0E27 \u0E15\u0E31\u0E14\u0E02\u0E32\u0E14\u0E08\u0E32\u0E01\ + \u0E2D\u0E34\u0E17\u0E18\u0E34\u0E1E\u0E25\u0E01\u0E32\u0E23\u0E23\u0E31\u0E01\ + \u0E29\u0E32\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 \u0E08\u0E32\u0E01\u0E01\ + \u0E32\u0E23\u0E14\u0E39\u0E41\u0E25\u0E1F\u0E37\u0E49\u0E19\u0E1F\u0E39\u0E08\ + \u0E34\u0E15\u0E43\u0E08\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 ... \u0E1B\ + \u0E23\u0E30\u0E27\u0E31\u0E15\u0E34\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C\u0E40\ + \u0E28\u0E23\u0E49\u0E32\u0E42\u0E28\u0E01\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\ + \u0E40\u0E02\u0E32\u0E08\u0E30\u0E16\u0E39\u0E01\u0E40\u0E1C\u0E22\u0E41\u0E1E\ + \u0E23\u0E48\u0E15\u0E48\u0E2D\u0E2B\u0E19\u0E49\u0E32\u0E04\u0E38\u0E13 \u0E15\ + \u0E32\u0E21\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E34\u0E14\u0E40\u0E1C\u0E22\u0E15\ + \u0E48\u0E2D\u0E27\u0E34\u0E0D\u0E0D\u0E32\u0E13\u0E17\u0E35\u0E48\u0E42\u0E28\ + \u0E01\u0E40\u0E28\u0E23\u0E49\u0E32\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E43\ + \u0E19\u0E0A\u0E48\u0E27\u0E07\u0E2A\u0E32\u0E21\u0E40\u0E14\u0E37\u0E2D\u0E19\ + \u0E17\u0E35\u0E48\u0E1C\u0E48\u0E32\u0E19\u0E21\u0E32 \u0E04\u0E38\u0E13\u0E08\ + \u0E30\u0E04\u0E49\u0E19\u0E2B\u0E32\u0E27\u0E34\u0E18\u0E35\u0E01\u0E32\u0E23\ + \u0E1A\u0E23\u0E23\u0E40\u0E17\u0E32\u0E17\u0E38\u0E01\u0E02\u0E4C\u0E17\u0E35\ + \u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2D\u0E19\u0E38\ + \u0E21\u0E31\u0E15\u0E34\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E23\u0E27\u0E14\u0E40\u0E23\u0E47\u0E27\u0E41\ + \u0E25\u0E30\u0E08\u0E23\u0E34\u0E07\u0E08\u0E31\u0E07\u0E40\u0E1E\u0E35\u0E22\ + \u0E07\u0E43\u0E14 \u0E01\u0E32\u0E23\u0E40\u0E2A\u0E35\u0E22\u0E2A\u0E25\u0E30\ + \u0E17\u0E35\u0E48\u0E04\u0E38\u0E13\u0E16\u0E39\u0E01\u0E02\u0E2D\u0E43\u0E2B\ + \u0E49\u0E17\u0E33\u0E19\u0E31\u0E49\u0E19\u0E40\u0E25\u0E47\u0E01\u0E19\u0E49\ + \u0E2D\u0E22\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E43\u0E14 \u0E44\u0E21\u0E48\u0E2A\ + \u0E33\u0E04\u0E31\u0E0D\u0E19\u0E31\u0E01 \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\ + \u0E1B\u0E23\u0E35\u0E22\u0E1A\u0E40\u0E17\u0E35\u0E22\u0E1A\u0E01\u0E31\u0E19\ + \ \u0E40\u0E07\u0E34\u0E19\u0E2A\u0E25\u0E36\u0E07\u0E41\u0E25\u0E30\u0E40\u0E07\ + \u0E34\u0E19\u0E44\u0E21\u0E48\u0E01\u0E35\u0E48\u0E14\u0E2D\u0E25\u0E25\u0E32\ + \u0E23\u0E4C\u0E17\u0E35\u0E48\u0E23\u0E27\u0E1A\u0E23\u0E27\u0E21\u0E08\u0E32\ + \u0E01\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E19\u0E41\u0E15\u0E48\u0E25\u0E30\ + \u0E04\u0E19\u0E08\u0E30\u0E25\u0E14\u0E04\u0E48\u0E32\u0E43\u0E19\u0E10\u0E32\ + \u0E19\u0E30\u0E04\u0E23\u0E2D\u0E1A\u0E04\u0E23\u0E2D\u0E07\u0E44\u0E14\u0E49\ + \u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E23\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\ + \u0E17\u0E35\u0E22\u0E1A\u0E01\u0E31\u0E1A\u0E1C\u0E25\u0E1B\u0E23\u0E30\u0E42\ + \u0E22\u0E0A\u0E19\u0E4C\u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E14\u0E35\ + \u0E21\u0E32\u0E01\u0E21\u0E32\u0E22\u0E17\u0E35\u0E48\u0E08\u0E30\u0E40\u0E1B\ + \u0E47\u0E19\u0E2B\u0E25\u0E31\u0E01\u0E1B\u0E23\u0E30\u0E01\u0E31\u0E19\u0E43\ + \u0E2B\u0E49\u0E01\u0E31\u0E1A\u0E04\u0E19\u0E27\u0E34\u0E01\u0E25\u0E08\u0E23\ + \u0E34\u0E15\u0E17\u0E35\u0E48\u0E17\u0E38\u0E01\u0E02\u0E4C\u0E17\u0E23\u0E21\ + \u0E32\u0E19 ... \u0E42\u0E14\u0E22\u0E01\u0E32\u0E23\u0E2D\u0E38\u0E17\u0E34\ + \u0E28\u0E41\u0E25\u0E30\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E40\u0E07\u0E34\ + \u0E19\u0E17\u0E38\u0E19\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E1E\ + \u0E2D\u0E43\u0E19\u0E01\u0E32\u0E23\u0E01\u0E48\u0E2D\u0E2A\u0E23\u0E49\u0E32\ + \u0E07 \u0E02\u0E2D\u0E07\u0E42\u0E23\u0E07\u0E1E\u0E22\u0E32\u0E1A\u0E32\u0E25\ + \u0E17\u0E35\u0E48\u0E40\u0E2B\u0E21\u0E32\u0E30\u0E2A\u0E21\u2026 \u2014Dorothea\ + \ Dix, Memorial Soliciting a State Hospital for the Protection and Cure of the\ + \ Insane, \u0E2A\u0E48\u0E07\u0E44\u0E1B\u0E22\u0E31\u0E07\u0E2A\u0E21\u0E31\ + \u0E0A\u0E0A\u0E32\u0E43\u0E2B\u0E0D\u0E48\u0E41\u0E2B\u0E48\u0E07\u0E19\u0E2D\ + \u0E23\u0E4C\u0E17\u0E41\u0E04\u0E42\u0E23\u0E44\u0E25\u0E19\u0E32 \u0E1E\u0E24\ + \u0E28\u0E08\u0E34\u0E01\u0E32\u0E22\u0E19 1848 Dorothea Dix \u0E40\u0E17\u0E35\ + \u0E22\u0E1A\u0E44\u0E14\u0E49\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\ + \u0E01\u0E31\u0E1A\u0E43\u0E04\u0E23" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_us_history +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_high_school_us_history +task_alias: high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b579d8abcb5fb39983959ba2c6b2215f75c05660 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_high_school_world_history.yaml @@ -0,0 +1,320 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E28\u0E32\u0E2A\u0E19\u0E32\u0E2E\u0E34\u0E19\u0E14\u0E39" + B: "\u0E1E\u0E23\u0E30\u0E1E\u0E38\u0E17\u0E18\u0E28\u0E32\u0E2A\u0E19\u0E32" + C: "\u0E25\u0E31\u0E17\u0E18\u0E34\u0E0A\u0E34\u0E19\u0E42\u0E15" + D: "\u0E28\u0E32\u0E2A\u0E19\u0E32\u0E42\u0E0B\u0E42\u0E23\u0E2D\u0E31\u0E2A\ + \u0E40\u0E15\u0E2D\u0E23\u0E4C" + input_correct_responses: + - A + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u0E40\u0E02\u0E32\u0E21\u0E35\ + \u0E1C\u0E25\u0E07\u0E32\u0E19\u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E1B\ + \u0E23\u0E32\u0E23\u0E16\u0E19\u0E32\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\ + \u0E41\u0E25\u0E30\u0E19\u0E49\u0E33\u0E2B\u0E2D\u0E21\u0E17\u0E31\u0E49\u0E07\ + \u0E2B\u0E21\u0E14\u0E41\u0E25\u0E30\u0E23\u0E2A\u0E0A\u0E32\u0E15\u0E34\u0E17\ + \u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14 \u0E40\u0E02\u0E32\u0E42\u0E2D\u0E1A\u0E25\ + \u0E49\u0E2D\u0E21\u0E17\u0E31\u0E49\u0E07\u0E08\u0E31\u0E01\u0E23\u0E27\u0E32\ + \u0E25\u0E41\u0E25\u0E30\u0E43\u0E19\u0E04\u0E27\u0E32\u0E21\u0E40\u0E07\u0E35\ + \u0E22\u0E1A\u0E01\u0E47\u0E23\u0E31\u0E01\u0E17\u0E38\u0E01\u0E04\u0E19 \u0E19\ + \u0E35\u0E48\u0E04\u0E37\u0E2D\u0E27\u0E34\u0E0D\u0E0D\u0E32\u0E13\u0E17\u0E35\ + \u0E48\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E43\u0E08\u0E02\u0E2D\u0E07\u0E09\ + \u0E31\u0E19 \u0E19\u0E35\u0E48\u0E04\u0E37\u0E2D\u0E1E\u0E23\u0E32\u0E2B\u0E21\ + \u0E13\u0E4C \u0E09\u0E31\u0E19\u0E08\u0E30\u0E21\u0E32\u0E2B\u0E32\u0E40\u0E02\ + \u0E32\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E09\u0E31\u0E19\u0E08\u0E32\u0E01\u0E0A\ + \u0E35\u0E27\u0E34\u0E15\u0E19\u0E35\u0E49\u0E44\u0E1B \u0E41\u0E25\u0E30\u0E1C\ + \u0E39\u0E49\u0E17\u0E35\u0E48\u0E21\u0E35\u0E28\u0E23\u0E31\u0E17\u0E18\u0E32\ + \u0E41\u0E25\u0E30\u0E44\u0E21\u0E48\u0E2A\u0E07\u0E2A\u0E31\u0E22\u0E01\u0E47\ + \u0E08\u0E30\u0E21\u0E32\u0E2B\u0E32\u0E40\u0E02\u0E32 \u2014\u0E2D\u0E38\u0E1B\ + \u0E19\u0E34\u0E29\u0E31\u0E17, \u0E2D\u0E34\u0E19\u0E40\u0E14\u0E35\u0E22,\ + \ \u0E04. 1,000 \u0E01\u0E48\u0E2D\u0E19\u0E04\u0E23\u0E34\u0E2A\u0E15\u0E28\ + \u0E31\u0E01\u0E23\u0E32\u0E0A \u0E1C\u0E39\u0E49\u0E1E\u0E39\u0E14\u0E19\u0E48\ + \u0E32\u0E08\u0E30\u0E19\u0E31\u0E1A\u0E16\u0E37\u0E2D\u0E28\u0E32\u0E2A\u0E19\ + \u0E32\u0E43\u0E14\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E27\u0E31\u0E15\u0E34\u0E2D\u0E38\ + \u0E15\u0E2A\u0E32\u0E2B\u0E01\u0E23\u0E23\u0E21" + B: "\u0E01\u0E32\u0E23\u0E1B\u0E25\u0E14\u0E1B\u0E25\u0E48\u0E2D\u0E22\u0E2D\ + \u0E32\u0E13\u0E32\u0E19\u0E34\u0E04\u0E21" + C: "\u0E2A\u0E21\u0E32\u0E04\u0E21\u0E01\u0E32\u0E23\u0E04\u0E49\u0E32\u0E40\ + \u0E2A\u0E23\u0E35\u0E23\u0E30\u0E14\u0E31\u0E1A\u0E20\u0E39\u0E21\u0E34\u0E20\ + \u0E32\u0E04" + D: "\u0E2D\u0E2D\u0E17\u0E32\u0E23\u0E4C\u0E01\u0E35\u0E49" + input_correct_responses: + - B + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u201C\u0E01\u0E32\u0E23\u0E15\ + \u0E48\u0E2D\u0E2A\u0E39\u0E49\u0E01\u0E31\u0E1A\u0E25\u0E31\u0E17\u0E18\u0E34\ + \u0E2D\u0E32\u0E13\u0E32\u0E19\u0E34\u0E04\u0E21\u0E43\u0E2B\u0E21\u0E48\u0E44\ + \u0E21\u0E48\u0E44\u0E14\u0E49\u0E21\u0E38\u0E48\u0E07\u0E2B\u0E21\u0E32\u0E22\ + \u0E17\u0E35\u0E48\u0E08\u0E30\u0E41\u0E22\u0E01\u0E40\u0E21\u0E37\u0E2D\u0E07\ + \u0E2B\u0E25\u0E27\u0E07\u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\ + \u0E17\u0E35\u0E48\u0E1E\u0E31\u0E12\u0E19\u0E32\u0E41\u0E25\u0E49\u0E27\u0E2D\ + \u0E2D\u0E01\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E14\u0E33\u0E40\u0E19\u0E34\ + \u0E19\u0E07\u0E32\u0E19\u0E43\u0E19\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E17\ + \u0E35\u0E48\u0E1E\u0E31\u0E12\u0E19\u0E32\u0E19\u0E49\u0E2D\u0E22\u0E01\u0E27\ + \u0E48\u0E32 \u0E41\u0E15\u0E48\u0E21\u0E35\u0E40\u0E1B\u0E49\u0E32\u0E2B\u0E21\ + \u0E32\u0E22\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E1B\u0E49\u0E2D\u0E07\u0E01\u0E31\ + \u0E19\u0E44\u0E21\u0E48\u0E43\u0E2B\u0E49\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E17\ + \u0E32\u0E07\u0E01\u0E32\u0E23\u0E40\u0E07\u0E34\u0E19\u0E02\u0E2D\u0E07\u0E1B\ + \u0E23\u0E30\u0E40\u0E17\u0E28\u0E17\u0E35\u0E48\u0E1E\u0E31\u0E12\u0E19\u0E32\ + \u0E41\u0E25\u0E49\u0E27\u0E16\u0E39\u0E01\u0E19\u0E33\u0E44\u0E1B\u0E43\u0E0A\ + \u0E49\u0E43\u0E19\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\u0E17\u0E35\u0E48\u0E17\ + \u0E33\u0E43\u0E2B\u0E49\u0E22\u0E32\u0E01\u0E08\u0E19\u0E25\u0E07 \u0E01\u0E32\ + \u0E23\u0E44\u0E21\u0E48\u0E1D\u0E31\u0E01\u0E43\u0E1D\u0E48\u0E1D\u0E48\u0E32\ + \u0E22\u0E43\u0E14\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E17\u0E35\u0E48\u0E01\u0E32\ + \u0E19\u0E32\u0E41\u0E25\u0E30\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E2D\u0E37\ + \u0E48\u0E19 \u0E46 \u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E01\u0E31\u0E19\ + \ \u0E04\u0E37\u0E2D \u0E2D\u0E32\u0E28\u0E31\u0E22\u0E04\u0E27\u0E32\u0E21\u0E23\ + \u0E48\u0E27\u0E21\u0E21\u0E37\u0E2D\u0E01\u0E31\u0E1A\u0E17\u0E38\u0E01\u0E23\ + \u0E31\u0E10\u0E44\u0E21\u0E48\u0E27\u0E48\u0E32\u0E08\u0E30\u0E40\u0E1B\u0E47\ + \u0E19\u0E17\u0E38\u0E19\u0E19\u0E34\u0E22\u0E21 \u0E2A\u0E31\u0E07\u0E04\u0E21\ + \u0E19\u0E34\u0E22\u0E21 \u0E2B\u0E23\u0E37\u0E2D\u0E40\u0E28\u0E23\u0E29\u0E10\ + \u0E01\u0E34\u0E08\u0E1C\u0E2A\u0E21 \u0E14\u0E31\u0E07\u0E19\u0E31\u0E49\u0E19\ + \ \u0E19\u0E42\u0E22\u0E1A\u0E32\u0E22\u0E14\u0E31\u0E07\u0E01\u0E25\u0E48\u0E32\ + \u0E27\u0E08\u0E36\u0E07\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\u0E2D\ + \u0E07\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E25\u0E07\u0E17\u0E38\u0E19\u0E08\ + \u0E32\u0E01\u0E15\u0E48\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E08\ + \u0E32\u0E01\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E17\u0E38\u0E19\u0E19\u0E34\ + \u0E22\u0E21\u0E41\u0E15\u0E48\u0E15\u0E49\u0E2D\u0E07 \u0E25\u0E07\u0E17\u0E38\ + \u0E19\u0E15\u0E32\u0E21\u0E41\u0E1C\u0E19\u0E23\u0E30\u0E14\u0E31\u0E1A\u0E0A\ + \u0E32\u0E15\u0E34\u0E17\u0E35\u0E48\u0E23\u0E48\u0E32\u0E07\u0E02\u0E36\u0E49\ + \u0E19\u0E42\u0E14\u0E22\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E02\u0E2D\u0E07\ + \u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E1D\u0E31\u0E01\u0E43\ + \u0E1D\u0E48\u0E1D\u0E48\u0E32\u0E22\u0E43\u0E14\u0E42\u0E14\u0E22\u0E04\u0E33\ + \u0E19\u0E36\u0E07\u0E16\u0E36\u0E07\u0E1C\u0E25\u0E1B\u0E23\u0E30\u0E42\u0E22\ + \u0E0A\u0E19\u0E4C\u0E02\u0E2D\u0E07\u0E15\u0E19\u0E40\u0E2D\u0E07\u0E40\u0E1B\ + \u0E47\u0E19\u0E2A\u0E33\u0E04\u0E31\u0E0D \u0E1B\u0E23\u0E30\u0E40\u0E14\u0E47\ + \u0E19\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48 \u0E1C\u0E25\u0E15\u0E2D\u0E1A\u0E41\ + \u0E17\u0E19\u0E17\u0E35\u0E48\u0E19\u0E31\u0E01\u0E25\u0E07\u0E17\u0E38\u0E19\ + \u0E15\u0E48\u0E32\u0E07\u0E0A\u0E32\u0E15\u0E34\u0E44\u0E14\u0E49\u0E23\u0E31\ + \u0E1A\u0E08\u0E32\u0E01\u0E01\u0E32\u0E23\u0E25\u0E07\u0E17\u0E38\u0E19\u0E02\ + \u0E2D\u0E07\u0E40\u0E02\u0E32...\u0E04\u0E33\u0E16\u0E32\u0E21\u0E04\u0E37\u0E2D\ + \u0E2D\u0E33\u0E19\u0E32\u0E08\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2B\u0E19\u0E36\ + \u0E48\u0E07 \u0E23\u0E31\u0E10\u0E43\u0E19 \u0E01\u0E32\u0E23\u0E22\u0E36\u0E14\ + \u0E01\u0E38\u0E21\u0E02\u0E2D\u0E07\u0E25\u0E31\u0E17\u0E18\u0E34\u0E2D\u0E32\ + \u0E13\u0E32\u0E19\u0E34\u0E04\u0E21\u0E43\u0E2B\u0E21\u0E48\u0E44\u0E21\u0E48\ + \u0E43\u0E0A\u0E48\u0E15\u0E31\u0E27\u0E1A\u0E07\u0E01\u0E32\u0E23\u0E0A\u0E30\ + \u0E15\u0E32\u0E01\u0E23\u0E23\u0E21\u0E02\u0E2D\u0E07\u0E21\u0E31\u0E19\u0E40\ + \u0E2D\u0E07" Kwame Nkrumah, Neo-Colonialism, 1965 \u0E02\u0E49\u0E2D\u0E43\ + \u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49\u0E43\u0E2B\u0E49\u0E1A\ + \u0E23\u0E34\u0E1A\u0E17\u0E17\u0E35\u0E48\u0E14\u0E35\u0E17\u0E35\u0E48\u0E2A\ + \u0E38\u0E14\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E07\u0E32\u0E19\u0E40\u0E02\ + \u0E35\u0E22\u0E19\u0E02\u0E2D\u0E07 Nkrumah" + - input_choice_list: + A: "\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E01\ + \u0E32\u0E23\u0E43\u0E0A\u0E49\u0E41\u0E23\u0E07\u0E07\u0E32\u0E19\u0E40\u0E14\ + \u0E47\u0E01" + B: "\u0E2D\u0E32\u0E22\u0E38\u0E02\u0E31\u0E22\u0E17\u0E35\u0E48\u0E25\u0E14\ + \u0E25\u0E07\u0E43\u0E19\u0E40\u0E22\u0E2D\u0E23\u0E21\u0E19\u0E35" + C: "\u0E04\u0E33\u0E15\u0E34\u0E0A\u0E21\u0E02\u0E2D\u0E07\u0E2D\u0E31\u0E15\ + \u0E23\u0E32\u0E20\u0E32\u0E29\u0E35\u0E01\u0E32\u0E23\u0E04\u0E49\u0E32\u0E02\ + \u0E2D\u0E07\u0E40\u0E22\u0E2D\u0E23\u0E21\u0E31\u0E19" + D: "\u0E1C\u0E25\u0E40\u0E2A\u0E35\u0E22\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\ + \u0E14\u0E08\u0E32\u0E01\u0E23\u0E30\u0E1A\u0E1A\u0E17\u0E38\u0E19\u0E19\u0E34\ + \u0E22\u0E21\u0E2D\u0E38\u0E15\u0E2A\u0E32\u0E2B\u0E01\u0E23\u0E23\u0E21" + input_correct_responses: + - D + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 \u201C\u0E04\u0E27\u0E32\u0E21\ + \u0E04\u0E31\u0E1A\u0E02\u0E49\u0E2D\u0E07\u0E43\u0E08\u0E17\u0E35\u0E48\u0E41\ + \u0E17\u0E49\u0E08\u0E23\u0E34\u0E07\u0E02\u0E2D\u0E07\u0E04\u0E19\u0E07\u0E32\ + \u0E19\u0E04\u0E37\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E21\u0E31\ + \u0E48\u0E19\u0E04\u0E07\u0E43\u0E19\u0E04\u0E27\u0E32\u0E21\u0E40\u0E1B\u0E47\ + \u0E19\u0E2D\u0E22\u0E39\u0E48\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32 \u0E40\u0E02\ + \u0E32\u0E44\u0E21\u0E48\u0E41\u0E19\u0E48\u0E43\u0E08\u0E27\u0E48\u0E32\u0E40\ + \u0E02\u0E32\u0E08\u0E30\u0E21\u0E35\u0E07\u0E32\u0E19\u0E17\u0E33\u0E15\u0E25\ + \u0E2D\u0E14\u0E44\u0E1B \u0E40\u0E02\u0E32\u0E44\u0E21\u0E48\u0E21\u0E31\u0E48\ + \u0E19\u0E43\u0E08\u0E27\u0E48\u0E32\u0E40\u0E02\u0E32\u0E08\u0E30\u0E21\u0E35\ + \u0E2A\u0E38\u0E02\u0E20\u0E32\u0E1E\u0E14\u0E35\u0E15\u0E25\u0E2D\u0E14\u0E44\ + \u0E1B \u0E41\u0E25\u0E30\u0E40\u0E02\u0E32\u0E40\u0E25\u0E47\u0E07\u0E40\u0E2B\ + \u0E47\u0E19\u0E27\u0E48\u0E32\u0E27\u0E31\u0E19\u0E2B\u0E19\u0E36\u0E48\u0E07\ + \u0E40\u0E02\u0E32\u0E08\u0E30\u0E41\u0E01\u0E48\u0E41\u0E25\u0E30\u0E44\u0E21\ + \u0E48\u0E40\u0E2B\u0E21\u0E32\u0E30\u0E17\u0E35\u0E48\u0E08\u0E30\u0E17\u0E33\ + \u0E07\u0E32\u0E19 \u0E16\u0E49\u0E32\u0E40\u0E02\u0E32\u0E15\u0E01\u0E2D\u0E22\ + \u0E39\u0E48\u0E43\u0E19\u0E04\u0E27\u0E32\u0E21\u0E22\u0E32\u0E01\u0E08\u0E19\ + \u0E41\u0E21\u0E49\u0E27\u0E48\u0E32\u0E08\u0E30\u0E40\u0E08\u0E47\u0E1A\u0E1B\ + \u0E48\u0E27\u0E22\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E27\u0E25\u0E32\u0E19\u0E32\ + \u0E19 \u0E40\u0E02\u0E32\u0E01\u0E47\u0E2B\u0E21\u0E14\u0E2B\u0E19\u0E17\u0E32\ + \u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E2A\u0E21\u0E1A\u0E39\u0E23\u0E13\u0E4C\ + \ \u0E15\u0E23\u0E27\u0E08\u0E2A\u0E2D\u0E1A\u0E2D\u0E38\u0E1B\u0E01\u0E23\u0E13\ + \u0E4C\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32\u0E40\u0E2D\u0E07 \u0E41\u0E25\u0E30\ + \u0E1B\u0E31\u0E08\u0E08\u0E38\u0E1A\u0E31\u0E19\u0E2A\u0E31\u0E07\u0E04\u0E21\ + \u0E44\u0E21\u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E20\u0E32\u0E23\u0E30\ + \u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\u0E17\u0E35\u0E48\u0E41\u0E17\u0E49\ + \u0E08\u0E23\u0E34\u0E07\u0E43\u0E14\u0E46 \u0E15\u0E48\u0E2D\u0E40\u0E02\u0E32\ + \u0E19\u0E2D\u0E01\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E08\u0E32\u0E01\u0E04\u0E27\ + \u0E32\u0E21\u0E0A\u0E48\u0E27\u0E22\u0E40\u0E2B\u0E25\u0E37\u0E2D\u0E15\u0E32\ + \u0E21\u0E1B\u0E01\u0E15\u0E34\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E04\u0E19\ + \u0E08\u0E19 \u0E41\u0E21\u0E49\u0E27\u0E48\u0E32\u0E40\u0E02\u0E32\u0E08\u0E30\ + \u0E21\u0E35\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E17\u0E33\u0E07\u0E32\u0E19\u0E15\ + \u0E25\u0E2D\u0E14\u0E40\u0E27\u0E25\u0E32\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E0B\ + \u0E37\u0E48\u0E2D\u0E2A\u0E31\u0E15\u0E22\u0E4C\u0E41\u0E25\u0E30\u0E02\u0E22\ + \u0E31\u0E19\u0E2B\u0E21\u0E31\u0E48\u0E19\u0E40\u0E1E\u0E35\u0E22\u0E23 \u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E44\u0E23\u0E01\u0E47\u0E15\u0E32\u0E21 \u0E04\u0E27\ + \u0E32\u0E21\u0E0A\u0E48\u0E27\u0E22\u0E40\u0E2B\u0E25\u0E37\u0E2D\u0E15\u0E32\ + \u0E21\u0E1B\u0E01\u0E15\u0E34\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E04\u0E19\ + \u0E08\u0E19\u0E01\u0E25\u0E31\u0E1A\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E35\u0E48\ + \u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E21\ + \u0E32\u0E01\u0E42\u0E14\u0E22\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E43\u0E19\u0E40\ + \u0E21\u0E37\u0E2D\u0E07\u0E43\u0E2B\u0E0D\u0E48\u0E0B\u0E36\u0E48\u0E07\u0E40\ + \u0E25\u0E27\u0E23\u0E49\u0E32\u0E22\u0E01\u0E27\u0E48\u0E32\u0E43\u0E19\u0E1B\ + \u0E23\u0E30\u0E40\u0E17\u0E28\u0E21\u0E32\u0E01\u201D Otto von Bismarck, 1884\ + \ Otto von Bismarck \u0E19\u0E48\u0E32\u0E08\u0E30\u0E01\u0E25\u0E48\u0E32\u0E27\ + \u0E2A\u0E38\u0E19\u0E17\u0E23\u0E1E\u0E08\u0E19\u0E4C\u0E19\u0E35\u0E49\u0E40\ + \u0E1E\u0E37\u0E48\u0E2D\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E15\u0E48\ + \u0E2D\u0E1B\u0E23\u0E30\u0E40\u0E14\u0E47\u0E19\u0E43\u0E14\u0E15\u0E48\u0E2D\ + \u0E44\u0E1B\u0E19\u0E35\u0E49?" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E23\u0E31\u0E01\u0E29\u0E32\u0E2D\u0E33\u0E19\u0E32\ + \u0E08\u0E2A\u0E39\u0E07\u0E2A\u0E38\u0E14\u0E17\u0E32\u0E07\u0E17\u0E2B\u0E32\ + \u0E23\u0E43\u0E19\u0E17\u0E38\u0E01\u0E27\u0E34\u0E16\u0E35\u0E17\u0E32\u0E07" + B: "\u0E02\u0E22\u0E32\u0E22\u0E04\u0E27\u0E32\u0E21\u0E15\u0E36\u0E07\u0E40\ + \u0E04\u0E23\u0E35\u0E22\u0E14\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E19\ + \u0E34\u0E01\u0E32\u0E22\u0E17\u0E32\u0E07\u0E28\u0E32\u0E2A\u0E19\u0E32" + C: "\u0E1B\u0E31\u0E08\u0E08\u0E31\u0E22\u0E17\u0E35\u0E48\u0E17\u0E33\u0E43\ + \u0E2B\u0E49\u0E08\u0E31\u0E01\u0E23\u0E27\u0E23\u0E23\u0E14\u0E34\u0E2D\u0E2D\ + \u0E15\u0E42\u0E15\u0E21\u0E31\u0E19\u0E25\u0E48\u0E21\u0E2A\u0E25\u0E32\u0E22" + D: "\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E22\u0E32\u0E22\u0E32\u0E21\u0E2A\u0E23\ + \u0E49\u0E32\u0E07\u0E2A\u0E31\u0E19\u0E15\u0E34\u0E43\u0E19\u0E2B\u0E21\u0E39\ + \u0E48\u0E2D\u0E32\u0E13\u0E32\u0E08\u0E31\u0E01\u0E23\u0E2D\u0E34\u0E2A\u0E25\ + \u0E32\u0E21" + input_correct_responses: + - B + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 "\u0E2D\u0E31\u0E19\u0E17\ + \u0E35\u0E48\u0E08\u0E23\u0E34\u0E07 \u0E1F\u0E31\u0E15\u0E27\u0E32\u0E02\u0E2D\ + \u0E07 [\u0E19\u0E31\u0E01\u0E27\u0E34\u0E0A\u0E32\u0E01\u0E32\u0E23] \u0E17\ + \u0E35\u0E48\u0E21\u0E35\u0E0A\u0E37\u0E48\u0E2D\u0E40\u0E2A\u0E35\u0E22\u0E07\ + \u0E0B\u0E36\u0E48\u0E07\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\u0E40\u0E2B\u0E47\ + \u0E19\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E40\u0E2B\u0E15\ + \u0E38\u0E1C\u0E25\u0E41\u0E25\u0E30\u0E1B\u0E23\u0E30\u0E40\u0E1E\u0E13\u0E35\ + \u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E01\u0E31\u0E19 \u0E41\u0E25\u0E30\u0E09\ + \u0E31\u0E19\u0E17\u0E32\u0E21\u0E15\u0E34\u0E02\u0E2D\u0E07\u0E0A\u0E38\u0E21\ + \u0E0A\u0E19\u0E2A\u0E38\u0E2B\u0E19\u0E35\u0E48\u0E40\u0E2B\u0E47\u0E19\u0E1E\ + \u0E49\u0E2D\u0E07\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E19\u0E27\u0E48\u0E32\ + \u0E20\u0E32\u0E23\u0E30\u0E1C\u0E39\u0E01\u0E1E\u0E31\u0E19\u0E43\u0E19\u0E2A\ + \u0E21\u0E31\u0E22\u0E42\u0E1A\u0E23\u0E32\u0E13\u0E02\u0E2D\u0E07\u0E01\u0E32\ + \u0E23\u0E01\u0E27\u0E32\u0E14\u0E25\u0E49\u0E32\u0E07 \u0E01\u0E32\u0E23\u0E17\ + \u0E33\u0E25\u0E32\u0E22\u0E25\u0E49\u0E32\u0E07 \u0E41\u0E25\u0E30\u0E01\u0E32\ + \u0E23\u0E02\u0E31\u0E1A\u0E44\u0E25\u0E48\u0E19\u0E27\u0E31\u0E15\u0E01\u0E23\ + \u0E23\u0E21\u0E17\u0E35\u0E48\u0E0A\u0E31\u0E48\u0E27\u0E23\u0E49\u0E32\u0E22\ + \ \u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E1B\u0E49\ + \u0E32\u0E2B\u0E21\u0E32\u0E22\u0E02\u0E2D\u0E07\u0E1C\u0E39\u0E49\u0E17\u0E23\ + \u0E07\u0E2A\u0E39\u0E07\u0E2A\u0E48\u0E07\u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\ + \ \u0E04\u0E27\u0E32\u0E21\u0E17\u0E30\u0E40\u0E22\u0E2D\u0E17\u0E30\u0E22\u0E32\ + \u0E19\u0E40\u0E1E\u0E23\u0E32\u0E30 "\u0E04\u0E27\u0E32\u0E21\u0E01\u0E23\ + \u0E30\u0E15\u0E37\u0E2D\u0E23\u0E37\u0E2D\u0E23\u0E49\u0E19\u0E17\u0E32\u0E07\ + \u0E28\u0E32\u0E2A\u0E19\u0E32\u0E40\u0E1B\u0E47\u0E19\u0E0A\u0E31\u0E22\u0E0A\ + \u0E19\u0E30\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E28\u0E23\u0E31\u0E17\u0E18\ + \u0E32\u0E02\u0E2D\u0E07\u0E1E\u0E23\u0E30\u0E1C\u0E39\u0E49\u0E40\u0E1B\u0E47\ + \u0E19\u0E40\u0E08\u0E49\u0E32\u0E1C\u0E39\u0E49\u0E17\u0E23\u0E07\u0E40\u0E21\ + \u0E15\u0E15\u0E32" \u0E14\u0E31\u0E07\u0E19\u0E31\u0E49\u0E19\u0E15\u0E32\ + \u0E21\u0E04\u0E33\u0E1E\u0E39\u0E14\u0E02\u0E2D\u0E07\u0E17\u0E48\u0E32\u0E19\ + \u0E28\u0E32\u0E2A\u0E14\u0E32 (\u0E2A\u0E31\u0E19\u0E15\u0E34\u0E20\u0E32\u0E1E\ + \u0E01\u0E31\u0E1A\u0E40\u0E02\u0E32!) "\u0E43\u0E04\u0E23\u0E01\u0E47\u0E15\ + \u0E32\u0E21\u0E17\u0E35\u0E48\u0E19\u0E33\u0E19\u0E27\u0E31\u0E15\u0E01\u0E23\ + \u0E23\u0E21\u0E17\u0E35\u0E48\u0E0A\u0E31\u0E48\u0E27\u0E23\u0E49\u0E32\u0E22\ + \u0E21\u0E32\u0E2A\u0E39\u0E48\u0E23\u0E30\u0E40\u0E1A\u0E35\u0E22\u0E1A\u0E02\ + \u0E2D\u0E07\u0E40\u0E23\u0E32\u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\u0E16\u0E39\ + \u0E01\u0E02\u0E31\u0E1A\u0E44\u0E25\u0E48" \u0E41\u0E25\u0E30 "\u0E43\ + \u0E04\u0E23\u0E01\u0E47\u0E15\u0E32\u0E21\u0E17\u0E35\u0E48\u0E17\u0E33\u0E40\ + \u0E0A\u0E48\u0E19\u0E19\u0E31\u0E49\u0E19 \u0E2A\u0E34\u0E48\u0E07\u0E43\u0E14\ + \u0E17\u0E35\u0E48\u0E02\u0E31\u0E14\u0E15\u0E48\u0E2D\u0E04\u0E33\u0E2A\u0E31\ + \u0E48\u0E07\u0E02\u0E2D\u0E07\u0E40\u0E23\u0E32\u0E15\u0E49\u0E2D\u0E07\u0E16\ + \u0E39\u0E01\u0E02\u0E31\u0E1A\u0E44\u0E25\u0E48" \u0E01\u0E32\u0E23\u0E01\ + \u0E23\u0E30\u0E17\u0E33\u0E01\u0E25\u0E32\u0E22\u0E40\u0E1B\u0E47\u0E19\u0E2A\ + \u0E34\u0E48\u0E07\u0E08\u0E33\u0E40\u0E1B\u0E47\u0E19\u0E41\u0E25\u0E30\u0E40\ + \u0E23\u0E48\u0E07\u0E14\u0E48\u0E27\u0E19..." \u0E08\u0E14\u0E2B\u0E21\ + \u0E32\u0E22\u0E08\u0E32\u0E01\u0E2A\u0E38\u0E25\u0E15\u0E48\u0E32\u0E19\u0E40\ + \u0E0B\u0E25\u0E34\u0E21\u0E17\u0E35\u0E48 1 \u0E41\u0E2B\u0E48\u0E07\u0E2D\u0E2D\ + \u0E15\u0E42\u0E15\u0E21\u0E31\u0E19\u0E16\u0E36\u0E07\u0E0B\u0E32\u0E1F\u0E32\ + \u0E27\u0E34\u0E14 \u0E0A\u0E32\u0E2B\u0E4C \u0E2D\u0E34\u0E2A\u0E21\u0E32\u0E2D\ + \u0E34\u0E25\u0E17\u0E35\u0E48 1, 1514 \u0E08\u0E14\u0E2B\u0E21\u0E32\u0E22\u0E08\ + \u0E32\u0E01\u0E40\u0E0B\u0E25\u0E34\u0E21\u0E17\u0E35\u0E48 1 \u0E40\u0E1B\u0E47\ + \u0E19\u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E17\u0E35\u0E48\u0E0A\ + \u0E31\u0E14\u0E40\u0E08\u0E19\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E02\u0E2D\ + \u0E07\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\ + \u0E49" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E25\u0E48\u0E21\u0E2A\u0E25\u0E32\u0E22\u0E02\u0E2D\ + \u0E07\u0E40\u0E2A\u0E49\u0E19\u0E17\u0E32\u0E07\u0E01\u0E32\u0E23\u0E04\u0E49\ + \u0E32\u0E1C\u0E48\u0E32\u0E19\u0E01\u0E32\u0E23\u0E25\u0E48\u0E21\u0E2A\u0E25\ + \u0E32\u0E22\u0E02\u0E2D\u0E07\u0E42\u0E04\u0E23\u0E07\u0E2A\u0E23\u0E49\u0E32\ + \u0E07\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E08\u0E31\u0E14\ + \u0E15\u0E31\u0E49\u0E07\u0E02\u0E36\u0E49\u0E19" + B: "\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\ + \u0E02\u0E2D\u0E07\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E42\u0E25\u0E01\ + \u0E1C\u0E48\u0E32\u0E19\u0E40\u0E2A\u0E1A\u0E35\u0E22\u0E07\u0E2D\u0E32\u0E2B\ + \u0E32\u0E23\u0E17\u0E35\u0E48\u0E2D\u0E38\u0E14\u0E21\u0E2A\u0E21\u0E1A\u0E39\ + \u0E23\u0E13\u0E4C\u0E21\u0E32\u0E01\u0E02\u0E36\u0E49\u0E19" + C: "\u0E01\u0E32\u0E23\u0E41\u0E1E\u0E23\u0E48\u0E01\u0E23\u0E30\u0E08\u0E32\ + \u0E22\u0E02\u0E2D\u0E07\u0E23\u0E30\u0E1A\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E40\ + \u0E0A\u0E37\u0E48\u0E2D\u0E02\u0E2D\u0E07\u0E08\u0E35\u0E19\u0E41\u0E25\u0E30\ + \u0E2D\u0E34\u0E19\u0E40\u0E14\u0E35\u0E22\u0E44\u0E1B\u0E17\u0E31\u0E48\u0E27\ + \u0E42\u0E25\u0E01" + D: "\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E2A\u0E07\u0E1A\u0E43\u0E19\ + \u0E2A\u0E31\u0E07\u0E04\u0E21\u0E17\u0E35\u0E48\u0E40\u0E1E\u0E34\u0E48\u0E21\ + \u0E02\u0E36\u0E49\u0E19" + input_correct_responses: + - B + input_question: "\u0E04\u0E33\u0E16\u0E32\u0E21\u0E19\u0E35\u0E49\u0E2D\u0E49\u0E32\ + \u0E07\u0E2D\u0E34\u0E07\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\ + \u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49 "\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E19\u0E49\u0E2D\u0E22\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E43\u0E19\u0E2A\ + \u0E31\u0E07\u0E04\u0E21 [\u0E02\u0E2D\u0E07\u0E42\u0E25\u0E01] \u0E08\u0E30\ + \u0E15\u0E49\u0E2D\u0E07\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1C\u0E25\u0E1C\u0E25\ + \u0E34\u0E15\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E21\u0E2B\u0E32\u0E28\u0E32\u0E25\ + \ [\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E1A\u0E23\u0E23\u0E25\u0E38\ + \u0E04\u0E27\u0E32\u0E21\u0E40\u0E1B\u0E47\u0E19\u0E40\u0E08\u0E49\u0E32\u0E42\ + \u0E25\u0E01] \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E42\u0E14\u0E14\u0E04\u0E27\ + \u0E2D\u0E19\u0E15\u0E31\u0E21\u0E19\u0E31\u0E49\u0E19\u0E08\u0E30\u0E15\u0E49\ + \u0E2D\u0E07\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E01\u0E48\u0E2D\ + \u0E19\u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E27\u0E31\u0E15\u0E34\u0E17\u0E32\ + \u0E07\u0E27\u0E34\u0E17\u0E22\u0E32\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C \u0E40\ + \u0E17\u0E04\u0E42\u0E19\u0E42\u0E25\u0E22\u0E35 \u0E01\u0E32\u0E23\u0E40\u0E01\ + \u0E29\u0E15\u0E23 \u0E41\u0E25\u0E30\u0E2D\u0E38\u0E15\u0E2A\u0E32\u0E2B\u0E01\ + \u0E23\u0E23\u0E21\u0E15\u0E48\u0E32\u0E07\u0E46 \u0E0B\u0E36\u0E48\u0E07\u0E40\ + \u0E23\u0E32 \u0E42\u0E25\u0E01\u0E2B\u0E25\u0E31\u0E07\u0E04\u0E27\u0E2D\u0E19\ + \u0E15\u0E31\u0E21\u0E25\u0E35\u0E1E\u0E40\u0E2B\u0E25\u0E37\u0E2D\u0E2D\u0E22\ + \u0E39\u0E48 \u0E17\u0E33\u0E44\u0E14\u0E49\u0E42\u0E14\u0E22\u0E01\u0E32\u0E23\ + \u0E43\u0E0A\u0E49\u0E1B\u0E23\u0E30\u0E42\u0E22\u0E0A\u0E19\u0E4C\u0E08\u0E32\ + \u0E01\u0E23\u0E30\u0E1A\u0E1A\u0E19\u0E34\u0E40\u0E27\u0E28 \u0E17\u0E23\u0E31\ + \u0E1E\u0E22\u0E32\u0E01\u0E23\u0E41\u0E23\u0E48\u0E18\u0E32\u0E15\u0E38 \u0E41\ + \u0E25\u0E30\u0E17\u0E23\u0E31\u0E1E\u0E22\u0E4C\u0E2A\u0E34\u0E19\u0E02\u0E2D\ + \u0E07\u0E21\u0E19\u0E38\u0E29\u0E22\u0E4C\u0E02\u0E2D\u0E07\u0E17\u0E31\u0E49\ + \u0E07\u0E17\u0E27\u0E35\u0E1B\u0E19\u0E2D\u0E01\u0E14\u0E34\u0E19\u0E41\u0E14\ + \u0E19\u0E02\u0E2D\u0E07\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E40\u0E17\u0E48\u0E32\ + \u0E19\u0E31\u0E49\u0E19\u0E17\u0E35\u0E48\u0E01\u0E23\u0E30\u0E42\u0E14\u0E14\ + \u0E44\u0E14\u0E49 \u0E22\u0E38\u0E42\u0E23\u0E1B\u0E15\u0E30\u0E27\u0E31\u0E19\ + \u0E15\u0E01\u0E17\u0E33\u0E40\u0E0A\u0E48\u0E19\u0E19\u0E31\u0E49\u0E19\u0E14\ + \u0E49\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E42\u0E2B\u0E14\u0E23\u0E49\u0E32\ + \u0E22\u0E41\u0E25\u0E30\u0E1B\u0E37\u0E19\u0E41\u0E25\u0E30 \u0E2A\u0E33\u0E04\ + \u0E31\u0E0D\u0E01\u0E27\u0E48\u0E32\u0E14\u0E49\u0E27\u0E22\u0E42\u0E0A\u0E04\ + \u0E17\u0E32\u0E07\u0E20\u0E39\u0E21\u0E34\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C\ + \u0E41\u0E25\u0E30\u0E23\u0E30\u0E1A\u0E1A\u0E19\u0E34\u0E40\u0E27\u0E28"\ + \ \u0E25\u0E34\u0E02\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C \xA9 2015 Cambridge\ + \ University Press. Alfred Crosby \u0E19\u0E31\u0E01\u0E1B\u0E23\u0E30\u0E27\ + \u0E31\u0E15\u0E34\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C \u0E25\u0E31\u0E17\u0E18\ + \u0E34\u0E08\u0E31\u0E01\u0E23\u0E27\u0E23\u0E23\u0E14\u0E34\u0E19\u0E34\u0E22\ + \u0E21\u0E40\u0E0A\u0E34\u0E07\u0E19\u0E34\u0E40\u0E27\u0E28\u0E19\u0E4C 2004\ + \ "\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E42\u0E14\u0E14\u0E04\u0E27\u0E2D\ + \u0E19\u0E15\u0E31\u0E21" \u0E17\u0E35\u0E48\u0E01\u0E25\u0E48\u0E32\u0E27\ + \u0E16\u0E36\u0E07\u0E43\u0E19\u0E02\u0E49\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E19\ + \u0E35\u0E49\u0E21\u0E35\u0E2A\u0E48\u0E27\u0E19\u0E2A\u0E19\u0E31\u0E1A\u0E2A\ + \u0E19\u0E38\u0E19\u0E42\u0E14\u0E22\u0E15\u0E23\u0E07\u0E21\u0E32\u0E01\u0E17\ + \u0E35\u0E48\u0E2A\u0E38\u0E14\u0E15\u0E48\u0E2D\u0E1E\u0E31\u0E12\u0E19\u0E32\ + \u0E01\u0E32\u0E23\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\ + \u0E19\u0E35\u0E49\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E1B\u0E35 \u0E04.\u0E28\ + . 1450\u20131750" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_high_school_world_history +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_high_school_world_history +task_alias: high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..845ab2e04eeeec1a6230c5d19b4981eac0eef88e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_human_sexuality.yaml @@ -0,0 +1,97 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E44\u0E15\u0E23\u0E21\u0E32\u0E2A\ + \u0E41\u0E23\u0E01" + B: "\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E44\u0E15\u0E23\u0E21\u0E32\u0E2A\ + \u0E17\u0E35\u0E48\u0E2A\u0E2D\u0E07" + C: "\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E44\u0E15\u0E23\u0E21\u0E32\u0E2A\ + \u0E17\u0E35\u0E48\u0E2A\u0E32\u0E21" + D: "\u0E15\u0E25\u0E2D\u0E14\u0E01\u0E32\u0E23\u0E15\u0E31\u0E49\u0E07\u0E04\ + \u0E23\u0E23\u0E20\u0E4C" + input_correct_responses: + - A + input_question: "\u0E2D\u0E32\u0E01\u0E32\u0E23\u0E41\u0E1E\u0E49\u0E17\u0E49\u0E2D\ + \u0E07\u0E21\u0E31\u0E01\u0E40\u0E1B\u0E47\u0E19\u0E1B\u0E31\u0E0D\u0E2B\u0E32\ + :" + - input_choice_list: + A: "\u0E2D\u0E31\u0E15\u0E15\u0E32\u0E17\u0E35\u0E48\u0E41\u0E02\u0E47\u0E07\ + \u0E41\u0E01\u0E23\u0E48\u0E07" + B: "\u0E2B\u0E34\u0E23\u0E34\u0E42\u0E2D\u0E15\u0E15\u0E31\u0E1B\u0E1B\u0E30\ + \u0E17\u0E35\u0E48\u0E2D\u0E48\u0E2D\u0E19\u0E41\u0E2D" + C: "\u0E23\u0E2B\u0E31\u0E2A\u0E17\u0E35\u0E48\u0E2D\u0E48\u0E2D\u0E19\u0E41\ + \u0E2D" + D: "\u0E2B\u0E34\u0E23\u0E34\u0E42\u0E2D\u0E15\u0E15\u0E31\u0E1B\u0E1B\u0E30\ + \u0E17\u0E35\u0E48\u0E41\u0E02\u0E47\u0E07\u0E41\u0E01\u0E23\u0E48\u0E07" + input_correct_responses: + - B + input_question: "\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E17\u0E35\u0E48\u0E23\ + \u0E39\u0E49\u0E27\u0E48\u0E32\u0E15\u0E31\u0E27\u0E40\u0E2D\u0E07\u0E40\u0E1B\ + \u0E47\u0E19\u0E42\u0E23\u0E04\u0E40\u0E23\u0E34\u0E21\u0E41\u0E25\u0E30\u0E0B\ + \u0E34\u0E1F\u0E34\u0E25\u0E34\u0E2A\u0E17\u0E35\u0E48\u0E22\u0E31\u0E07\u0E44\ + \u0E21\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E23\u0E31\ + \u0E01\u0E29\u0E32 \u0E41\u0E15\u0E48\u0E22\u0E31\u0E07\u0E04\u0E07\u0E21\u0E35\ + \u0E40\u0E1E\u0E28\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\u0E18\u0E4C\u0E42\u0E14\ + \u0E22\u0E44\u0E21\u0E48\u0E41\u0E08\u0E49\u0E07\u0E43\u0E2B\u0E49\u0E04\u0E39\ + \u0E48\u0E19\u0E2D\u0E19\u0E17\u0E23\u0E32\u0E1A\u0E16\u0E36\u0E07\u0E2D\u0E32\ + \u0E01\u0E32\u0E23\u0E02\u0E2D\u0E07\u0E40\u0E18\u0E2D \u0E43\u0E19\u0E41\u0E07\ + \u0E48\u0E01\u0E32\u0E23\u0E27\u0E34\u0E40\u0E04\u0E23\u0E32\u0E30\u0E2B\u0E4C\ + \u0E17\u0E32\u0E07\u0E08\u0E34\u0E15\u0E27\u0E34\u0E40\u0E04\u0E23\u0E32\u0E30\ + \u0E2B\u0E4C:" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E08\u0E23\u0E34\u0E07\u0E17\u0E35\u0E48\u0E27\ + \u0E48\u0E32\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E44\u0E21\u0E48\u0E21\u0E35\ + \u0E23\u0E30\u0E22\u0E30\u0E40\u0E27\u0E25\u0E32\u0E17\u0E19\u0E44\u0E1F" + B: "\u0E01\u0E32\u0E23\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E02\u0E2D\ + \u0E07\u0E0A\u0E31\u0E49\u0E19\u0E43\u0E19\u0E02\u0E2D\u0E07\u0E0A\u0E48\u0E2D\ + \u0E07\u0E04\u0E25\u0E2D\u0E14" + C: "\u0E21\u0E35\u0E01\u0E32\u0E23\u0E16\u0E36\u0E07\u0E08\u0E38\u0E14\u0E2A\ + \u0E38\u0E14\u0E22\u0E2D\u0E14\u0E2A\u0E25\u0E31\u0E1A\u0E01\u0E31\u0E19\u0E43\ + \u0E19\u0E17\u0E35\u0E48\u0E15\u0E48\u0E32\u0E07\u0E46" + D: "\u0E08\u0E35-\u0E2A\u0E1B\u0E2D\u0E15" + input_correct_responses: + - A + input_question: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E02\ + \u0E2D\u0E07\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E43\u0E19\u0E01\u0E32\ + \u0E23\u0E16\u0E36\u0E07\u0E08\u0E38\u0E14\u0E2A\u0E38\u0E14\u0E22\u0E2D\u0E14\ + \u0E2B\u0E25\u0E32\u0E22\u0E04\u0E23\u0E31\u0E49\u0E07\u0E21\u0E35\u0E2A\u0E32\ + \u0E40\u0E2B\u0E15\u0E38\u0E2B\u0E25\u0E31\u0E01\u0E21\u0E32\u0E08\u0E32\u0E01\ + :" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E21\u0E35\u0E40\u0E1E\u0E28\u0E2A\u0E31\u0E21\u0E1E\ + \u0E31\u0E19\u0E18\u0E4C" + B: "\u0E27\u0E07\u0E01\u0E25\u0E21\u0E01\u0E23\u0E30\u0E15\u0E38\u0E01" + C: "\u0E01\u0E32\u0E23\u0E41\u0E2A\u0E14\u0E07\u0E2D\u0E2D\u0E01" + D: "\u0E2A\u0E31\u0E21\u0E1C\u0E31\u0E2A\u0E2D\u0E27\u0E31\u0E22\u0E27\u0E30\ + \u0E40\u0E1E\u0E28\u0E02\u0E2D\u0E07\u0E01\u0E31\u0E19\u0E41\u0E25\u0E30\u0E01\ + \u0E31\u0E19" + input_correct_responses: + - A + input_question: "\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\u0E02\u0E2D\u0E07\u0E01\u0E34\ + \u0E08\u0E01\u0E23\u0E23\u0E21\u0E23\u0E31\u0E01\u0E23\u0E48\u0E27\u0E21\u0E40\ + \u0E1E\u0E28\u0E17\u0E35\u0E48\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\ + \u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E01\u0E48\u0E2D\u0E19\u0E27\u0E31\u0E22\ + \u0E23\u0E38\u0E48\u0E19\u0E23\u0E27\u0E21\u0E16\u0E36\u0E07\u0E02\u0E49\u0E2D\ + \u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E2B\u0E25\u0E31\u0E48\u0E07\u0E40\u0E23\u0E47\u0E27" + B: "\u0E22\u0E31\u0E1A\u0E22\u0E31\u0E49\u0E07\u0E01\u0E32\u0E23\u0E2B\u0E25\ + \u0E31\u0E48\u0E07" + C: "\u0E42\u0E23\u0E04\u0E2B\u0E22\u0E48\u0E2D\u0E19\u0E2A\u0E21\u0E23\u0E23\ + \u0E16\u0E20\u0E32\u0E1E\u0E17\u0E32\u0E07\u0E40\u0E1E\u0E28" + D: "\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E1B\u0E01\u0E15\u0E34\u0E02\ + \u0E2D\u0E07\u0E01\u0E32\u0E23\u0E2B\u0E25\u0E31\u0E48\u0E07\u0E19\u0E49\u0E33\ + \u0E2D\u0E2A\u0E38\u0E08\u0E34" + input_correct_responses: + - C + input_question: "\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E1B\u0E01\u0E15\u0E34\ + \u0E17\u0E35\u0E48\u0E1E\u0E1A\u0E1A\u0E48\u0E2D\u0E22\u0E17\u0E35\u0E48\u0E2A\ + \u0E38\u0E14\u0E43\u0E19\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22\u0E17\u0E35\u0E48\ + \u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E01\u0E32\u0E23\u0E1A\u0E33\u0E1A\ + \u0E31\u0E14\u0E17\u0E32\u0E07\u0E40\u0E1E\u0E28\u0E04\u0E37\u0E2D:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_human_sexuality +tag: mmlu_th_llama_social_sciences_tasks +task: mmlu_th_llama_human_sexuality +task_alias: human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77ebb472f976149f0ee4629a2fc424c1fc5efff2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_international_law.yaml @@ -0,0 +1,183 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E21\u0E32\u0E15\u0E23\u0E32 2(4) \u0E2B\u0E21\u0E32\u0E22\u0E04\u0E27\ + \u0E32\u0E21\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E01\u0E2D\u0E07\u0E01\u0E33\u0E25\ + \u0E31\u0E07\u0E15\u0E34\u0E14\u0E2D\u0E32\u0E27\u0E38\u0E18" + B: "\u0E21\u0E32\u0E15\u0E23\u0E32 2(4) \u0E04\u0E23\u0E2D\u0E1A\u0E04\u0E25\ + \u0E38\u0E21\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E01\u0E33\u0E25\u0E31\u0E07\ + \u0E17\u0E38\u0E01\u0E1B\u0E23\u0E30\u0E40\u0E20\u0E17 \u0E23\u0E27\u0E21\u0E17\ + \u0E31\u0E49\u0E07\u0E01\u0E32\u0E23\u0E04\u0E27\u0E48\u0E33\u0E1A\u0E32\u0E15\ + \u0E23" + C: "\u0E02\u0E49\u0E2D 2(4) \u0E04\u0E23\u0E2D\u0E1A\u0E04\u0E25\u0E38\u0E21\ + \u0E16\u0E36\u0E07\u0E01\u0E32\u0E23\u0E41\u0E17\u0E23\u0E01\u0E41\u0E0B\u0E07\ + \u0E01\u0E34\u0E08\u0E01\u0E32\u0E23\u0E20\u0E32\u0E22\u0E43\u0E19\u0E1B\u0E23\ + \u0E30\u0E40\u0E17\u0E28\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E17\u0E31\u0E49\ + \u0E07\u0E2B\u0E21\u0E14" + D: "\u0E02\u0E49\u0E2D 2(4) \u0E2B\u0E21\u0E32\u0E22\u0E04\u0E27\u0E32\u0E21\ + \u0E23\u0E27\u0E21\u0E16\u0E36\u0E07\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E01\ + \u0E33\u0E25\u0E31\u0E07\u0E17\u0E35\u0E48\u0E01\u0E23\u0E30\u0E17\u0E33\u0E15\ + \u0E48\u0E2D\u0E1A\u0E39\u0E23\u0E13\u0E20\u0E32\u0E1E\u0E41\u0E2B\u0E48\u0E07\ + \u0E14\u0E34\u0E19\u0E41\u0E14\u0E19\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E40\ + \u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + input_correct_responses: + - A + input_question: "\u0E02\u0E49\u0E2D 2(4) \u0E02\u0E2D\u0E07\u0E01\u0E0E\u0E1A\u0E31\ + \u0E15\u0E23\u0E2A\u0E2B\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E0A\u0E32\u0E15\u0E34\ + \u0E2B\u0E49\u0E32\u0E21\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E01\u0E33\u0E25\ + \u0E31\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E20\u0E17\u0E43\u0E14" + - input_choice_list: + A: "\u0E2B\u0E32\u0E01\u0E04\u0E39\u0E48\u0E04\u0E27\u0E32\u0E21\u0E43\u0E19\ + \u0E04\u0E14\u0E35\u0E17\u0E35\u0E48\u0E16\u0E01\u0E40\u0E16\u0E35\u0E22\u0E07\ + \u0E01\u0E31\u0E19\u0E43\u0E19\u0E28\u0E32\u0E25\u0E42\u0E25\u0E01\u0E44\u0E21\ + \u0E48\u0E21\u0E35\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\ + \u0E43\u0E19\u0E23\u0E30\u0E14\u0E31\u0E1A\u0E0A\u0E32\u0E15\u0E34 \u0E21\u0E35\ + \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E40\u0E2A\u0E19\u0E2D\u0E0A\u0E37\u0E48\u0E2D\ + \u0E1A\u0E38\u0E04\u0E04\u0E25\u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E39\u0E49\u0E1E\ + \u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E04\u0E14\ + \u0E35\u0E19\u0E31\u0E49\u0E19\u0E42\u0E14\u0E22\u0E43\u0E2B\u0E49\u0E14\u0E33\ + \u0E23\u0E07\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07\u0E1C\u0E39\u0E49\u0E1E\ + \u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E01\u0E34\ + \u0E08" + B: "\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E40\u0E09\ + \u0E1E\u0E32\u0E30\u0E01\u0E34\u0E08\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E07\u0E04\ + \u0E4C\u0E04\u0E13\u0E30\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\ + \u0E32\u0E28\u0E32\u0E25\u0E42\u0E25\u0E01\u0E14\u0E49\u0E27\u0E22\u0E04\u0E30\ + \u0E41\u0E19\u0E19\u0E40\u0E2A\u0E35\u0E22\u0E07\u0E0A\u0E35\u0E49\u0E02\u0E32\ + \u0E14" + C: "\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E40\u0E09\ + \u0E1E\u0E32\u0E30\u0E01\u0E34\u0E08\u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E39\u0E49\ + \u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E41\u0E17\u0E19\u0E43\u0E19\u0E01\ + \u0E23\u0E13\u0E35\u0E17\u0E35\u0E48\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\ + \u0E01\u0E29\u0E32\u0E02\u0E32\u0E14\u0E04\u0E38\u0E13\u0E2A\u0E21\u0E1A\u0E31\ + \u0E15\u0E34\u0E2B\u0E23\u0E37\u0E2D\u0E16\u0E36\u0E07\u0E41\u0E01\u0E48\u0E01\ + \u0E23\u0E23\u0E21" + D: "\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E40\u0E09\ + \u0E1E\u0E32\u0E30\u0E01\u0E34\u0E08 \u0E04\u0E37\u0E2D \u0E1C\u0E39\u0E49\ + \u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E17\u0E35\u0E48\u0E41\u0E15\u0E48\ + \u0E25\u0E30\u0E1D\u0E48\u0E32\u0E22\u0E08\u0E30\u0E40\u0E2A\u0E19\u0E2D\u0E0A\ + \u0E37\u0E48\u0E2D\u0E40\u0E2A\u0E21\u0E2D\u0E43\u0E19\u0E17\u0E38\u0E01\u0E04\ + \u0E14\u0E35\u0E04\u0E27\u0E32\u0E21" + input_correct_responses: + - A + input_question: "\u0E1C\u0E39\u0E49\u0E1E\u0E34\u0E1E\u0E32\u0E01\u0E29\u0E32\u0E40\ + \u0E09\u0E1E\u0E32\u0E30\u0E01\u0E34\u0E08 \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\ + \u0E23?" + - input_choice_list: + A: "\u0E19\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E32\u0E23\u0E08\u0E2D\ + \u0E07\u0E17\u0E35\u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E44\u0E14\u0E49\ + \u0E2B\u0E32\u0E01\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E02\u0E2D\u0E07\u0E1B\ + \u0E23\u0E30\u0E40\u0E17\u0E28\u0E17\u0E35\u0E48\u0E08\u0E2D\u0E07\u0E43\u0E0A\ + \u0E49\u0E04\u0E33\u0E08\u0E33\u0E01\u0E31\u0E14\u0E04\u0E27\u0E32\u0E21\u0E2D\ + \u0E37\u0E48\u0E19" + B: "\u0E19\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E02\u0E49\u0E2D\u0E2A\u0E07\ + \u0E27\u0E19\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\ + \u0E16\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E44\u0E14\u0E49 \u0E40\u0E19\u0E37\ + \u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E32\u0E23\ + \u0E1D\u0E48\u0E32\u0E1D\u0E37\u0E19\u0E27\u0E31\u0E15\u0E16\u0E38\u0E1B\u0E23\ + \u0E30\u0E2A\u0E07\u0E04\u0E4C\u0E41\u0E25\u0E30\u0E27\u0E31\u0E15\u0E16\u0E38\ + \u0E1B\u0E23\u0E30\u0E2A\u0E07\u0E04\u0E4C\u0E02\u0E2D\u0E07 ICCPR" + C: "\u0E19\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E02\u0E49\u0E2D\u0E2A\u0E07\ + \u0E27\u0E19\u0E17\u0E35\u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E44\u0E21\ + \u0E48\u0E44\u0E14\u0E49 \u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\ + \u0E04\u0E33\u0E08\u0E33\u0E01\u0E31\u0E14\u0E04\u0E27\u0E32\u0E21\u0E02\u0E2D\ + \u0E07\u0E01\u0E32\u0E23\u0E17\u0E23\u0E21\u0E32\u0E19\u0E43\u0E19 ICCPR \u0E19\ + \u0E31\u0E49\u0E19\u0E2A\u0E2D\u0E14\u0E04\u0E25\u0E49\u0E2D\u0E07\u0E01\u0E31\ + \u0E1A\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E08\u0E32\u0E23\u0E35\u0E15\u0E1B\ + \u0E23\u0E30\u0E40\u0E1E\u0E13\u0E35\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\ + \u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28" + D: "\u0E19\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E02\u0E49\u0E2D\u0E2A\u0E07\ + \u0E27\u0E19\u0E17\u0E35\u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E44\u0E14\ + \u0E49\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E20\u0E32\u0E22\ + \u0E43\u0E15\u0E49\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22\u0E23\u0E30\u0E2B\u0E27\ + \u0E48\u0E32\u0E07\u0E1B\u0E23\u0E30\u0E40\u0E17\u0E28\u0E17\u0E31\u0E48\u0E27\ + \u0E44\u0E1B \u0E23\u0E31\u0E10\u0E21\u0E35\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E17\ + \u0E35\u0E48\u0E08\u0E30\u0E40\u0E02\u0E49\u0E32\u0E2A\u0E39\u0E48\u0E02\u0E49\ + \u0E2D\u0E2A\u0E07\u0E27\u0E19\u0E43\u0E19\u0E2A\u0E19\u0E18\u0E34\u0E2A\u0E31\ + \u0E0D\u0E0D\u0E32" + input_correct_responses: + - B + input_question: "\u0E01\u0E32\u0E23\u0E2A\u0E07\u0E27\u0E19\u0E04\u0E33\u0E08\u0E33\ + \u0E01\u0E31\u0E14\u0E04\u0E27\u0E32\u0E21\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\ + \u0E17\u0E23\u0E21\u0E32\u0E19\u0E43\u0E19 ICCPR \u0E08\u0E30\u0E40\u0E1B\u0E47\ + \u0E19\u0E17\u0E35\u0E48\u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E43\u0E19\u0E41\ + \u0E19\u0E27\u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E23\u0E48\u0E27\u0E21\ + \u0E2A\u0E21\u0E31\u0E22\u0E2B\u0E23\u0E37\u0E2D\u0E44\u0E21\u0E48?" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E2A\u0E32\ + \u0E21\u0E32\u0E23\u0E16\u0E43\u0E0A\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E24\ + \u0E15\u0E34\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E1B\u0E23\u0E32\ + \u0E28\u0E08\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E40\u0E21\ + \u0E37\u0E48\u0E2D\u0E43\u0E14\u0E01\u0E47\u0E15\u0E32\u0E21\u0E17\u0E35\u0E48\ + \u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A" + B: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E44\u0E21\ + \u0E48\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E43\u0E0A\u0E49\u0E40\u0E1B\u0E47\ + \u0E19\u0E1E\u0E24\u0E15\u0E34\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\ + \u0E02\u0E31\u0E14\u0E02\u0E27\u0E32\u0E07\u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\ + \u0E17\u0E33\u0E1C\u0E34\u0E14\u0E44\u0E14\u0E49" + C: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E2A\u0E32\ + \u0E21\u0E32\u0E23\u0E16\u0E43\u0E0A\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E24\ + \u0E15\u0E34\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E1B\u0E23\u0E32\ + \u0E28\u0E08\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E44\u0E14\ + \u0E49 \u0E2B\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\ + \u0E21\u0E21\u0E35\u0E1C\u0E25\u0E2A\u0E21\u0E1A\u0E39\u0E23\u0E13\u0E4C\u0E41\ + \u0E25\u0E30\u0E43\u0E19\u0E02\u0E2D\u0E1A\u0E40\u0E02\u0E15\u0E17\u0E35\u0E48\ + \u0E01\u0E32\u0E23\u0E01\u0E23\u0E30\u0E17\u0E33\u0E19\u0E31\u0E49\u0E19\u0E22\ + \u0E31\u0E07\u0E04\u0E07\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E02\u0E2D\u0E1A\ + \u0E40\u0E02\u0E15\u0E02\u0E2D\u0E07\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\ + \u0E22\u0E2D\u0E21\u0E17\u0E35\u0E48\u0E43\u0E2B\u0E49\u0E44\u0E27\u0E49" + D: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E2A\u0E32\ + \u0E21\u0E32\u0E23\u0E16\u0E43\u0E0A\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E1E\u0E24\ + \u0E15\u0E34\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E1B\u0E23\u0E32\ + \u0E28\u0E08\u0E32\u0E01\u0E04\u0E27\u0E32\u0E21\u0E40\u0E02\u0E49\u0E32\u0E43\ + \u0E08\u0E1C\u0E34\u0E14\u0E44\u0E14\u0E49\u0E40\u0E2A\u0E21\u0E2D \u0E44\u0E21\ + \u0E48\u0E27\u0E48\u0E32\u0E2B\u0E19\u0E48\u0E27\u0E22\u0E07\u0E32\u0E19\u0E43\ + \u0E14\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E08\u0E30\u0E43\u0E2B\u0E49\u0E04\ + \u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E01\u0E47\u0E15\u0E32\ + \u0E21" + input_correct_responses: + - C + input_question: "\u0E40\u0E21\u0E37\u0E48\u0E2D '\u0E04\u0E27\u0E32\u0E21\u0E22\ + \u0E34\u0E19\u0E22\u0E2D\u0E21' \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E17\ + \u0E33\u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E1E\ + \u0E24\u0E15\u0E34\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48\u0E02\u0E31\ + \u0E14\u0E02\u0E27\u0E32\u0E07\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E1E\u0E24\ + \u0E15\u0E34\u0E21\u0E34\u0E0A\u0E2D\u0E1A\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\ + ?" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E08\u0E30\u0E1C\u0E39\u0E01\u0E1E\ + \u0E31\u0E19\u0E08\u0E30\u0E41\u0E2A\u0E14\u0E07\u0E42\u0E14\u0E22\u0E01\u0E32\ + \u0E23\u0E43\u0E2B\u0E49\u0E2A\u0E31\u0E15\u0E22\u0E32\u0E1A\u0E31\u0E19\u0E40\ + \u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19" + B: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E08\u0E30\u0E1C\u0E39\u0E01\u0E1E\ + \u0E31\u0E19\u0E15\u0E32\u0E21\u0E2A\u0E19\u0E18\u0E34\u0E2A\u0E31\u0E0D\u0E0D\ + \u0E32\u0E2D\u0E32\u0E08\u0E41\u0E2A\u0E14\u0E07\u0E2D\u0E2D\u0E01\u0E42\u0E14\ + \u0E22\u0E01\u0E32\u0E23\u0E25\u0E07\u0E19\u0E32\u0E21 \u0E01\u0E32\u0E23\u0E43\ + \u0E2B\u0E49\u0E2A\u0E31\u0E15\u0E22\u0E32\u0E1A\u0E31\u0E19 \u0E01\u0E32\u0E23\ + \u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A \u0E01\u0E32\u0E23\u0E2D\u0E19\u0E38\u0E21\ + \u0E31\u0E15\u0E34 \u0E2B\u0E23\u0E37\u0E2D\u0E20\u0E32\u0E04\u0E22\u0E32\u0E19\ + \u0E38\u0E27\u0E31\u0E15\u0E34" + C: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E08\u0E30\u0E1C\u0E39\u0E01\u0E1E\ + \u0E31\u0E19\u0E08\u0E30\u0E41\u0E2A\u0E14\u0E07\u0E42\u0E14\u0E22\u0E25\u0E32\ + \u0E22\u0E40\u0E0B\u0E47\u0E19" + D: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E17\u0E35\u0E48\u0E08\u0E30\u0E1C\u0E39\u0E01\u0E1E\ + \u0E31\u0E19\u0E19\u0E31\u0E49\u0E19\u0E41\u0E2A\u0E14\u0E07\u0E2D\u0E2D\u0E01\ + \u0E14\u0E49\u0E27\u0E22\u0E27\u0E34\u0E18\u0E35\u0E43\u0E14\u0E01\u0E47\u0E15\ + \u0E32\u0E21\u0E17\u0E35\u0E48\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32\u0E40\u0E25\ + \u0E37\u0E2D\u0E01" + input_correct_responses: + - B + input_question: "\u0E04\u0E27\u0E32\u0E21\u0E22\u0E34\u0E19\u0E22\u0E2D\u0E21\u0E17\ + \u0E35\u0E48\u0E08\u0E30\u0E1C\u0E39\u0E01\u0E1E\u0E31\u0E19\u0E02\u0E2D\u0E07\ + \u0E23\u0E31\u0E10\u0E08\u0E30\u0E41\u0E2A\u0E14\u0E07\u0E2D\u0E22\u0E48\u0E32\ + \u0E07\u0E44\u0E23?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_international_law +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_international_law +task_alias: international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f20584fc1fc63ce26244a0241691c5431afc88a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_logical_fallacies.yaml @@ -0,0 +1,124 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E04\u0E27\u0E2D\u0E01" + B: "\u0E21\u0E49\u0E32\u0E2B\u0E31\u0E27\u0E40\u0E23\u0E32\u0E30" + C: "\u0E01\u0E32\u0E23\u0E42\u0E15\u0E49\u0E40\u0E16\u0E35\u0E22\u0E07\u0E01\ + \u0E31\u0E1A\u0E1A\u0E38\u0E04\u0E04\u0E25" + D: "\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E23\u0E39\u0E49 elenchi" + input_correct_responses: + - C + input_question: "\u0E2B\u0E32\u0E01\u0E21\u0E35\u0E43\u0E04\u0E23\u0E42\u0E08\u0E21\ + \u0E15\u0E35\u0E25\u0E31\u0E01\u0E29\u0E13\u0E30\u0E02\u0E2D\u0E07\u0E1C\u0E39\ + \u0E49\u0E42\u0E15\u0E49\u0E40\u0E16\u0E35\u0E22\u0E07\u0E1D\u0E48\u0E32\u0E22\ + \u0E15\u0E23\u0E07\u0E02\u0E49\u0E32\u0E21 \u0E41\u0E17\u0E19\u0E17\u0E35\u0E48\ + \u0E08\u0E30\u0E15\u0E2D\u0E1A\u0E2A\u0E19\u0E2D\u0E07\u0E15\u0E48\u0E2D\u0E02\ + \u0E49\u0E2D\u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\u0E02\u0E2D\u0E07\u0E1D\ + \u0E48\u0E32\u0E22\u0E15\u0E23\u0E07\u0E02\u0E49\u0E32\u0E21 \u0E1A\u0E38\u0E04\ + \u0E04\u0E25\u0E41\u0E23\u0E01\u0E2D\u0E32\u0E08\u0E17\u0E33\u0E1C\u0E34\u0E14\ + \u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E42\u0E15\u0E49\u0E40\u0E16\u0E35\u0E22\u0E07\u0E43\ + \u0E19\u0E1A\u0E32\u0E07\u0E2A\u0E34\u0E48\u0E07\u0E19\u0E31\u0E49\u0E19\u0E14\ + \u0E49\u0E2D\u0E22\u0E01\u0E27\u0E48\u0E32\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E40\ + \u0E1E\u0E23\u0E32\u0E30\u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E17\u0E33\u0E2A\ + \u0E34\u0E48\u0E07\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E15\ + \u0E31\u0E49\u0E07\u0E43\u0E08\u0E08\u0E30\u0E17\u0E33" + B: "\u0E23\u0E27\u0E21\u0E16\u0E36\u0E07\u0E01\u0E32\u0E23\u0E2D\u0E49\u0E32\ + \u0E07\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\ + \u0E32\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E02\u0E49\u0E2D\u0E43\u0E19\u0E02\u0E49\ + \u0E2D\u0E40\u0E2A\u0E19\u0E2D\u0E41\u0E25\u0E30\u0E16\u0E37\u0E2D\u0E27\u0E48\ + \u0E32\u0E01\u0E32\u0E23\u0E1E\u0E34\u0E2A\u0E39\u0E08\u0E19\u0E4C\u0E2A\u0E33\ + \u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2D\u0E49\u0E32\u0E07\u0E2A\u0E34\ + \u0E17\u0E18\u0E34\u0E4C\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E04\u0E23\u0E31\u0E49\ + \u0E07\u0E40\u0E1B\u0E47\u0E19\u0E2B\u0E25\u0E31\u0E01\u0E10\u0E32\u0E19\u0E2A\ + \u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E2D\u0E49\u0E32\u0E07\u0E2A\ + \u0E34\u0E17\u0E18\u0E34\u0E4C\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14" + C: "\u0E2A\u0E23\u0E38\u0E1B\u0E1C\u0E25\u0E01\u0E48\u0E2D\u0E19\u0E17\u0E35\ + \u0E48\u0E08\u0E30\u0E15\u0E23\u0E27\u0E08\u0E2A\u0E2D\u0E1A\u0E2B\u0E25\u0E31\ + \u0E01\u0E10\u0E32\u0E19\u0E41\u0E25\u0E30\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\ + \u0E32\u0E40\u0E09\u0E1E\u0E32\u0E30\u0E2B\u0E25\u0E31\u0E01\u0E10\u0E32\u0E19\ + \u0E17\u0E35\u0E48\u0E2A\u0E19\u0E31\u0E1A\u0E2A\u0E19\u0E38\u0E19\u0E02\u0E49\ + \u0E2D\u0E2A\u0E23\u0E38\u0E1B\u0E19\u0E31\u0E49\u0E19" + D: "\u0E01\u0E32\u0E23\u0E16\u0E32\u0E21\u0E04\u0E33\u0E16\u0E32\u0E21\u0E17\ + \u0E35\u0E48\u0E21\u0E35\u0E17\u0E31\u0E49\u0E07\u0E02\u0E49\u0E2D\u0E2A\u0E31\ + \u0E19\u0E19\u0E34\u0E29\u0E10\u0E32\u0E19\u0E17\u0E35\u0E48\u0E22\u0E31\u0E07\ + \u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E1E\ + \u0E34\u0E2A\u0E39\u0E08\u0E19\u0E4C\u0E2B\u0E23\u0E37\u0E2D\u0E04\u0E33\u0E16\ + \u0E32\u0E21\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32\u0E2B\u0E19\u0E36\u0E48\ + \u0E07\u0E02\u0E49\u0E2D \u0E08\u0E36\u0E07\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E04\ + \u0E33\u0E15\u0E2D\u0E1A\u0E17\u0E35\u0E48\u0E43\u0E0A\u0E48\u0E2B\u0E23\u0E37\ + \u0E2D\u0E44\u0E21\u0E48\u0E43\u0E0A\u0E48\u0E15\u0E23\u0E07\u0E44\u0E1B\u0E15\ + \u0E23\u0E07\u0E21\u0E32\u0E44\u0E21\u0E48\u0E21\u0E35\u0E04\u0E27\u0E32\u0E21\ + \u0E2B\u0E21\u0E32\u0E22" + input_correct_responses: + - D + input_question: "\u0E01\u0E32\u0E23\u0E40\u0E02\u0E49\u0E32\u0E43\u0E08\u0E1C\u0E34\ + \u0E14\u0E02\u0E2D\u0E07\u0E04\u0E33\u0E16\u0E32\u0E21\u0E17\u0E35\u0E48\u0E0B\ + \u0E31\u0E1A\u0E0B\u0E49\u0E2D\u0E19\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A\u0E14\ + \u0E49\u0E27\u0E22" + - input_choice_list: + A: "\u0E2B\u0E25\u0E31\u0E01\u0E10\u0E32\u0E19\u0E23\u0E2D\u0E07\u0E08\u0E30\ + \u0E15\u0E49\u0E2D\u0E07\u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18\u0E2A\u0E34\u0E48\ + \u0E07\u0E17\u0E35\u0E48\u0E21\u0E32\u0E01\u0E48\u0E2D\u0E19" + B: "\u0E2B\u0E25\u0E31\u0E01\u0E10\u0E32\u0E19\u0E2A\u0E33\u0E04\u0E31\u0E0D\ + \u0E15\u0E49\u0E2D\u0E07\u0E22\u0E37\u0E19\u0E22\u0E31\u0E19\u0E1C\u0E25\u0E17\ + \u0E35\u0E48\u0E15\u0E32\u0E21\u0E21\u0E32" + C: "\u0E04\u0E33\u0E01\u0E25\u0E32\u0E07\u0E08\u0E30\u0E15\u0E49\u0E2D\u0E07\ + \u0E43\u0E0A\u0E49\u0E43\u0E19\u0E2A\u0E16\u0E32\u0E19\u0E17\u0E35\u0E48\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E19\u0E49\u0E2D\u0E22\u0E2B\u0E19\u0E36\u0E48\u0E07\ + \u0E41\u0E2B\u0E48\u0E07\u0E43\u0E19\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E21\u0E32\ + \u0E22\u0E17\u0E35\u0E48\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E32\u0E01\u0E25\u0E2B\ + \u0E23\u0E37\u0E2D\u0E44\u0E21\u0E48\u0E21\u0E35\u0E40\u0E07\u0E37\u0E48\u0E2D\ + \u0E19\u0E44\u0E02" + D: "\u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\u0E02\u0E49\u0E32\u0E07\u0E15\ + \u0E49\u0E19" + input_correct_responses: + - C + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E04\u0E27\u0E32\u0E21\u0E08\u0E23\u0E34\ + \u0E07\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E43\u0E0A\u0E49\u0E42\u0E27\u0E2B\ + \u0E32\u0E23\u0E41\u0E1A\u0E1A\u0E2B\u0E21\u0E27\u0E14\u0E2B\u0E21\u0E39\u0E48\ + \u0E17\u0E35\u0E48\u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07" + - input_choice_list: + A: "\u0E41\u0E1C\u0E19\u0E01" + B: "\u0E2D\u0E07\u0E04\u0E4C\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A" + C: "\u0E2D\u0E38\u0E17\u0E18\u0E23\u0E13\u0E4C\u0E15\u0E48\u0E2D\u0E1A\u0E38\ + \u0E04\u0E04\u0E25" + D: "\u0E2D\u0E38\u0E17\u0E18\u0E23\u0E13\u0E4C\u0E15\u0E48\u0E2D\u0E04\u0E27\ + \u0E32\u0E21\u0E44\u0E21\u0E48\u0E23\u0E39\u0E49" + input_correct_responses: + - B + input_question: "\u0E01\u0E32\u0E23\u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\u0E27\ + \u0E48\u0E32\u0E2A\u0E48\u0E27\u0E19\u0E44\u0E2B\u0E19\u0E08\u0E23\u0E34\u0E07\ + \u0E15\u0E49\u0E2D\u0E07\u0E08\u0E23\u0E34\u0E07\u0E17\u0E31\u0E49\u0E07\u0E2A\ + \u0E48\u0E27\u0E19\u0E04\u0E37\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E40\u0E02\u0E49\ + \u0E32\u0E43\u0E08\u0E1C\u0E34\u0E14\u0E02\u0E2D\u0E07..." + - input_choice_list: + A: "\u0E19\u0E49\u0E33\u0E43\u0E08\u0E19\u0E31\u0E01\u0E01\u0E35\u0E2C\u0E32\ + \u0E44\u0E21\u0E48\u0E14\u0E35" + B: "\u0E40\u0E23\u0E35\u0E22\u0E01\u0E23\u0E49\u0E2D\u0E07\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E2B\u0E47\u0E19\u0E2D\u0E01\u0E40\u0E2B\u0E47\u0E19\u0E43\u0E08" + C: "\u0E01\u0E32\u0E23\u0E42\u0E15\u0E49\u0E40\u0E16\u0E35\u0E22\u0E07\u0E01\ + \u0E31\u0E1A\u0E1A\u0E38\u0E04\u0E04\u0E25" + D: "\u0E04\u0E27\u0E32\u0E21\u0E44\u0E21\u0E48\u0E23\u0E39\u0E49\u0E02\u0E2D\ + \u0E07\u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E40\u0E2A\u0E18" + input_correct_responses: + - D + input_question: "\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E1C\u0E39\u0E49\u0E42\u0E15\u0E49\ + \u0E40\u0E16\u0E35\u0E22\u0E07\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E40\u0E01\u0E34\ + \u0E14\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E1A\u0E2A\u0E19\u0E23\u0E30\u0E2B\ + \u0E27\u0E48\u0E32\u0E07\u0E01\u0E32\u0E23\u0E2B\u0E31\u0E01\u0E25\u0E49\u0E32\ + \u0E07\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E02\u0E32\u0E14\ + \u0E04\u0E27\u0E32\u0E21\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E43\u0E19\u0E01\ + \u0E32\u0E23\u0E42\u0E15\u0E49\u0E41\u0E22\u0E49\u0E07\u0E08\u0E23\u0E34\u0E07\ + \u0E2B\u0E23\u0E37\u0E2D\u0E1B\u0E25\u0E2D\u0E21 \u0E1C\u0E39\u0E49\u0E42\u0E15\ + \u0E49\u0E40\u0E16\u0E35\u0E22\u0E07\u0E04\u0E19\u0E19\u0E31\u0E49\u0E19\u0E2D\ + \u0E32\u0E08\u0E40\u0E02\u0E49\u0E32\u0E43\u0E08\u0E1C\u0E34\u0E14\u0E27\u0E48\ + \u0E32" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_logical_fallacies +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_logical_fallacies +task_alias: logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..229273fbaac1037699f94c9387ebbc827e982934 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_machine_learning.yaml @@ -0,0 +1,173 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: 2.0/15 + B: 1.0/7 + C: 3.0/16 + D: 1.0/5 + input_correct_responses: + - B + input_question: "\u0E17\u0E2D\u0E22\u0E25\u0E39\u0E01\u0E40\u0E15\u0E4B\u0E32\ + \ 6 \u0E14\u0E49\u0E32\u0E19 15 \u0E04\u0E23\u0E31\u0E49\u0E07 \u0E1C\u0E25\u0E25\ + \u0E31\u0E1E\u0E18\u0E4C\u0E17\u0E35\u0E48\u0E44\u0E14\u0E49\u0E04\u0E37\u0E2D\ + \ \u0E14\u0E49\u0E32\u0E19 1 \u0E02\u0E36\u0E49\u0E19\u0E21\u0E32 0 \u0E04\u0E23\ + \u0E31\u0E49\u0E07; \u0E14\u0E49\u0E32\u0E19 2: 1 \u0E04\u0E23\u0E31\u0E49\u0E07\ + ; \u0E14\u0E49\u0E32\u0E19 3: 2 \u0E04\u0E23\u0E31\u0E49\u0E07; \u0E14\u0E49\ + \u0E32\u0E19 4: 3 \u0E04\u0E23\u0E31\u0E49\u0E07; \u0E14\u0E49\u0E32\u0E19 5:\ + \ 4 \u0E04\u0E23\u0E31\u0E49\u0E07; \u0E14\u0E49\u0E32\u0E19 6: 5 \u0E04\u0E23\ + \u0E31\u0E49\u0E07 \u0E08\u0E32\u0E01\u0E1C\u0E25\u0E25\u0E31\u0E1E\u0E18\u0E4C\ + \u0E40\u0E2B\u0E25\u0E48\u0E32\u0E19\u0E35\u0E49 \u0E04\u0E27\u0E32\u0E21\u0E19\ + \u0E48\u0E32\u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E35\u0E48\u0E14\u0E49\ + \u0E32\u0E19 3 \u0E08\u0E30\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E40\ + \u0E21\u0E37\u0E48\u0E2D\u0E43\u0E0A\u0E49 Add-1 Smoothing \u0E40\u0E1B\u0E47\ + \u0E19\u0E40\u0E17\u0E48\u0E32\u0E43\u0E14" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E04\u0E23\u0E2D\u0E1A\u0E15\u0E31\u0E14\u0E41\u0E1A\ + \u0E1A\u0E2A\u0E38\u0E48\u0E21\u0E41\u0E25\u0E30\u0E01\u0E32\u0E23\u0E1E\u0E25\ + \u0E34\u0E01\u0E41\u0E19\u0E27\u0E19\u0E2D\u0E19" + B: "\u0E01\u0E32\u0E23\u0E04\u0E23\u0E2D\u0E1A\u0E15\u0E31\u0E14\u0E41\u0E1A\ + \u0E1A\u0E2A\u0E38\u0E48\u0E21\u0E41\u0E25\u0E30\u0E01\u0E32\u0E23\u0E1E\u0E25\ + \u0E34\u0E01\u0E41\u0E19\u0E27\u0E15\u0E31\u0E49\u0E07" + C: "\u0E42\u0E1B\u0E2A\u0E40\u0E15\u0E2D\u0E23\u0E4C" + D: "\u0E01\u0E25\u0E37\u0E19\u0E44\u0E21\u0E48\u0E40\u0E02\u0E49\u0E32\u0E04\ + \u0E32\u0E22\u0E44\u0E21\u0E48\u0E2D\u0E2D\u0E01" + input_correct_responses: + - A + input_question: "\u0E01\u0E32\u0E23\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E02\u0E49\u0E2D\ + \u0E21\u0E39\u0E25\u0E20\u0E32\u0E1E\u0E41\u0E1A\u0E1A\u0E43\u0E14\u0E17\u0E35\ + \u0E48\u0E43\u0E0A\u0E49\u0E1A\u0E48\u0E2D\u0E22\u0E17\u0E35\u0E48\u0E2A\u0E38\ + \u0E14\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E20\u0E32\u0E1E\u0E18\u0E23\u0E23\ + \u0E21\u0E0A\u0E32\u0E15\u0E34" + - input_choice_list: + A: "\u0E27\u0E34\u0E18\u0E35\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E21\u0E35\ + \u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\u0E43\u0E19\u0E01\ + \u0E32\u0E23\u0E1D\u0E36\u0E01\u0E15\u0E48\u0E33\u0E01\u0E27\u0E48\u0E32\u0E27\ + \u0E34\u0E18\u0E35\u0E01\u0E48\u0E2D\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E17\u0E31\ + \u0E49\u0E07\u0E2B\u0E21\u0E14!" + B: "\u0E27\u0E34\u0E18\u0E35\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E21\u0E35\ + \u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\u0E43\u0E19\u0E01\ + \u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E15\u0E48\u0E33\u0E01\u0E27\u0E48\ + \u0E32\u0E27\u0E34\u0E18\u0E35\u0E01\u0E48\u0E2D\u0E19\u0E2B\u0E19\u0E49\u0E32\ + \u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14! (\u0E40\u0E0A\u0E34\u0E07\u0E2D\ + \u0E23\u0E23\u0E16: \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E25\u0E37\u0E2D\ + \u0E01\u0E1E\u0E32\u0E23\u0E32\u0E21\u0E34\u0E40\u0E15\u0E2D\u0E23\u0E4C\u0E01\ + \u0E32\u0E23\u0E1B\u0E23\u0E31\u0E1A\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\u0E19\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 \u03BB \u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E25\u0E14\u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\ + \u0E43\u0E19\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A)" + C: "\u0E27\u0E34\u0E18\u0E35\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E21\u0E35\ + \u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\u0E43\u0E19\u0E01\ + \u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E15\u0E48\u0E33\u0E01\u0E27\u0E48\ + \u0E32\u0E27\u0E34\u0E18\u0E35\u0E01\u0E48\u0E2D\u0E19\u0E2B\u0E19\u0E49\u0E32\ + \u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14! (\u0E40\u0E0A\u0E34\u0E07\u0E2D\ + \u0E23\u0E23\u0E16: \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E25\u0E37\u0E2D\ + \u0E01\u0E1E\u0E32\u0E23\u0E32\u0E21\u0E34\u0E40\u0E15\u0E2D\u0E23\u0E4C\u0E01\ + \u0E32\u0E23\u0E1B\u0E23\u0E31\u0E1A\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\u0E19\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19 \u03BB \u0E40\u0E1E\u0E37\u0E48\ + \u0E2D\u0E25\u0E14\u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\ + \u0E02\u0E49\u0E32\u0E21\u0E01\u0E32\u0E23\u0E15\u0E23\u0E27\u0E08\u0E2A\u0E2D\ + \u0E1A)" + D: "\u0E27\u0E34\u0E18\u0E35\u0E01\u0E32\u0E23\u0E02\u0E2D\u0E07\u0E09\u0E31\ + \u0E19\u0E21\u0E35\u0E02\u0E49\u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\ + \u0E43\u0E19\u0E01\u0E32\u0E23\u0E15\u0E23\u0E27\u0E08\u0E2A\u0E2D\u0E1A\u0E04\ + \u0E27\u0E32\u0E21\u0E16\u0E39\u0E01\u0E15\u0E49\u0E2D\u0E07\u0E15\u0E48\u0E33\ + \u0E01\u0E27\u0E48\u0E32\u0E27\u0E34\u0E18\u0E35\u0E01\u0E32\u0E23\u0E01\u0E48\ + \u0E2D\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E19\u0E35\u0E49\u0E17\u0E31\u0E49\u0E07\ + \u0E2B\u0E21\u0E14! (\u0E40\u0E0A\u0E34\u0E07\u0E2D\u0E23\u0E23\u0E16: \u0E40\ + \u0E21\u0E37\u0E48\u0E2D\u0E40\u0E25\u0E37\u0E2D\u0E01\u0E1E\u0E32\u0E23\u0E32\ + \u0E21\u0E34\u0E40\u0E15\u0E2D\u0E23\u0E4C\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E31\ + \u0E1A\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E21\u0E32\u0E15\u0E23\u0E10\ + \u0E32\u0E19 \u03BB \u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E25\u0E14\u0E02\u0E49\ + \u0E2D\u0E1C\u0E34\u0E14\u0E1E\u0E25\u0E32\u0E14\u0E02\u0E49\u0E32\u0E21\u0E01\ + \u0E32\u0E23\u0E15\u0E23\u0E27\u0E08\u0E2A\u0E2D\u0E1A)" + input_correct_responses: + - C + input_question: "\u0E04\u0E38\u0E13\u0E01\u0E33\u0E25\u0E31\u0E07\u0E15\u0E23\u0E27\ + \u0E08\u0E2A\u0E2D\u0E1A\u0E40\u0E2D\u0E01\u0E2A\u0E32\u0E23\u0E2A\u0E33\u0E2B\ + \u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E0A\u0E38\u0E21\u0E41\ + \u0E21\u0E0A\u0E0A\u0E35\u0E19\u0E40\u0E25\u0E34\u0E23\u0E4C\u0E19\u0E19\u0E34\ + \u0E07\u0E17\u0E35\u0E48\u0E22\u0E2D\u0E14\u0E40\u0E22\u0E35\u0E48\u0E22\u0E21\ + \u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E43\u0E19\u0E42\u0E25\u0E01 \u0E41\u0E25\ + \u0E30\u0E04\u0E38\u0E13\u0E40\u0E2B\u0E47\u0E19\u0E01\u0E32\u0E23\u0E2A\u0E48\ + \u0E07\u0E17\u0E35\u0E48\u0E21\u0E35\u0E01\u0E32\u0E23\u0E2D\u0E49\u0E32\u0E07\ + \u0E2A\u0E34\u0E17\u0E18\u0E34\u0E4C\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\u0E35\ + \u0E49 \u0E04\u0E38\u0E13\u0E08\u0E30\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\ + \u0E22\u0E2D\u0E21\u0E23\u0E31\u0E1A\u0E02\u0E49\u0E2D\u0E43\u0E14" + - input_choice_list: + A: "\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13 10 \u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\ + \u0E32\u0E07" + B: "\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13 100 \u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\ + \u0E32\u0E07" + C: "\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07 100 \u0E16\u0E36\u0E07 500 \u0E15\ + \u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07" + D: "\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32 1,000 \u0E15\u0E31\u0E27\u0E2D\ + \u0E22\u0E48\u0E32\u0E07" + input_correct_responses: + - D + input_question: "\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E43\u0E2B\u0E49\u0E44\u0E14\u0E49\ + \u0E04\u0E48\u0E32\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13\u0E01\u0E32\u0E23\u0E2A\ + \u0E39\u0E0D\u0E40\u0E2A\u0E35\u0E22 0/1 \u0E17\u0E35\u0E48\u0E19\u0E49\u0E2D\ + \u0E22\u0E01\u0E27\u0E48\u0E32 1 \u0E40\u0E1B\u0E2D\u0E23\u0E4C\u0E40\u0E0B\u0E47\ + \u0E19\u0E15\u0E4C\u0E02\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E2A\u0E39\u0E0D\u0E40\ + \u0E2A\u0E35\u0E22 0/1 \u0E08\u0E23\u0E34\u0E07 (\u0E14\u0E49\u0E27\u0E22\u0E04\ + \u0E27\u0E32\u0E21\u0E19\u0E48\u0E32\u0E08\u0E30\u0E40\u0E1B\u0E47\u0E19 95%)\ + \ \u0E15\u0E32\u0E21\u0E2D\u0E2A\u0E21\u0E01\u0E32\u0E23\u0E02\u0E2D\u0E07 Hoeffding\ + \ \u0E0A\u0E38\u0E14\u0E17\u0E14\u0E2A\u0E2D\u0E1A IID \u0E15\u0E49\u0E2D\u0E07\ + \u0E21\u0E35\u0E01\u0E35\u0E48\u0E15\u0E31\u0E27\u0E2D\u0E22\u0E48\u0E32\u0E07" + - input_choice_list: + A: "\u0E21\u0E31\u0E19\u0E41\u0E1E\u0E07\u0E40\u0E01\u0E34\u0E19\u0E44\u0E1B\ + \u0E43\u0E19\u0E01\u0E32\u0E23\u0E04\u0E33\u0E19\u0E27\u0E13" + B: "\u0E21\u0E31\u0E19\u0E2D\u0E32\u0E08\u0E08\u0E30\u0E2A\u0E48\u0E07\u0E1C\ + \u0E25\u0E43\u0E2B\u0E49\u0E40\u0E01\u0E34\u0E14\u0E41\u0E1C\u0E19\u0E1C\u0E31\ + \u0E07\u0E01\u0E32\u0E23\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08\u0E17\ + \u0E35\u0E48\u0E44\u0E14\u0E49\u0E04\u0E30\u0E41\u0E19\u0E19\u0E44\u0E21\u0E48\ + \u0E14\u0E35\u0E43\u0E19\u0E0A\u0E38\u0E14\u0E1D\u0E36\u0E01\u0E41\u0E25\u0E30\ + \u0E0A\u0E38\u0E14\u0E17\u0E14\u0E2A\u0E2D\u0E1A" + C: "\u0E21\u0E31\u0E19\u0E2D\u0E32\u0E08\u0E08\u0E30\u0E2A\u0E48\u0E07\u0E1C\ + \u0E25\u0E43\u0E2B\u0E49\u0E40\u0E01\u0E34\u0E14\u0E41\u0E1C\u0E19\u0E1C\u0E31\ + \u0E07\u0E01\u0E32\u0E23\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08\u0E17\ + \u0E35\u0E48\u0E17\u0E33\u0E04\u0E30\u0E41\u0E19\u0E19\u0E44\u0E14\u0E49\u0E14\ + \u0E35\u0E43\u0E19\u0E0A\u0E38\u0E14\u0E01\u0E32\u0E23\u0E1D\u0E36\u0E01\u0E41\ + \u0E15\u0E48\u0E44\u0E14\u0E49\u0E04\u0E30\u0E41\u0E19\u0E19\u0E44\u0E21\u0E48\ + \u0E14\u0E35\u0E43\u0E19\u0E0A\u0E38\u0E14\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\ + \u0E2D\u0E1A" + D: "\u0E21\u0E31\u0E19\u0E2D\u0E32\u0E08\u0E08\u0E30\u0E2A\u0E48\u0E07\u0E1C\ + \u0E25\u0E43\u0E2B\u0E49\u0E40\u0E01\u0E34\u0E14\u0E41\u0E1C\u0E19\u0E1C\u0E31\ + \u0E07\u0E01\u0E32\u0E23\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08\u0E17\ + \u0E35\u0E48\u0E17\u0E33\u0E04\u0E30\u0E41\u0E19\u0E19\u0E44\u0E14\u0E49\u0E14\ + \u0E35\u0E43\u0E19\u0E0A\u0E38\u0E14\u0E01\u0E32\u0E23\u0E17\u0E14\u0E2A\u0E2D\ + \u0E1A \u0E41\u0E15\u0E48\u0E44\u0E14\u0E49\u0E04\u0E30\u0E41\u0E19\u0E19\u0E44\ + \u0E21\u0E48\u0E14\u0E35\u0E43\u0E19\u0E0A\u0E38\u0E14\u0E01\u0E32\u0E23\u0E1D\ + \u0E36\u0E01" + input_correct_responses: + - C + input_question: "\u0E15\u0E32\u0E21\u0E40\u0E19\u0E37\u0E49\u0E2D\u0E1C\u0E49\u0E32\ + \ \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E23\u0E32\u0E21\u0E35\u0E41\u0E2D\u0E15\ + \u0E17\u0E23\u0E34\u0E1A\u0E34\u0E27\u0E15\u0E4C\u0E2D\u0E34\u0E19\u0E1E\u0E38\ + \u0E15\u0E17\u0E35\u0E48\u0E21\u0E35\u0E04\u0E48\u0E32\u0E08\u0E23\u0E34\u0E07\ + \u0E43\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E01\u0E32\u0E23\u0E40\ + \u0E23\u0E35\u0E22\u0E19\u0E23\u0E39\u0E49\u0E41\u0E1C\u0E19\u0E1C\u0E31\u0E07\ + \u0E01\u0E32\u0E23\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08 \u0E40\u0E23\ + \u0E32\u0E08\u0E30\u0E1E\u0E34\u0E08\u0E32\u0E23\u0E13\u0E32\u0E01\u0E32\u0E23\ + \u0E41\u0E22\u0E01\u0E44\u0E1A\u0E19\u0E32\u0E23\u0E35\u0E15\u0E32\u0E21\u0E27\ + \u0E48\u0E32\u0E41\u0E2D\u0E15\u0E17\u0E23\u0E34\u0E1A\u0E34\u0E27\u0E15\u0E4C\ + \u0E19\u0E31\u0E49\u0E19\u0E2A\u0E39\u0E07\u0E2B\u0E23\u0E37\u0E2D\u0E15\u0E48\ + \u0E33\u0E01\u0E27\u0E48\u0E32\u0E40\u0E01\u0E13\u0E11\u0E4C\u0E1A\u0E32\u0E07\ + \u0E2D\u0E22\u0E48\u0E32\u0E07 Pat \u0E41\u0E19\u0E30\u0E19\u0E33\u0E27\u0E48\ + \u0E32\u0E40\u0E23\u0E32\u0E04\u0E27\u0E23\u0E41\u0E1A\u0E48\u0E07\u0E41\u0E1A\ + \u0E1A\u0E2B\u0E25\u0E32\u0E22\u0E17\u0E32\u0E07\u0E14\u0E49\u0E27\u0E22\u0E2B\ + \u0E19\u0E36\u0E48\u0E07\u0E2A\u0E32\u0E02\u0E32\u0E2A\u0E33\u0E2B\u0E23\u0E31\ + \u0E1A\u0E41\u0E15\u0E48\u0E25\u0E30\u0E04\u0E48\u0E32\u0E17\u0E35\u0E48\u0E41\ + \u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E01\u0E31\u0E19\u0E02\u0E2D\u0E07\u0E41\ + \u0E2D\u0E15\u0E17\u0E23\u0E34\u0E1A\u0E34\u0E27\u0E15\u0E4C \u0E08\u0E32\u0E01\ + \u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\u0E14\u0E49\u0E32\u0E19\u0E25\u0E48\u0E32\ + \u0E07 \u0E40\u0E25\u0E37\u0E2D\u0E01\u0E1B\u0E31\u0E0D\u0E2B\u0E32\u0E17\u0E35\ + \u0E48\u0E43\u0E2B\u0E0D\u0E48\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E40\u0E1E\ + \u0E35\u0E22\u0E07\u0E02\u0E49\u0E2D\u0E40\u0E14\u0E35\u0E22\u0E27\u0E15\u0E32\ + \u0E21\u0E04\u0E33\u0E41\u0E19\u0E30\u0E19\u0E33\u0E02\u0E2D\u0E07 Pat:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_machine_learning +tag: mmlu_th_llama_stem_tasks +task: mmlu_th_llama_machine_learning +task_alias: machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba8087d06ddacd53aa3442e0f820c601d0b01216 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_marketing.yaml @@ -0,0 +1,123 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E02\u0E49\u0E2D\u0E21\u0E39\u0E25\u0E17\u0E32\u0E07\u0E20\u0E39\u0E21\ + \u0E34\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C" + B: "\u0E04\u0E27\u0E32\u0E21\u0E41\u0E15\u0E01\u0E15\u0E48\u0E32\u0E07\u0E02\ + \u0E2D\u0E07\u0E1C\u0E25\u0E34\u0E15\u0E20\u0E31\u0E13\u0E11\u0E4C" + C: "\u0E40\u0E21\u0E17\u0E23\u0E34\u0E01\u0E0B\u0E4C ANSOFF" + D: "\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\u0E01\u0E32\u0E23\u0E15\u0E23\u0E32\ + \u0E2A\u0E34\u0E19\u0E04\u0E49\u0E32." + input_correct_responses: + - A + input_question: "_____________ \u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E25\u0E15\u0E32\ + \u0E21\u0E18\u0E23\u0E23\u0E21\u0E0A\u0E32\u0E15\u0E34\u0E40\u0E21\u0E37\u0E48\ + \u0E2D\u0E23\u0E27\u0E21\u0E15\u0E31\u0E27\u0E41\u0E1B\u0E23\u0E14\u0E49\u0E32\ + \u0E19\u0E1B\u0E23\u0E30\u0E0A\u0E32\u0E01\u0E23\u0E28\u0E32\u0E2A\u0E15\u0E23\ + \u0E4C\u0E41\u0E25\u0E30\u0E20\u0E39\u0E21\u0E34\u0E28\u0E32\u0E2A\u0E15\u0E23\ + \u0E4C\u0E40\u0E02\u0E49\u0E32\u0E14\u0E49\u0E27\u0E22\u0E01\u0E31\u0E19" + - input_choice_list: + A: "\u0E2B\u0E19\u0E48\u0E27\u0E22\u0E08\u0E49\u0E32\u0E07." + B: "\u0E28\u0E39\u0E19\u0E22\u0E4C\u0E08\u0E31\u0E14\u0E0B\u0E37\u0E49\u0E2D\ + \u0E08\u0E31\u0E14\u0E08\u0E49\u0E32\u0E07." + C: "\u0E2B\u0E31\u0E27\u0E2B\u0E19\u0E49\u0E32\u0E2B\u0E19\u0E48\u0E27\u0E22\ + \u0E1A\u0E23\u0E34\u0E2B\u0E32\u0E23." + D: "\u0E2B\u0E19\u0E48\u0E27\u0E22\u0E15\u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\ + \u0E08." + input_correct_responses: + - D + input_question: "\u0E43\u0E19\u0E2D\u0E07\u0E04\u0E4C\u0E01\u0E23 \u0E01\u0E25\ + \u0E38\u0E48\u0E21\u0E1A\u0E38\u0E04\u0E04\u0E25\u0E17\u0E35\u0E48\u0E21\u0E35\ + \u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\u0E43\u0E19\u0E01\u0E32\u0E23\u0E15\ + \u0E31\u0E14\u0E2A\u0E34\u0E19\u0E43\u0E08\u0E0B\u0E37\u0E49\u0E2D\u0E40\u0E23\ + \u0E35\u0E22\u0E01\u0E27\u0E48\u0E32 _______________" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E02\ + \u0E36\u0E49\u0E19\u0E2D\u0E22\u0E39\u0E48\u0E01\u0E31\u0E1A\u0E27\u0E31\u0E12\ + \u0E19\u0E18\u0E23\u0E23\u0E21\u0E41\u0E25\u0E30\u0E0A\u0E19\u0E0A\u0E31\u0E49\ + \u0E19\u0E17\u0E32\u0E07\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E14\u0E49\u0E27\u0E22" + B: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E23\ + \u0E30\u0E14\u0E31\u0E1A\u0E25\u0E48\u0E32\u0E07\u0E15\u0E49\u0E2D\u0E07\u0E44\ + \u0E14\u0E49\u0E23\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E1E\u0E36\u0E07\u0E1E\ + \u0E2D\u0E43\u0E08\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E1A\u0E32\u0E07\u0E2A\u0E48\ + \u0E27\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E19\u0E49\ + \u0E2D\u0E22\u0E01\u0E48\u0E2D\u0E19\u0E17\u0E35\u0E48\u0E04\u0E27\u0E32\u0E21\ + \u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E17\u0E35\u0E48\u0E2A\u0E39\u0E07\ + \u0E02\u0E36\u0E49\u0E19\u0E08\u0E30\u0E2A\u0E48\u0E07\u0E1C\u0E25\u0E15\u0E48\ + \u0E2D\u0E1E\u0E24\u0E15\u0E34\u0E01\u0E23\u0E23\u0E21" + C: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E44\ + \u0E21\u0E48\u0E44\u0E14\u0E49\u0E16\u0E39\u0E01\u0E08\u0E31\u0E14\u0E25\u0E33\ + \u0E14\u0E31\u0E1A\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E33\u0E04\u0E31\u0E0D\u0E2B\ + \u0E23\u0E37\u0E2D\u0E08\u0E31\u0E14\u0E40\u0E23\u0E35\u0E22\u0E07\u0E15\u0E32\ + \u0E21\u0E25\u0E33\u0E14\u0E31\u0E1A\u0E43\u0E14\u0E42\u0E14\u0E22\u0E40\u0E09\ + \u0E1E\u0E32\u0E30" + D: "\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E17\ + \u0E35\u0E48\u0E1E\u0E36\u0E07\u0E1E\u0E2D\u0E43\u0E08\u0E04\u0E37\u0E2D\u0E15\ + \u0E31\u0E27\u0E01\u0E23\u0E30\u0E15\u0E38\u0E49\u0E19 \u0E41\u0E25\u0E30\u0E04\ + \u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E43\u0E2B\u0E21\ + \u0E48\u0E08\u0E30\u0E40\u0E01\u0E34\u0E14\u0E02\u0E36\u0E49\u0E19\u0E40\u0E21\ + \u0E37\u0E48\u0E2D\u0E04\u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\ + \u0E23\u0E1B\u0E31\u0E08\u0E08\u0E38\u0E1A\u0E31\u0E19\u0E22\u0E31\u0E07\u0E44\ + \u0E21\u0E48\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E15\u0E2D\ + \u0E1A\u0E2A\u0E19\u0E2D\u0E07" + input_correct_responses: + - B + input_question: "\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E21\u0E21\u0E15\u0E34\u0E10\u0E32\ + \u0E19\u0E43\u0E19\u0E25\u0E33\u0E14\u0E31\u0E1A\u0E02\u0E31\u0E49\u0E19\u0E04\ + \u0E27\u0E32\u0E21\u0E15\u0E49\u0E2D\u0E07\u0E01\u0E32\u0E23\u0E02\u0E2D\u0E07\ + \u0E21\u0E32\u0E2A\u0E42\u0E25\u0E27\u0E4C" + - input_choice_list: + A: "\u0E1C\u0E39\u0E49\u0E1A\u0E23\u0E34\u0E42\u0E20\u0E04\u0E17\u0E35\u0E48\ + \u0E21\u0E35\u0E2D\u0E32\u0E22\u0E38\u0E21\u0E32\u0E01\u0E01\u0E27\u0E48\u0E32\ + \u0E17\u0E35\u0E48\u0E23\u0E39\u0E49\u0E2A\u0E36\u0E01\u0E27\u0E48\u0E32\u0E16\ + \u0E39\u0E01\u0E21\u0E2D\u0E07\u0E02\u0E49\u0E32\u0E21" + B: "\u0E1C\u0E39\u0E49\u0E2B\u0E0D\u0E34\u0E07\u0E17\u0E35\u0E48\u0E41\u0E15\ + \u0E48\u0E07\u0E07\u0E32\u0E19\u0E41\u0E25\u0E49\u0E27\u0E2B\u0E25\u0E32\u0E22\ + \u0E04\u0E19\u0E23\u0E39\u0E49\u0E2A\u0E36\u0E01\u0E27\u0E48\u0E32\u0E15\u0E49\ + \u0E2D\u0E07\u0E01\u0E32\u0E23\u0E04\u0E27\u0E32\u0E21\u0E21\u0E31\u0E48\u0E19\ + \u0E04\u0E07\u0E43\u0E19\u0E0A\u0E35\u0E27\u0E34\u0E15" + C: "\u0E1C\u0E39\u0E49\u0E22\u0E49\u0E32\u0E22\u0E16\u0E34\u0E48\u0E19\u0E10\ + \u0E32\u0E19\u0E43\u0E2B\u0E21\u0E48\u0E17\u0E35\u0E48\u0E15\u0E49\u0E2D\u0E07\ + \u0E01\u0E32\u0E23\u0E2B\u0E25\u0E2D\u0E21\u0E23\u0E27\u0E21\u0E40\u0E02\u0E49\ + \u0E32\u0E01\u0E31\u0E1A\u0E27\u0E31\u0E12\u0E19\u0E18\u0E23\u0E23\u0E21\u0E43\ + \u0E2B\u0E21\u0E48\u0E02\u0E2D\u0E07\u0E1E\u0E27\u0E01\u0E40\u0E02\u0E32" + D: "\u0E40\u0E14\u0E47\u0E01\u0E17\u0E35\u0E48\u0E15\u0E31\u0E14\u0E2A\u0E34\ + \u0E19\u0E43\u0E08\u0E0B\u0E37\u0E49\u0E2D\u0E2A\u0E48\u0E27\u0E19\u0E43\u0E2B\ + \u0E0D\u0E48\u0E21\u0E32\u0E08\u0E32\u0E01\u0E2D\u0E34\u0E17\u0E18\u0E34\u0E1E\ + \u0E25\u0E20\u0E32\u0E22\u0E19\u0E2D\u0E01" + input_correct_responses: + - D + input_question: "\u0E01\u0E25\u0E38\u0E48\u0E21\u0E40\u0E14\u0E35\u0E22\u0E27\u0E43\ + \u0E19\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E17\u0E35\u0E48\u0E40\u0E2A\u0E35\u0E48\ + \u0E22\u0E07\u0E15\u0E48\u0E2D\u0E2D\u0E34\u0E17\u0E18\u0E34\u0E1E\u0E25\u0E02\ + \u0E2D\u0E07\u0E01\u0E25\u0E38\u0E48\u0E21\u0E2D\u0E49\u0E32\u0E07\u0E2D\u0E34\ + \u0E07\u0E21\u0E32\u0E01\u0E17\u0E35\u0E48\u0E2A\u0E38\u0E14\u0E04\u0E37\u0E2D\ + :" + - input_choice_list: + A: "\u0E2A\u0E32\u0E22\u0E01\u0E32\u0E23\u0E14\u0E39\u0E41\u0E25" + B: "\u0E08\u0E14\u0E2B\u0E21\u0E32\u0E22\u0E42\u0E14\u0E22\u0E15\u0E23\u0E07" + C: "\u0E40\u0E21\u0E47\u0E14\u0E21\u0E35\u0E14" + D: "\u0E1B\u0E23\u0E30\u0E15\u0E39\u0E44\u0E1B\u0E17\u0E35\u0E48\u0E1B\u0E23\ + \u0E30\u0E15\u0E39" + input_correct_responses: + - D + input_question: "\u0E41\u0E21\u0E49\u0E27\u0E48\u0E32\u0E40\u0E19\u0E37\u0E49\u0E2D\ + \u0E2B\u0E32\u0E41\u0E25\u0E30\u0E04\u0E38\u0E13\u0E20\u0E32\u0E1E\u0E08\u0E30\ + \u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16\u0E04\u0E27\u0E1A\u0E04\u0E38\u0E21\u0E44\ + \u0E14\u0E49\u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E01\u0E31\u0E1A\u0E01\u0E32\ + \u0E23\u0E2A\u0E48\u0E07\u0E08\u0E14\u0E2B\u0E21\u0E32\u0E22\u0E42\u0E14\u0E22\ + \u0E15\u0E23\u0E07 \u0E41\u0E15\u0E48\u0E2D\u0E31\u0E15\u0E23\u0E32\u0E01\u0E32\ + \u0E23\u0E15\u0E2D\u0E1A\u0E01\u0E25\u0E31\u0E1A\u0E02\u0E2D\u0E07\u0E2A\u0E37\ + \u0E48\u0E2D\u0E19\u0E35\u0E49\u0E15\u0E48\u0E33\u0E01\u0E27\u0E48\u0E32\u0E40\ + \u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E44\u0E21\u0E48\u0E21\u0E35\ + \u0E01\u0E25\u0E44\u0E01\u0E17\u0E35\u0E48\u0E2D\u0E22\u0E39\u0E48\u0E2A\u0E48\ + \u0E27\u0E19\u0E1A\u0E38\u0E04\u0E04\u0E25 \u0E23\u0E39\u0E1B\u0E41\u0E1A\u0E1A\ + \u0E2A\u0E37\u0E48\u0E2D\u0E19\u0E35\u0E49\u0E40\u0E23\u0E35\u0E22\u0E01\u0E27\ + \u0E48\u0E32:" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_marketing +tag: mmlu_th_llama_other_tasks +task: mmlu_th_llama_marketing +task_alias: marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48bd0bc1a2a42e77e36f31ae0ab33678a718dcd8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_moral_scenarios.yaml @@ -0,0 +1,141 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E1C\u0E34\u0E14 \u0E1C\u0E34\u0E14" + B: "\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + C: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14\u0E2B\u0E23\u0E2D\u0E01 \u0E1C\u0E34\ + \u0E14" + D: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + input_correct_responses: + - C + input_question: "\u0E15\u0E31\u0E27\u0E25\u0E30\u0E04\u0E23\u0E2B\u0E25\u0E31\u0E01\ + \ (\u0E0B\u0E36\u0E48\u0E07\u0E43\u0E0A\u0E49 I/me/my) \u0E17\u0E33\u0E2D\u0E30\ + \u0E44\u0E23\u0E1C\u0E34\u0E14\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19\u0E15\u0E32\u0E21\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\ + \u0E21\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E43\u0E19\u0E2A\u0E2B\u0E23\u0E31\ + \u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\u0E1B\u0E35 2020\ + \ \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\ + \u0E13\u0E4C\u0E43\u0E14 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\ + \u0E17\u0E35\u0E48 1 | \u0E09\u0E31\u0E19\u0E23\u0E31\u0E01\u0E40\u0E14\u0E47\ + \u0E01\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22\u0E41\u0E23\u0E01\u0E40\u0E01\u0E34\ + \u0E14\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19\u0E08\u0E23\u0E34\u0E07\u0E46 \u0E2A\ + \u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48 2 | \u0E40\ + \u0E14\u0E47\u0E01\u0E43\u0E2B\u0E21\u0E48\u0E2D\u0E32\u0E22\u0E17\u0E35\u0E48\ + \u0E08\u0E30\u0E2D\u0E22\u0E39\u0E48\u0E43\u0E19\u0E17\u0E35\u0E48\u0E08\u0E2D\ + \u0E14\u0E23\u0E16\u0E1E\u0E48\u0E27\u0E07 \u0E14\u0E31\u0E07\u0E19\u0E31\u0E49\ + \u0E19\u0E09\u0E31\u0E19\u0E08\u0E36\u0E07\u0E40\u0E0A\u0E34\u0E0D\u0E0A\u0E31\ + \u0E49\u0E19\u0E40\u0E23\u0E35\u0E22\u0E19\u0E44\u0E1B\u0E17\u0E35\u0E48\u0E1A\ + \u0E49\u0E32\u0E19\u0E02\u0E2D\u0E07\u0E40\u0E18\u0E2D" + - input_choice_list: + A: "\u0E1C\u0E34\u0E14 \u0E1C\u0E34\u0E14" + B: "\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + C: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14\u0E2B\u0E23\u0E2D\u0E01 \u0E1C\u0E34\ + \u0E14" + D: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + input_correct_responses: + - C + input_question: "\u0E15\u0E31\u0E27\u0E25\u0E30\u0E04\u0E23\u0E2B\u0E25\u0E31\u0E01\ + \ (\u0E0B\u0E36\u0E48\u0E07\u0E43\u0E0A\u0E49 I/me/my) \u0E17\u0E33\u0E2D\u0E30\ + \u0E44\u0E23\u0E1C\u0E34\u0E14\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19\u0E15\u0E32\u0E21\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\ + \u0E21\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E43\u0E19\u0E2A\u0E2B\u0E23\u0E31\ + \u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\u0E1B\u0E35 2020\ + \ \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\ + \u0E13\u0E4C\u0E43\u0E14 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\ + \u0E17\u0E35\u0E48 1 | \u0E09\u0E31\u0E19\u0E02\u0E2D\u0E1A\u0E38\u0E2B\u0E23\ + \u0E35\u0E48\u0E08\u0E32\u0E01\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22\u0E19\u0E2D\ + \u0E01\u0E23\u0E49\u0E32\u0E19 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\ + \u0E4C\u0E17\u0E35\u0E48 2 | \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E08\u0E49\ + \u0E32\u0E2B\u0E19\u0E49\u0E32\u0E17\u0E35\u0E48\u0E23\u0E31\u0E01\u0E29\u0E32\ + \u0E04\u0E27\u0E32\u0E21\u0E1B\u0E25\u0E2D\u0E14\u0E20\u0E31\u0E22\u0E2D\u0E2D\ + \u0E01\u0E08\u0E32\u0E01\u0E15\u0E33\u0E41\u0E2B\u0E19\u0E48\u0E07 \u0E09\u0E31\ + \u0E19\u0E44\u0E14\u0E49\u0E40\u0E01\u0E47\u0E1A\u0E40\u0E2B\u0E23\u0E35\u0E22\ + \u0E0D\u0E2B\u0E32\u0E22\u0E32\u0E01\u0E44\u0E27\u0E49\u0E43\u0E19\u0E01\u0E23\ + \u0E30\u0E40\u0E1B\u0E4B\u0E32" + - input_choice_list: + A: "\u0E1C\u0E34\u0E14 \u0E1C\u0E34\u0E14" + B: "\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + C: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14\u0E2B\u0E23\u0E2D\u0E01 \u0E1C\u0E34\ + \u0E14" + D: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + input_correct_responses: + - D + input_question: "\u0E15\u0E31\u0E27\u0E25\u0E30\u0E04\u0E23\u0E2B\u0E25\u0E31\u0E01\ + \ (\u0E0B\u0E36\u0E48\u0E07\u0E43\u0E0A\u0E49 I/me/my) \u0E17\u0E33\u0E2D\u0E30\ + \u0E44\u0E23\u0E1C\u0E34\u0E14\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19\u0E15\u0E32\u0E21\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\ + \u0E21\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E43\u0E19\u0E2A\u0E2B\u0E23\u0E31\ + \u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\u0E1B\u0E35 2020\ + \ \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\ + \u0E13\u0E4C\u0E43\u0E14 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\ + \u0E17\u0E35\u0E48 1 | \u0E09\u0E31\u0E19\u0E0B\u0E37\u0E49\u0E2D\u0E23\u0E16\ + \u0E01\u0E23\u0E30\u0E1A\u0E30\u0E40\u0E01\u0E48\u0E32\u0E2A\u0E33\u0E2B\u0E23\ + \u0E31\u0E1A\u0E17\u0E33\u0E07\u0E32\u0E19 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\ + \u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48 2 | \u0E09\u0E31\u0E19\u0E21\u0E2D\u0E07\ + \u0E02\u0E36\u0E49\u0E19\u0E44\u0E1B\u0E17\u0E35\u0E48\u0E19\u0E49\u0E33\u0E15\ + \u0E01\u0E17\u0E35\u0E48\u0E07\u0E14\u0E07\u0E32\u0E21\u0E41\u0E25\u0E30\u0E02\ + \u0E27\u0E49\u0E32\u0E07\u0E01\u0E49\u0E2D\u0E19\u0E2B\u0E34\u0E19\u0E43\u0E2A\ + \u0E48\u0E21\u0E31\u0E19" + - input_choice_list: + A: "\u0E1C\u0E34\u0E14 \u0E1C\u0E34\u0E14" + B: "\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + C: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14\u0E2B\u0E23\u0E2D\u0E01 \u0E1C\u0E34\ + \u0E14" + D: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + input_correct_responses: + - B + input_question: "\u0E15\u0E31\u0E27\u0E25\u0E30\u0E04\u0E23\u0E2B\u0E25\u0E31\u0E01\ + \ (\u0E0B\u0E36\u0E48\u0E07\u0E43\u0E0A\u0E49 I/me/my) \u0E17\u0E33\u0E2D\u0E30\ + \u0E44\u0E23\u0E1C\u0E34\u0E14\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19\u0E15\u0E32\u0E21\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\ + \u0E21\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E43\u0E19\u0E2A\u0E2B\u0E23\u0E31\ + \u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\u0E1B\u0E35 2020\ + \ \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\ + \u0E13\u0E4C\u0E43\u0E14 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\ + \u0E17\u0E35\u0E48 1 | \u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E17\u0E35\ + \u0E48\u0E23\u0E37\u0E49\u0E2D\u0E1A\u0E49\u0E32\u0E19 \u0E09\u0E31\u0E19\u0E1C\ + \u0E25\u0E31\u0E01\u0E40\u0E02\u0E32\u0E0A\u0E19\u0E01\u0E33\u0E41\u0E1E\u0E07\ + \ \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48 2\ + \ | \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E09\u0E31\u0E19\u0E40\u0E25\u0E37\u0E2D\ + \u0E01\u0E1E\u0E35\u0E48\u0E40\u0E25\u0E35\u0E49\u0E22\u0E07\u0E40\u0E14\u0E47\ + \u0E01\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E2B\u0E25\u0E32\u0E19\u0E2A\u0E32\ + \u0E27\u0E1D\u0E32\u0E41\u0E1D\u0E14\u0E02\u0E2D\u0E07\u0E09\u0E31\u0E19 \u0E09\ + \u0E31\u0E19\u0E44\u0E21\u0E48\u0E2A\u0E19\u0E43\u0E08\u0E20\u0E39\u0E21\u0E34\ + \u0E2B\u0E25\u0E31\u0E07\u0E17\u0E32\u0E07\u0E40\u0E0A\u0E37\u0E49\u0E2D\u0E0A\ + \u0E32\u0E15\u0E34\u0E02\u0E2D\u0E07\u0E40\u0E18\u0E2D" + - input_choice_list: + A: "\u0E1C\u0E34\u0E14 \u0E1C\u0E34\u0E14" + B: "\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + C: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14\u0E2B\u0E23\u0E2D\u0E01 \u0E1C\u0E34\ + \u0E14" + D: "\u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14 \u0E44\u0E21\u0E48\u0E1C\u0E34\u0E14" + input_correct_responses: + - A + input_question: "\u0E15\u0E31\u0E27\u0E25\u0E30\u0E04\u0E23\u0E2B\u0E25\u0E31\u0E01\ + \ (\u0E0B\u0E36\u0E48\u0E07\u0E43\u0E0A\u0E49 I/me/my) \u0E17\u0E33\u0E2D\u0E30\ + \u0E44\u0E23\u0E1C\u0E34\u0E14\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\u0E21\u0E2D\ + \u0E22\u0E48\u0E32\u0E07\u0E0A\u0E31\u0E14\u0E40\u0E08\u0E19\u0E15\u0E32\u0E21\ + \u0E21\u0E32\u0E15\u0E23\u0E10\u0E32\u0E19\u0E28\u0E35\u0E25\u0E18\u0E23\u0E23\ + \u0E21\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E43\u0E19\u0E2A\u0E2B\u0E23\u0E31\ + \u0E10\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E43\u0E19\u0E1B\u0E35 2020\ + \ \u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\ + \u0E13\u0E4C\u0E43\u0E14 \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\u0E23\u0E13\u0E4C\ + \u0E17\u0E35\u0E48 1 | \u0E09\u0E31\u0E19\u0E2B\u0E22\u0E34\u0E1A\u0E01\u0E23\ + \u0E30\u0E40\u0E1B\u0E4B\u0E32\u0E2A\u0E15\u0E32\u0E07\u0E04\u0E4C\u0E02\u0E2D\ + \u0E07\u0E40\u0E02\u0E32\u0E2B\u0E25\u0E31\u0E07\u0E08\u0E32\u0E01\u0E17\u0E35\ + \u0E48\u0E40\u0E02\u0E32\u0E25\u0E27\u0E19\u0E25\u0E32\u0E21\u0E09\u0E31\u0E19\ + \u0E17\u0E35\u0E48\u0E1A\u0E32\u0E23\u0E4C \u0E2A\u0E16\u0E32\u0E19\u0E01\u0E32\ + \u0E23\u0E13\u0E4C\u0E17\u0E35\u0E48 2 | \u0E09\u0E31\u0E19\u0E43\u0E2B\u0E49\ + \u0E04\u0E27\u0E32\u0E21\u0E23\u0E31\u0E01\u0E01\u0E31\u0E1A\u0E17\u0E32\u0E23\ + \u0E01\u0E41\u0E23\u0E01\u0E40\u0E01\u0E34\u0E14\u0E02\u0E2D\u0E07\u0E09\u0E31\ + \u0E19" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_moral_scenarios +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_moral_scenarios +task_alias: moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfe34386b3446c8443d7c6004cf9e858ec3dfd4d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_prehistory.yaml @@ -0,0 +1,142 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2A\u0E19\u0E2D\u0E07\u0E19\u0E31\u0E01\u0E1A\u0E27\u0E0A\u0E19\u0E31\ + \u0E01\u0E14\u0E32\u0E23\u0E32\u0E28\u0E32\u0E2A\u0E15\u0E23\u0E4C\u0E0A\u0E32\ + \u0E27\u0E21\u0E32\u0E22\u0E31\u0E19\u0E17\u0E35\u0E48\u0E17\u0E23\u0E07\u0E1E\ + \u0E25\u0E31\u0E07" + B: "\u0E41\u0E2A\u0E14\u0E07\u0E04\u0E27\u0E32\u0E21\u0E40\u0E2D\u0E37\u0E49\ + \u0E2D\u0E40\u0E1F\u0E37\u0E49\u0E2D\u0E40\u0E1C\u0E37\u0E48\u0E2D\u0E41\u0E1C\ + \u0E48\u0E15\u0E48\u0E2D\u0E04\u0E19\u0E17\u0E31\u0E48\u0E27\u0E44\u0E1B\u0E40\ + \u0E1E\u0E23\u0E32\u0E30\u0E44\u0E14\u0E49\u0E23\u0E31\u0E1A\u0E2D\u0E19\u0E38\ + \u0E0D\u0E32\u0E15\u0E43\u0E2B\u0E49\u0E2D\u0E22\u0E39\u0E48\u0E08\u0E33\u0E1E\ + \u0E23\u0E23\u0E29\u0E32\u0E44\u0E14\u0E49" + C: "\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E28\u0E31\u0E15\u0E23\u0E39\u0E2B\u0E27\ + \u0E32\u0E14\u0E01\u0E25\u0E31\u0E27\u0E42\u0E14\u0E22\u0E40\u0E09\u0E1E\u0E32\ + \u0E30\u0E0A\u0E32\u0E27\u0E2A\u0E40\u0E1B\u0E19" + D: "\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E29\u0E31\ + \u0E15\u0E23\u0E34\u0E22\u0E4C\u0E42\u0E14\u0E22\u0E0A\u0E2D\u0E1A\u0E14\u0E49\ + \u0E27\u0E22\u0E01\u0E0E\u0E2B\u0E21\u0E32\u0E22 \u0E40\u0E19\u0E37\u0E48\u0E2D\ + \u0E07\u0E08\u0E32\u0E01\u0E1A\u0E34\u0E14\u0E32\u0E44\u0E21\u0E48\u0E43\u0E0A\ + \u0E48\u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C" + input_correct_responses: + - D + input_question: "Pacal \u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E21\u0E32\ + \u0E22\u0E31\u0E19\u0E1C\u0E39\u0E49\u0E22\u0E34\u0E48\u0E07\u0E43\u0E2B\u0E0D\ + \u0E48\u0E2A\u0E23\u0E49\u0E32\u0E07\u0E27\u0E31\u0E14\u0E43\u0E19\u0E40\u0E21\ + \u0E37\u0E2D\u0E07 Palenque \u0E40\u0E1E\u0E37\u0E48\u0E2D:" + - input_choice_list: + A: "\u0E28\u0E39\u0E19\u0E22\u0E4C\u0E01\u0E25\u0E32\u0E07\u0E02\u0E2D\u0E07\ + \u0E2D\u0E32\u0E23\u0E22\u0E18\u0E23\u0E23\u0E21\u0E21\u0E34\u0E2A\u0E0B\u0E34\ + \u0E2A\u0E0B\u0E34\u0E1B\u0E1B\u0E35\u0E17\u0E35\u0E48\u0E21\u0E35\u0E2A\u0E20\ + \u0E32\u0E1E\u0E04\u0E25\u0E49\u0E32\u0E22\u0E01\u0E31\u0E1A\u0E01\u0E32\u0E23\ + \u0E02\u0E36\u0E49\u0E19\u0E02\u0E2D\u0E07\u0E23\u0E31\u0E10\u0E43\u0E19\u0E22\ + \u0E38\u0E04\u0E41\u0E23\u0E01" + B: "\u0E02\u0E49\u0E2D \u0E08\u0E33\u0E01\u0E31\u0E14 \u0E02\u0E2D\u0E07\u0E2D\ + \u0E33\u0E19\u0E32\u0E08\u0E43\u0E19\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E2D\u0E40\ + \u0E21\u0E23\u0E34\u0E01\u0E31\u0E19\u0E1E\u0E37\u0E49\u0E19\u0E40\u0E21\u0E37\ + \u0E2D\u0E07\u0E02\u0E2D\u0E07\u0E1C\u0E39\u0E49\u0E2B\u0E32\u0E2D\u0E32\u0E2B\ + \u0E32\u0E23\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E40\u0E2A\u0E21\u0E2D\u0E20\u0E32\ + \u0E04" + C: "\u0E01\u0E32\u0E23\u0E1B\u0E01\u0E04\u0E23\u0E2D\u0E07\u0E41\u0E1A\u0E1A\ + \u0E07\u0E48\u0E32\u0E22\u0E46 \u0E2B\u0E23\u0E37\u0E2D\u0E1A\u0E32\u0E07\u0E17\ + \u0E35\u0E2D\u0E32\u0E08\u0E40\u0E1B\u0E47\u0E19\u0E01\u0E32\u0E23\u0E1B\u0E01\ + \u0E04\u0E23\u0E2D\u0E07\u0E41\u0E1A\u0E1A\u0E0B\u0E31\u0E1A\u0E0B\u0E49\u0E2D\ + \u0E19\u0E44\u0E14\u0E49\u0E1E\u0E31\u0E12\u0E19\u0E32\u0E02\u0E36\u0E49\u0E19\ + \u0E43\u0E19\u0E1B\u0E35 \u0E04.\u0E28. 1500" + D: "\u0E28\u0E39\u0E19\u0E22\u0E4C\u0E01\u0E25\u0E32\u0E07\u0E2D\u0E32\u0E23\ + \u0E22\u0E18\u0E23\u0E23\u0E21\u0E21\u0E34\u0E2A\u0E0B\u0E34\u0E2A\u0E0B\u0E34\ + \u0E1B\u0E1B\u0E35\u0E49\u0E17\u0E35\u0E48\u0E21\u0E35\u0E2A\u0E20\u0E32\u0E1E\ + \u0E04\u0E25\u0E49\u0E32\u0E22\u0E2A\u0E31\u0E07\u0E04\u0E21\u0E17\u0E32\u0E07\ + \u0E0A\u0E32\u0E22\u0E1D\u0E31\u0E48\u0E07\u0E15\u0E30\u0E27\u0E31\u0E19\u0E15\ + \u0E01\u0E40\u0E09\u0E35\u0E22\u0E07\u0E40\u0E2B\u0E19\u0E37\u0E2D\u0E02\u0E2D\ + \u0E07\u0E17\u0E27\u0E35\u0E1B\u0E2D\u0E40\u0E21\u0E23\u0E34\u0E01\u0E32\u0E40\ + \u0E2B\u0E19\u0E37\u0E2D" + input_correct_responses: + - A + input_question: "\u0E15\u0E32\u0E21\u0E17\u0E35\u0E48 Timothy Pauketat \u0E2B\u0E25\ + \u0E31\u0E01\u0E10\u0E32\u0E19\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\ + \u0E23\u0E41\u0E1A\u0E48\u0E07\u0E0A\u0E31\u0E49\u0E19\u0E17\u0E32\u0E07\u0E2A\ + \u0E31\u0E07\u0E04\u0E21\u0E41\u0E25\u0E30\u0E2D\u0E33\u0E19\u0E32\u0E08\u0E17\ + \u0E32\u0E07\u0E01\u0E32\u0E23\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E17\u0E35\u0E48\ + \ Cahokia \u0E41\u0E2A\u0E14\u0E07\u0E43\u0E2B\u0E49\u0E40\u0E2B\u0E47\u0E19\ + :" + - input_choice_list: + A: "\u0E20\u0E31\u0E22\u0E1E\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E1A\u0E32\u0E07\ + \u0E0A\u0E19\u0E34\u0E14 \u0E40\u0E0A\u0E48\u0E19 \u0E41\u0E1C\u0E48\u0E19\ + \u0E14\u0E34\u0E19\u0E44\u0E2B\u0E27 \u0E20\u0E39\u0E40\u0E02\u0E32\u0E44\u0E1F\ + \ \u0E2B\u0E23\u0E37\u0E2D\u0E2A\u0E36\u0E19\u0E32\u0E21\u0E34" + B: "\u0E04\u0E27\u0E32\u0E21\u0E40\u0E2A\u0E37\u0E48\u0E2D\u0E21\u0E42\u0E17\ + \u0E23\u0E21\u0E02\u0E2D\u0E07\u0E23\u0E30\u0E1A\u0E1A\u0E19\u0E34\u0E40\u0E27\ + \u0E28\u0E2D\u0E31\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E25\u0E08\u0E32\u0E01\ + \u0E40\u0E17\u0E04\u0E19\u0E34\u0E04\u0E01\u0E32\u0E23\u0E17\u0E33\u0E44\u0E23\ + \u0E48\u0E44\u0E16\u0E19\u0E32" + C: "\u0E2A\u0E07\u0E04\u0E23\u0E32\u0E21\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\ + \u0E21\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E34\u0E49\u0E19\u0E2A\u0E38\u0E14\u0E23\ + \u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E19\u0E04\u0E23\u0E23\u0E31\u0E10\u0E02\ + \u0E2D\u0E07\u0E0A\u0E32\u0E27\u0E21\u0E32\u0E22\u0E31\u0E19\u0E17\u0E35\u0E48\ + \u0E2D\u0E22\u0E39\u0E48\u0E43\u0E01\u0E25\u0E49\u0E40\u0E04\u0E35\u0E22\u0E07" + D: "\u0E01\u0E32\u0E23\u0E1B\u0E0F\u0E34\u0E1A\u0E31\u0E15\u0E34\u0E02\u0E2D\ + \u0E07\u0E01\u0E32\u0E23\u0E1C\u0E2A\u0E21\u0E02\u0E49\u0E32\u0E21\u0E1E\u0E31\ + \u0E19\u0E18\u0E38\u0E4C\u0E17\u0E35\u0E48\u0E19\u0E33\u0E44\u0E1B\u0E2A\u0E39\ + \u0E48\u0E04\u0E27\u0E32\u0E21\u0E1C\u0E34\u0E14\u0E1B\u0E01\u0E15\u0E34 \u0E41\ + \u0E15\u0E48\u0E01\u0E33\u0E40\u0E19\u0E34\u0E14\u0E17\u0E35\u0E48\u0E40\u0E1E\ + \u0E34\u0E48\u0E21\u0E02\u0E36\u0E49\u0E19\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E21\ + \u0E32\u0E01" + input_correct_responses: + - B + input_question: "\u0E15\u0E2D\u0E19\u0E19\u0E35\u0E49\u0E19\u0E31\u0E01\u0E27\u0E34\ + \u0E08\u0E31\u0E22\u0E40\u0E0A\u0E37\u0E48\u0E2D\u0E27\u0E48\u0E32\u0E01\u0E32\ + \u0E23\u0E25\u0E14\u0E25\u0E07\u0E02\u0E2D\u0E07\u0E21\u0E32\u0E22\u0E32\u0E21\ + \u0E35\u0E2A\u0E32\u0E40\u0E2B\u0E15\u0E38\u0E2B\u0E25\u0E31\u0E01\u0E21\u0E32\ + \u0E08\u0E32\u0E01:" + - input_choice_list: + A: "\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\ + \u0E02\u0E2D\u0E07\u0E2A\u0E1B\u0E35\u0E0A\u0E35\u0E2A\u0E4C\u0E08\u0E33\u0E19\ + \u0E27\u0E19\u0E21\u0E32\u0E01 \u0E2B\u0E23\u0E37\u0E2D\u0E2A\u0E1B\u0E35\u0E0A\ + \u0E35\u0E2A\u0E4C\u0E40\u0E14\u0E35\u0E22\u0E27\u0E17\u0E35\u0E48\u0E21\u0E35\ + \u0E04\u0E27\u0E32\u0E21\u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\u0E21\ + \u0E32\u0E01" + B: "\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\ + \u0E02\u0E2D\u0E07\u0E2A\u0E1B\u0E35\u0E0A\u0E35\u0E2A\u0E4C\u0E19\u0E49\u0E2D\ + \u0E22\u0E21\u0E32\u0E01\u0E43\u0E19\u0E0A\u0E48\u0E27\u0E07\u0E40\u0E27\u0E25\ + \u0E32\u0E19\u0E35\u0E49\u0E41\u0E25\u0E30\u0E21\u0E35 hominids \u0E19\u0E49\ + \u0E2D\u0E22\u0E21\u0E32\u0E01" + C: "\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\ + \u0E02\u0E2D\u0E07\u0E2A\u0E32\u0E22\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E4C\u0E25\ + \u0E14\u0E25\u0E07\u0E40\u0E19\u0E37\u0E48\u0E2D\u0E07\u0E08\u0E32\u0E01\u0E22\ + \u0E38\u0E04\u0E19\u0E49\u0E33\u0E41\u0E02\u0E47\u0E07\u0E17\u0E35\u0E48\u0E22\ + \u0E37\u0E14\u0E40\u0E22\u0E37\u0E49\u0E2D\u0E15\u0E32\u0E21\u0E21\u0E32\u0E14\ + \u0E49\u0E27\u0E22\u0E04\u0E27\u0E32\u0E21\u0E41\u0E2B\u0E49\u0E07\u0E41\u0E25\ + \u0E49\u0E07\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E23\u0E38\u0E19\u0E41\u0E23\u0E07" + D: "\u0E04\u0E27\u0E32\u0E21\u0E2B\u0E25\u0E32\u0E01\u0E2B\u0E25\u0E32\u0E22\ + \u0E02\u0E2D\u0E07\u0E0A\u0E19\u0E34\u0E14\u0E25\u0E14\u0E25\u0E07 \u0E41\u0E15\ + \u0E48\u0E08\u0E33\u0E19\u0E27\u0E19\u0E2B\u0E34\u0E19\u0E04\u0E49\u0E2D\u0E19\ + \u0E41\u0E25\u0E30\u0E2A\u0E30\u0E40\u0E01\u0E47\u0E14\u0E40\u0E1E\u0E34\u0E48\ + \u0E21\u0E02\u0E36\u0E49\u0E19 \u0E0B\u0E36\u0E48\u0E07\u0E1A\u0E48\u0E07\u0E0A\ + \u0E35\u0E49\u0E16\u0E36\u0E07\u0E01\u0E32\u0E23\u0E1C\u0E25\u0E34\u0E15\u0E40\ + \u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E21\u0E37\u0E2D\u0E2B\u0E34\u0E19" + input_correct_responses: + - A + input_question: "\u0E01\u0E32\u0E23\u0E27\u0E34\u0E08\u0E31\u0E22\u0E25\u0E48\u0E32\ + \u0E2A\u0E38\u0E14\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E01\u0E31\u0E1A\u0E2A\ + \u0E32\u0E22\u0E1E\u0E31\u0E19\u0E18\u0E38\u0E4C hominid \u0E17\u0E35\u0E48\u0E21\ + \u0E32\u0E08\u0E32\u0E01 Middle Pliocene \u0E23\u0E30\u0E1A\u0E38\u0E27\u0E48\ + \u0E32\u0E21\u0E35 (\u0E13 \u0E1B\u0E35 2020):" + - input_choice_list: + A: "\u0E44\u0E21\u0E48\u0E40\u0E01\u0E34\u0E19 650 \u0E0B\u0E35\u0E0B\u0E35" + B: "\u0E1B\u0E23\u0E30\u0E21\u0E32\u0E13 800 \u0E0B\u0E35\u0E0B\u0E35" + C: "\u0E40\u0E1E\u0E35\u0E22\u0E07\u0E44\u0E21\u0E48\u0E40\u0E01\u0E34\u0E19\ + \ 1,000 \u0E0B\u0E35\u0E0B\u0E35" + D: "1200 \u0E0B\u0E35\u0E0B\u0E35" + input_correct_responses: + - C + input_question: "\u0E04\u0E27\u0E32\u0E21\u0E08\u0E38\u0E01\u0E23\u0E30\u0E42\u0E2B\ + \u0E25\u0E01\u0E40\u0E09\u0E25\u0E35\u0E48\u0E22\u0E02\u0E2D\u0E07 Homo erectus\ + \ \u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_prehistory +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_prehistory +task_alias: prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb0ed31fcc7c96ee9d6c249dfbf920c9247695de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_professional_accounting.yaml @@ -0,0 +1,165 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "70,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C" + B: "75,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C" + C: "80,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C" + D: 100,000 + input_correct_responses: + - D + input_question: "Box \u0E2D\u0E07\u0E04\u0E4C\u0E01\u0E23\u0E1E\u0E31\u0E12\u0E19\ + \u0E32\u0E40\u0E2D\u0E01\u0E0A\u0E19\u0E17\u0E35\u0E48\u0E44\u0E21\u0E48\u0E41\ + \u0E2A\u0E27\u0E07\u0E2B\u0E32\u0E1C\u0E25\u0E01\u0E33\u0E44\u0E23\u0E21\u0E35\ + \u0E18\u0E38\u0E23\u0E01\u0E23\u0E23\u0E21\u0E15\u0E48\u0E2D\u0E44\u0E1B\u0E19\ + \u0E35\u0E49\u0E43\u0E19\u0E23\u0E30\u0E2B\u0E27\u0E48\u0E32\u0E07\u0E1B\u0E35\ + : \u0E40\u0E07\u0E34\u0E19\u0E2A\u0E14\u0E23\u0E31\u0E1A\u0E08\u0E32\u0E01\u0E01\ + \u0E32\u0E23\u0E02\u0E32\u0E22\u0E40\u0E07\u0E34\u0E19\u0E25\u0E07\u0E17\u0E38\ + \u0E19 80,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C \u0E0B\u0E37\u0E49\u0E2D\ + \u0E2D\u0E32\u0E04\u0E32\u0E23\u0E41\u0E25\u0E30\u0E2D\u0E38\u0E1B\u0E01\u0E23\ + \u0E13\u0E4C 10,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C \u0E40\u0E07\u0E34\ + \u0E19\u0E2A\u0E14\u0E23\u0E31\u0E1A\u0E08\u0E32\u0E01\u0E2B\u0E19\u0E35\u0E49\ + \u0E23\u0E30\u0E22\u0E30\u0E22\u0E32\u0E27 100,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\ + \u0E23\u0E4C \u0E02\u0E32\u0E14\u0E17\u0E38\u0E19\u0E08\u0E32\u0E01\u0E01\u0E32\ + \u0E23\u0E02\u0E32\u0E22\u0E40\u0E07\u0E34\u0E19\u0E25\u0E07\u0E17\u0E38\u0E19\ + \ 5,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C \u0E08\u0E33\u0E19\u0E27\u0E19\ + \u0E40\u0E07\u0E34\u0E19\u0E17\u0E35\u0E48\u0E04\u0E27\u0E23\u0E23\u0E32\u0E22\ + \u0E07\u0E32\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E2A\u0E38\u0E17\u0E18\u0E34 \u0E40\ + \u0E07\u0E34\u0E19\u0E2A\u0E14\u0E44\u0E14\u0E49\u0E21\u0E32\u0E08\u0E32\u0E01\ + \u0E01\u0E34\u0E08\u0E01\u0E23\u0E23\u0E21\u0E08\u0E31\u0E14\u0E2B\u0E32\u0E40\ + \u0E07\u0E34\u0E19\u0E43\u0E19\u0E07\u0E1A\u0E01\u0E23\u0E30\u0E41\u0E2A\u0E40\ + \u0E07\u0E34\u0E19\u0E2A\u0E14\u0E02\u0E2D\u0E07 Box?" + - input_choice_list: + A: $13,000 + B: $600 + C: $15,000 + D: $28,000 + input_correct_responses: + - A + input_question: "\u0E2B\u0E19\u0E36\u0E48\u0E07\u0E23\u0E49\u0E2D\u0E22\u0E1B\u0E35\ + \u0E17\u0E35\u0E48\u0E41\u0E25\u0E49\u0E27 \u0E04\u0E38\u0E13\u0E17\u0E27\u0E14\ + \u0E02\u0E2D\u0E07\u0E04\u0E38\u0E13\u0E25\u0E07\u0E17\u0E38\u0E19 $100 \u0E14\ + \u0E49\u0E27\u0E22\u0E14\u0E2D\u0E01\u0E40\u0E1A\u0E35\u0E49\u0E22 5% \u0E15\ + \u0E48\u0E2D\u0E1B\u0E35 \u0E01\u0E32\u0E23\u0E25\u0E07\u0E17\u0E38\u0E19\u0E27\ + \u0E31\u0E19\u0E19\u0E35\u0E49\u0E04\u0E38\u0E49\u0E21\u0E04\u0E48\u0E32\u0E41\ + \u0E04\u0E48\u0E44\u0E2B\u0E19?" + - input_choice_list: + A: $0 + B: $500 + C: $1,650 + D: $16,500 + input_correct_responses: + - A + input_question: "Krete \u0E40\u0E1B\u0E47\u0E19\u0E1C\u0E39\u0E49\u0E40\u0E2A\u0E35\ + \u0E22\u0E20\u0E32\u0E29\u0E35\u0E42\u0E2A\u0E14\u0E17\u0E35\u0E48\u0E21\u0E35\ + \u0E23\u0E32\u0E22\u0E44\u0E14\u0E49\u0E08\u0E32\u0E01\u0E04\u0E48\u0E32\u0E08\ + \u0E49\u0E32\u0E07\u0E40\u0E17\u0E48\u0E32\u0E19\u0E31\u0E49\u0E19 \u0E20\u0E32\ + \u0E22\u0E43\u0E19\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48 31 \u0E18\u0E31\u0E19\ + \u0E27\u0E32\u0E04\u0E21 \u0E1B\u0E35\u0E17\u0E35\u0E48 1 \u0E19\u0E32\u0E22\ + \u0E08\u0E49\u0E32\u0E07\u0E02\u0E2D\u0E07 Krete \u0E44\u0E14\u0E49\u0E2B\u0E31\ + \u0E01\u0E20\u0E32\u0E29\u0E35\u0E23\u0E32\u0E22\u0E44\u0E14\u0E49\u0E02\u0E2D\ + \u0E07\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E01\u0E25\u0E32\u0E07\u0E08\u0E33\ + \u0E19\u0E27\u0E19 16,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C \u0E41\u0E25\ + \u0E30 Krete \u0E22\u0E31\u0E07\u0E44\u0E21\u0E48\u0E44\u0E14\u0E49\u0E08\u0E48\ + \u0E32\u0E22\u0E20\u0E32\u0E29\u0E35\u0E42\u0E14\u0E22\u0E1B\u0E23\u0E30\u0E21\ + \u0E32\u0E13 \u0E43\u0E19\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48 15 \u0E40\u0E21\ + \u0E29\u0E32\u0E22\u0E19 \u0E1B\u0E35\u0E17\u0E35\u0E48 2 \u0E04\u0E23\u0E35\ + \u0E15\u0E44\u0E14\u0E49\u0E22\u0E37\u0E48\u0E19\u0E04\u0E33\u0E23\u0E49\u0E2D\ + \u0E07\u0E02\u0E2D\u0E02\u0E22\u0E32\u0E22\u0E40\u0E27\u0E25\u0E32\u0E22\u0E37\ + \u0E48\u0E19\u0E41\u0E1A\u0E1A\u0E41\u0E2A\u0E14\u0E07\u0E23\u0E32\u0E22\u0E01\ + \u0E32\u0E23\u0E20\u0E32\u0E29\u0E35\u0E2A\u0E48\u0E27\u0E19\u0E1A\u0E38\u0E04\ + \u0E04\u0E25\u0E02\u0E2D\u0E07\u0E40\u0E18\u0E2D\u0E2D\u0E22\u0E48\u0E32\u0E07\ + \u0E17\u0E31\u0E19\u0E17\u0E48\u0E27\u0E07\u0E17\u0E35 \u0E41\u0E25\u0E30\u0E08\ + \u0E48\u0E32\u0E22\u0E20\u0E32\u0E29\u0E35\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E40\ + \u0E15\u0E34\u0E21 300 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C \u0E04\u0E27\ + \u0E32\u0E21\u0E23\u0E31\u0E1A\u0E1C\u0E34\u0E14\u0E17\u0E32\u0E07\u0E20\u0E32\ + \u0E29\u0E35\u0E1B\u0E35\u0E17\u0E35\u0E48 1 \u0E02\u0E2D\u0E07 Krete \u0E2D\ + \u0E22\u0E39\u0E48\u0E17\u0E35\u0E48 16,500 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\ + \u0E4C \u0E40\u0E21\u0E37\u0E48\u0E2D\u0E40\u0E18\u0E2D\u0E22\u0E37\u0E48\u0E19\ + \u0E41\u0E1A\u0E1A\u0E41\u0E2A\u0E14\u0E07\u0E23\u0E32\u0E22\u0E01\u0E32\u0E23\ + \u0E20\u0E32\u0E29\u0E35\u0E43\u0E19\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48 30\ + \ \u0E40\u0E21\u0E29\u0E32\u0E22\u0E19 \u0E1B\u0E35\u0E17\u0E35\u0E48 2 \u0E15\ + \u0E32\u0E21\u0E01\u0E33\u0E2B\u0E19\u0E14\u0E40\u0E27\u0E25\u0E32 \u0E41\u0E25\ + \u0E30\u0E0A\u0E33\u0E23\u0E30\u0E22\u0E2D\u0E14\u0E20\u0E32\u0E29\u0E35\u0E17\ + \u0E35\u0E48\u0E40\u0E2B\u0E25\u0E37\u0E2D\u0E2D\u0E22\u0E39\u0E48 \u0E08\u0E33\ + \u0E19\u0E27\u0E19\u0E40\u0E17\u0E48\u0E32\u0E43\u0E14\u0E17\u0E35\u0E48\u0E08\ + \u0E30\u0E15\u0E49\u0E2D\u0E07\u0E16\u0E39\u0E01\u0E1B\u0E23\u0E31\u0E1A\u0E2A\ + \u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E01\u0E32\u0E23\u0E0A\u0E33\u0E23\u0E30\u0E20\ + \u0E32\u0E29\u0E35\u0E17\u0E35\u0E48\u0E1B\u0E23\u0E30\u0E40\u0E21\u0E34\u0E19\ + \u0E44\u0E27\u0E49\u0E15\u0E48\u0E33\u0E01\u0E27\u0E48\u0E32\u0E04\u0E27\u0E32\ + \u0E21\u0E40\u0E1B\u0E47\u0E19\u0E08\u0E23\u0E34\u0E07?" + - input_choice_list: + A: "5,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C" + B: $13,500 + C: $16,000 + D: $20,000 + input_correct_responses: + - B + input_question: "\u0E40\u0E21\u0E37\u0E48\u0E2D\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48\ + \ 1 \u0E21\u0E01\u0E23\u0E32\u0E04\u0E21 \u0E1B\u0E35\u0E17\u0E35\u0E48 1 Alpha\ + \ Co. \u0E44\u0E14\u0E49\u0E25\u0E07\u0E19\u0E32\u0E21\u0E43\u0E19\u0E02\u0E49\ + \u0E2D\u0E15\u0E01\u0E25\u0E07\u0E01\u0E32\u0E23\u0E1A\u0E33\u0E23\u0E38\u0E07\ + \u0E23\u0E31\u0E01\u0E29\u0E32\u0E23\u0E32\u0E22\u0E1B\u0E35\u0E01\u0E31\u0E1A\ + \u0E1C\u0E39\u0E49\u0E43\u0E2B\u0E49\u0E1A\u0E23\u0E34\u0E01\u0E32\u0E23\u0E0B\ + \u0E2D\u0E1F\u0E15\u0E4C\u0E41\u0E27\u0E23\u0E4C\u0E43\u0E19\u0E23\u0E32\u0E04\ + \u0E32 15,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\u0E23\u0E4C \u0E41\u0E25\u0E30\u0E23\ + \u0E30\u0E22\u0E30\u0E40\u0E27\u0E25\u0E32\u0E01\u0E32\u0E23\u0E1A\u0E33\u0E23\ + \u0E38\u0E07\u0E23\u0E31\u0E01\u0E29\u0E32\u0E08\u0E30\u0E40\u0E23\u0E34\u0E48\ + \u0E21\u0E43\u0E19\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48 1 \u0E21\u0E35\u0E19\u0E32\ + \u0E04\u0E21 \u0E1B\u0E35\u0E17\u0E35\u0E48 2 \u0E19\u0E2D\u0E01\u0E08\u0E32\ + \u0E01\u0E19\u0E35\u0E49 Alpha \u0E22\u0E31\u0E07\u0E21\u0E35\u0E04\u0E48\u0E32\ + \u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22 5,000 \u0E14\u0E2D\u0E25\u0E25\u0E32\ + \u0E23\u0E4C\u0E43\u0E19\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48 1 \u0E21\u0E01\u0E23\ + \u0E32\u0E04\u0E21 \u0E1B\u0E35\u0E17\u0E35\u0E48 1 \u0E0B\u0E36\u0E48\u0E07\ + \u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\u0E02\u0E49\u0E2D\u0E07\u0E01\u0E31\u0E1A\ + \u0E01\u0E32\u0E23\u0E1B\u0E23\u0E31\u0E1A\u0E40\u0E1B\u0E25\u0E35\u0E48\u0E22\ + \u0E19\u0E0B\u0E2D\u0E1F\u0E15\u0E4C\u0E41\u0E27\u0E23\u0E4C \u0E04\u0E33\u0E02\ + \u0E2D\u0E17\u0E35\u0E48\u0E08\u0E30\u0E40\u0E1E\u0E34\u0E48\u0E21\u0E1F\u0E31\ + \u0E07\u0E01\u0E4C\u0E0A\u0E31\u0E19\u0E01\u0E32\u0E23\u0E17\u0E33\u0E07\u0E32\ + \u0E19\u0E02\u0E2D\u0E07\u0E0B\u0E2D\u0E1F\u0E15\u0E4C\u0E41\u0E27\u0E23\u0E4C\ + \ Alpha \u0E08\u0E30\u0E04\u0E34\u0E14\u0E04\u0E48\u0E32\u0E40\u0E2A\u0E37\u0E48\ + \u0E2D\u0E21\u0E23\u0E32\u0E04\u0E32\u0E41\u0E25\u0E30\u0E15\u0E31\u0E14\u0E08\ + \u0E33\u0E2B\u0E19\u0E48\u0E32\u0E22\u0E2A\u0E34\u0E19\u0E17\u0E23\u0E31\u0E1E\ + \u0E22\u0E4C\u0E04\u0E2D\u0E21\u0E1E\u0E34\u0E27\u0E40\u0E15\u0E2D\u0E23\u0E4C\ + \u0E41\u0E25\u0E30\u0E0B\u0E2D\u0E1F\u0E15\u0E4C\u0E41\u0E27\u0E23\u0E4C\u0E40\ + \u0E1B\u0E47\u0E19\u0E40\u0E27\u0E25\u0E32\u0E2B\u0E49\u0E32\u0E1B\u0E35\u0E42\ + \u0E14\u0E22\u0E43\u0E0A\u0E49\u0E27\u0E34\u0E18\u0E35\u0E40\u0E2A\u0E49\u0E19\ + \u0E15\u0E23\u0E07 \u0E04\u0E48\u0E32\u0E43\u0E0A\u0E49\u0E08\u0E48\u0E32\u0E22\ + \u0E17\u0E31\u0E49\u0E07\u0E2B\u0E21\u0E14\u0E17\u0E35\u0E48 Alpha \u0E04\u0E27\ + \u0E23\u0E23\u0E31\u0E1A\u0E23\u0E39\u0E49\u0E40\u0E01\u0E35\u0E48\u0E22\u0E27\ + \u0E01\u0E31\u0E1A\u0E02\u0E49\u0E2D\u0E15\u0E01\u0E25\u0E07\u0E01\u0E32\u0E23\ + \u0E1A\u0E33\u0E23\u0E38\u0E07\u0E23\u0E31\u0E01\u0E29\u0E32\u0E41\u0E25\u0E30\ + \u0E01\u0E32\u0E23\u0E41\u0E01\u0E49\u0E44\u0E02\u0E0B\u0E2D\u0E1F\u0E15\u0E4C\ + \u0E41\u0E27\u0E23\u0E4C\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E1B\u0E35\u0E2A\ + \u0E34\u0E49\u0E19\u0E2A\u0E38\u0E14\u0E27\u0E31\u0E19\u0E17\u0E35\u0E48 31\ + \ \u0E18\u0E31\u0E19\u0E27\u0E32\u0E04\u0E21 \u0E1B\u0E35\u0E17\u0E35\u0E48\ + \ 1 \u0E40\u0E1B\u0E47\u0E19\u0E08\u0E33\u0E19\u0E27\u0E19\u0E40\u0E17\u0E48\ + \u0E32\u0E43\u0E14" + - input_choice_list: + A: "\u0E01\u0E32\u0E23\u0E1B\u0E23\u0E30\u0E40\u0E21\u0E34\u0E19\u0E21\u0E39\ + \u0E25\u0E04\u0E48\u0E32\u0E41\u0E25\u0E30\u0E01\u0E32\u0E23\u0E08\u0E31\u0E14\ + \u0E2A\u0E23\u0E23" + B: "\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E21\u0E1A\u0E39\u0E23\u0E13\u0E4C" + C: "\u0E2A\u0E34\u0E17\u0E18\u0E34\u0E41\u0E25\u0E30\u0E2B\u0E19\u0E49\u0E32\ + \u0E17\u0E35\u0E48" + D: "\u0E01\u0E32\u0E23\u0E19\u0E33\u0E40\u0E2A\u0E19\u0E2D\u0E41\u0E25\u0E30\ + \u0E01\u0E32\u0E23\u0E40\u0E1B\u0E34\u0E14\u0E40\u0E1C\u0E22\u0E02\u0E49\u0E2D\ + \u0E21\u0E39\u0E25" + input_correct_responses: + - B + input_question: "\u0E1C\u0E39\u0E49\u0E2A\u0E2D\u0E1A\u0E1A\u0E31\u0E0D\u0E0A\u0E35\ + \u0E15\u0E34\u0E14\u0E15\u0E32\u0E21\u0E2B\u0E21\u0E32\u0E22\u0E40\u0E25\u0E02\ + \u0E1B\u0E23\u0E30\u0E08\u0E33\u0E40\u0E04\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E1A\ + \u0E19\u0E2D\u0E38\u0E1B\u0E01\u0E23\u0E13\u0E4C\u0E44\u0E1B\u0E22\u0E31\u0E07\ + \u0E1A\u0E31\u0E0D\u0E0A\u0E35\u0E41\u0E22\u0E01\u0E1B\u0E23\u0E30\u0E40\u0E20\ + \u0E17\u0E22\u0E48\u0E2D\u0E22\u0E02\u0E2D\u0E07\u0E1C\u0E39\u0E49\u0E44\u0E21\ + \u0E48\u0E2D\u0E2D\u0E01\u0E15\u0E23\u0E32\u0E2A\u0E32\u0E23 \u0E01\u0E32\u0E23\ + \u0E17\u0E14\u0E2A\u0E2D\u0E1A\u0E19\u0E35\u0E49\u0E23\u0E2D\u0E07\u0E23\u0E31\ + \u0E1A\u0E01\u0E32\u0E23\u0E22\u0E37\u0E19\u0E22\u0E31\u0E19\u0E01\u0E32\u0E23\ + \u0E08\u0E31\u0E14\u0E01\u0E32\u0E23\u0E02\u0E49\u0E2D\u0E43\u0E14\u0E15\u0E48\ + \u0E2D\u0E44\u0E1B\u0E19\u0E35\u0E49" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_professional_accounting +tag: mmlu_th_llama_other_tasks +task: mmlu_th_llama_professional_accounting +task_alias: professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1de3bcc3c25ad1b26a4daa5c9b0bb0a567768bf5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/llama3/instruct/mmlu_th/mmlu_th_world_religions.yaml @@ -0,0 +1,68 @@ +fewshot_config: + sampler: first_n + samples: + - input_choice_list: + A: "\u0E2A\u0E32\u0E22\u0E23\u0E38\u0E49\u0E07" + B: "\u0E01\u0E32\u0E23\u0E02\u0E25\u0E34\u0E1A" + C: "\u0E25\u0E39\u0E01\u0E0A\u0E32\u0E22" + D: "\u0E1A\u0E32\u0E23\u0E4C\u0E21\u0E34\u0E15\u0E0B\u0E4C\u0E27\u0E32\u0E2B\ + \u0E4C" + input_correct_responses: + - B + input_question: "\u0E2D\u0E30\u0E44\u0E23\u0E04\u0E37\u0E2D\u0E2A\u0E31\u0E0D\u0E0D\ + \u0E32\u0E13\u0E02\u0E2D\u0E07\u0E1E\u0E31\u0E19\u0E18\u0E2A\u0E31\u0E0D\u0E0D\ + \u0E32\u0E2A\u0E33\u0E2B\u0E23\u0E31\u0E1A\u0E1C\u0E39\u0E49\u0E0A\u0E32\u0E22\ + \u0E0A\u0E32\u0E27\u0E22\u0E34\u0E27?" + - input_choice_list: + A: "\u0E1E\u0E23\u0E30\u0E18\u0E23\u0E23\u0E21" + B: "\u0E04\u0E13\u0E30\u0E2A\u0E07\u0E06\u0E4C" + C: "\u0E1E\u0E23\u0E30\u0E1E\u0E38\u0E17\u0E18\u0E40\u0E08\u0E49\u0E32" + D: "\u0E1E\u0E23\u0E30\u0E42\u0E1E\u0E18\u0E34\u0E2A\u0E31\u0E15\u0E27\u0E4C" + input_correct_responses: + - A + input_question: "\u0E2D\u0E31\u0E0D\u0E21\u0E13\u0E35\u0E17\u0E35\u0E48\u0E2A\u0E2D\ + \u0E07\u0E43\u0E19\u0E1E\u0E23\u0E30\u0E1E\u0E38\u0E17\u0E18\u0E28\u0E32\u0E2A\ + \u0E19\u0E32\u0E04\u0E37\u0E2D\u0E2D\u0E30\u0E44\u0E23?" + - input_choice_list: + A: "\u0E0B\u0E32\u0E07" + B: "\u0E42\u0E08\u0E27" + C: "\u0E2E\u0E31\u0E48\u0E19" + D: "\u0E40\u0E0B\u0E35\u0E48\u0E22" + input_correct_responses: + - B + input_question: "\u0E23\u0E32\u0E0A\u0E27\u0E07\u0E28\u0E4C\u0E43\u0E14\u0E17\u0E35\ + \u0E48 "\u0E2D\u0E32\u0E13\u0E31\u0E15\u0E34\u0E41\u0E2B\u0E48\u0E07\u0E2A\ + \u0E27\u0E23\u0E23\u0E04\u0E4C" \u0E1E\u0E31\u0E12\u0E19\u0E32\u0E02\u0E36\ + \u0E49\u0E19\u0E40\u0E1E\u0E37\u0E48\u0E2D\u0E17\u0E33\u0E43\u0E2B\u0E49\u0E1C\ + \u0E39\u0E49\u0E1B\u0E01\u0E04\u0E23\u0E2D\u0E07\u0E43\u0E2B\u0E21\u0E48\u0E0A\ + \u0E2D\u0E1A\u0E18\u0E23\u0E23\u0E21?" + - input_choice_list: + A: "\u0E42\u0E2E\u0E40\u0E19\u0E47\u0E19" + B: "\u0E17\u0E32\u0E19\u0E32\u0E01\u0E30" + C: "\u0E42\u0E17\u0E04\u0E38\u0E01\u0E32\u0E27\u0E30" + D: "\u0E40\u0E21\u0E08\u0E34" + input_correct_responses: + - D + input_question: "\u0E23\u0E31\u0E10\u0E1A\u0E32\u0E25\u0E0D\u0E35\u0E48\u0E1B\u0E38\ + \u0E48\u0E19\u0E43\u0E14\u0E17\u0E35\u0E48\u0E2A\u0E48\u0E07\u0E40\u0E2A\u0E23\ + \u0E34\u0E21\u0E25\u0E31\u0E17\u0E18\u0E34\u0E1B\u0E23\u0E30\u0E08\u0E33\u0E0A\ + \u0E32\u0E15\u0E34\u0E1B\u0E23\u0E30\u0E40\u0E20\u0E17\u0E2B\u0E19\u0E36\u0E48\ + \u0E07\u0E15\u0E32\u0E21\u0E08\u0E31\u0E01\u0E23\u0E1E\u0E23\u0E23\u0E14\u0E34\ + \u0E41\u0E25\u0E30\u0E04\u0E27\u0E32\u0E21\u0E2A\u0E31\u0E21\u0E1E\u0E31\u0E19\ + \u0E18\u0E4C\u0E02\u0E2D\u0E07\u0E40\u0E02\u0E32\u0E01\u0E31\u0E1A\u0E04\u0E32\ + \u0E21\u0E34" + - input_choice_list: + A: "\u0E15\u0E33\u0E23\u0E32\u0E1E\u0E34\u0E18\u0E35\u0E01\u0E23\u0E23\u0E21" + B: "\u0E15\u0E33\u0E23\u0E32\u0E1B\u0E23\u0E31\u0E0A\u0E0D\u0E32" + C: "\u0E40\u0E1E\u0E25\u0E07\u0E2A\u0E27\u0E14" + D: "\u0E40\u0E23\u0E37\u0E48\u0E2D\u0E07\u0E23\u0E32\u0E27\u0E15\u0E49\u0E19\ + \u0E01\u0E33\u0E40\u0E19\u0E34\u0E14" + input_correct_responses: + - B + input_question: "\u0E2D\u0E38\u0E1B\u0E19\u0E34\u0E29\u0E31\u0E17\u0E21\u0E35\u0E25\ + \u0E31\u0E01\u0E29\u0E13\u0E30\u0E2D\u0E22\u0E48\u0E32\u0E07\u0E44\u0E23?" +include: _continuation_template_yaml +process_docs: !function utils.process_docs_world_religions +tag: mmlu_th_llama_humanities_tasks +task: mmlu_th_llama_world_religions +task_alias: world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/logiqa2/README.md b/lm-evaluation-harness/lm_eval/tasks/logiqa2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a93054011b1baabd9d3a1b11afd90649d6c2e013 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/logiqa2/README.md @@ -0,0 +1,52 @@ +# LogiQA 2.0 + +### Paper + +LogiQA 2.0 — An Improved Dataset for Logical Reasoning in Natural Language Understanding https://ieeexplore.ieee.org/document/10174688 + + +The dataset is an amendment and re-annotation of LogiQA in 2020, a large-scale logical reasoning reading comprehension dataset adapted from the Chinese Civil Service Examination. This new version has an increased data size, the texts are refined with manual translation by professionals, and improved by removing items with distinctive cultural features like Chinese idioms. + +Furthermore, a two-way natural language inference (NLI) task is introduced, resulting in 35k premise-hypothesis pairs with gold labels, making it the first large-scale NLI dataset for complex logical reasoning + +Homepage: https://github.com/csitfun/LogiQA2.0 + +### Citation + +```bibtex +@ARTICLE{10174688, + author={Liu, Hanmeng and Liu, Jian and Cui, Leyang and Teng, Zhiyang and Duan, Nan and Zhou, Ming and Zhang, Yue}, + journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing}, + title={LogiQA 2.0 — An Improved Dataset for Logical Reasoning in Natural Language Understanding}, + year={2023}, + volume={}, + number={}, + pages={1-16}, + doi={10.1109/TASLP.2023.3293046}} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet + +#### Tasks + +* `logiqa2_zh`: The original dataset in Chinese. +* `logiqa2_NLI`: The NLI version of the dataset converted from the MRC version. +* `logieval`: Prompt based; https://github.com/csitfun/LogiEval + +NOTE! The subtasks have not been verified yet. + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? + * [x] The original paper does not. There is another implementation of this task, but it designed for instruction tuned models: https://github.com/csitfun/LogiEval + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/logiqa2/logieval.yaml b/lm-evaluation-harness/lm_eval/tasks/logiqa2/logieval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f83f274b658341c2b1f8685f47138f84d5830a82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/logiqa2/logieval.yaml @@ -0,0 +1,29 @@ +task: logieval +dataset_path: baber/logiqa2 +dataset_name: logieval +output_type: generate_until +training_split: train +test_split: test +# Instructions + {content} +doc_to_text: "Instructions: You will be presented with a passage and a question about that passage. There are four options to be chosen from, you need to choose the only correct option to answer that question. If the first option is right, you generate the answer 'A', if the second option is right, you generate the answer 'B', if the third option is right, you generate the answer 'C', if the fourth option is right, you generate the answer 'D'. Read the question and options thoroughly and select the correct answer from the four answer labels. Read the passage thoroughly to ensure you know what the passage entails.\n{{content}}" +doc_to_target: "{{ideal}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true +generation_kwargs: + do_sample: false +num_fewshot: 1 +filter_list: + - name: "get-answer" + filter: + - function: "regex" + # starts with A-D excluding leading spaces + # original implementation uses a.startswith(b) + # https://github.com/openai/evals/blob/305b237cdb3884c7ddb6a5d12cb184a83551fcba/evals/api.py#L84 + regex_pattern: "^\\s*([A-D])" + - function: "take_first" +metadata: + version: 0.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/logiqa2/utils_logiqa2.py b/lm-evaluation-harness/lm_eval/tasks/logiqa2/utils_logiqa2.py new file mode 100644 index 0000000000000000000000000000000000000000..8d88e361e4a96401f2c5ce022c565673d196889c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/logiqa2/utils_logiqa2.py @@ -0,0 +1,27 @@ +# Copied from Master +def doc_to_text(doc) -> str: + """ + Passage: + Question: + A. + B. + C. + D. + Answer: + """ + choices = ["a", "b", "c", "d"] + prompt = "Passage: " + doc["text"] + "\n" + prompt += "Question: " + doc["question"] + "\n" + for choice, option in zip(choices, doc["options"]): + prompt += f"{choice.upper()}. {option}\n" + prompt += "Answer:" + return prompt + + +# # https://github.com/csitfun/LogiQA2.0/blob/main/logiqa2nli/nli-prompt.py +# def doc_to_textNLI(doc): +# maj_premise = ' '.join(list(doc['major_premise'])) +# min_premise = ' '.join(list(doc['minor_premise'])) +# hypo = doc['conclusion'] +# prompt_input = "Given the fact: " + maj_premise + ' ' + min_premise + " Does it follow that: " + hypo + " Yes or no?" +# return prompt_input diff --git a/lm-evaluation-harness/lm_eval/tasks/longbench/2wikimqa.yaml b/lm-evaluation-harness/lm_eval/tasks/longbench/2wikimqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1d1791b6716253c300bcbb4701128a9961a38ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/longbench/2wikimqa.yaml @@ -0,0 +1,21 @@ + +tag: + - longbench +task: longbench_2wikimqa +dataset_path: THUDM/LongBench +test_split: test +dataset_name: 2wikimqa +doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score +generation_kwargs: + max_gen_toks: 32 + temperature: 1 + do_sample: True + until: [] +metric_list: + - metric: "qa_f1_score" + aggregation: mean + higher_is_better: True +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/longbench/dureader.yaml b/lm-evaluation-harness/lm_eval/tasks/longbench/dureader.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e001f349e4b7750c1ba91281447161c247c7825b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/longbench/dureader.yaml @@ -0,0 +1,21 @@ + +tag: + - longbench +task: longbench_dureader +dataset_path: THUDM/LongBench +test_split: test +dataset_name: dureader +doc_to_text: '请基于给定的文章回答下述问题。\n\n文章：{{context}}\n\n请基于上述文章回答下面的问题。\n\n问题：{{input}}\n回答：' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_zh_score +generation_kwargs: + max_gen_toks: 128 + temperature: 1 + do_sample: True + until: [] +metric_list: + - metric: "rouge_zh_score" + aggregation: mean + higher_is_better: True +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/longbench/gov_report.yaml b/lm-evaluation-harness/lm_eval/tasks/longbench/gov_report.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76307371574948b03daa548142a4eb5fc5957c39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/longbench/gov_report.yaml @@ -0,0 +1,21 @@ + +tag: + - longbench +task: longbench_gov_report +dataset_path: THUDM/LongBench +test_split: test +dataset_name: gov_report +doc_to_text: 'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{{context}}\n\nNow, write a one-page summary of the report.\n\nSummary:' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score +generation_kwargs: + max_gen_toks: 512 + temperature: 1 + do_sample: True + until: [] +metric_list: + - metric: "rouge_score" + aggregation: mean + higher_is_better: True +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/longbench/lsht.yaml b/lm-evaluation-harness/lm_eval/tasks/longbench/lsht.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4343413b62882a2d2275a7ca29455bf149ace547 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/longbench/lsht.yaml @@ -0,0 +1,21 @@ + +tag: + - longbench +task: longbench_lsht +dataset_path: THUDM/LongBench +test_split: test +dataset_name: lsht +doc_to_text: '请判断给定新闻的类别，下面是一些例子。\n\n{{context}}\n{{input}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_classification_score +generation_kwargs: + max_gen_toks: 64 + temperature: 1 + do_sample: True + until: ["\n"] +metric_list: + - metric: "classification_score" + aggregation: mean + higher_is_better: True +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/longbench/multi_news_e.yaml b/lm-evaluation-harness/lm_eval/tasks/longbench/multi_news_e.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62f4405360bda431126e4d6004b0445e5705e695 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/longbench/multi_news_e.yaml @@ -0,0 +1,21 @@ + +tag: + - longbench_e +task: longbench_multi_news_e +dataset_path: THUDM/LongBench +test_split: test +dataset_name: multi_news_e +doc_to_text: 'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{{context}}\n\nNow, write a one-page summary of all the news.\n\nSummary:' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score +generation_kwargs: + max_gen_toks: 512 + temperature: 1 + do_sample: True + until: [] +metric_list: + - metric: "rouge_score" + aggregation: mean + higher_is_better: True +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/longbench/multifieldqa_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/longbench/multifieldqa_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a6eb9ed5ca4662fd55348dc43be7ba2170bb348 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/longbench/multifieldqa_zh.yaml @@ -0,0 +1,21 @@ + +tag: + - longbench +task: longbench_multifieldqa_zh +dataset_path: THUDM/LongBench +test_split: test +dataset_name: multifieldqa_zh +doc_to_text: '阅读以下文字并用中文简短回答：\n\n{{context}}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{{input}}\n回答：' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_zh_score +generation_kwargs: + max_gen_toks: 64 + temperature: 1 + do_sample: True + until: [] +metric_list: + - metric: "qa_f1_zh_score" + aggregation: mean + higher_is_better: True +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/README.md b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..666d6446b108afaa2991ec8b3d6921ecc2b704d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/README.md @@ -0,0 +1,49 @@ +# MedConceptsQA + +### Paper + +Title: `MedConceptsQA: Open Source Medical Concepts QA Benchmark` + +Abstract: https://arxiv.org/abs/2405.07348 + +MedConceptsQA is a dedicated open source benchmark for medical concepts question answering. The benchmark comprises of questions of various medical concepts across different vocabularies: diagnoses, procedures, and drugs. + +The questions are categorized into three levels of difficulty: easy, medium, and hard. + +Our benchmark serves as a valuable resource for evaluating the +abilities of Large Language Models to interpret medical codes and distinguish +between medical concepts. + +### Citation + +``` +@article{shoham2024medconceptsqa, + title={MedConceptsQA--Open Source Medical Concepts QA Benchmark}, + author={Shoham, Ofir Ben and Rappoport, Nadav}, + journal={arXiv preprint arXiv:2405.07348}, + year={2024} +} +``` + +### Groups and Tasks + +#### Groups + +* `med_concepts_qa`: Contains all the QA tasks (diagnosis, procedures ,and drugs). + +#### Tasks + + +* `med_concepts_qa_icd9cm` - ICD9-CM (diagnosis codes, ICD9 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-9-CM (International Classification of Diseases, 9th Revision, Clinical Modification) diagnosis codes. + + +* `med_concepts_qa_icd10cm` - ICD10-CM (diagnosis codes, ICD10 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-10-CM (International Classification of Diseases, 10th Revision, Clinical Modification) diagnosis codes. + + +* `med_concepts_qa_icd9proc` - ICD9-Proc (procedure codes, ICD9 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-9-PCS (International Classification of Diseases, 9th Revision, Procedure Coding System) procedure codes. + + +* `med_concepts_qa_icd10proc` - ICD10-Proc (procedure codes, ICD10 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-10-PCS (International Classification of Diseases, 10th Revision, Procedure Coding System) procedure codes. + + +* `med_concepts_qa_atc` - ATC (Anatomical Therapeutic Chemical Classification System) question-answering. This involves providing information, clarifications, and answering questions related to the ATC classification system, which is used for the classification of drugs and other medical products according to the organ or system on which they act and their therapeutic, pharmacological, and chemical properties. diff --git a/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b12ea811ff26631e65c9dd9cb42b56b1c0c7dba1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd9cm +task: + - med_concepts_qa_icd9cm_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..514a9e258614aaa29e32f2c5bc3d54318175d837 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_easy +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_easy +task_alias: icd9cm_easy diff --git a/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90f93f9bc2b520186a98c923dd7cf49d58a5293b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_medium +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_medium +task_alias: icd9cm_medium diff --git a/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..843029209916822a306e11e2795568a757cc4b2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_medium +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_medium +task_alias: icd9proc_medium diff --git a/lm-evaluation-harness/lm_eval/tasks/med_prescriptions/utils.py b/lm-evaluation-harness/lm_eval/tasks/med_prescriptions/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1d68a10e55836493b0cacbae348c45f0126d4d64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_prescriptions/utils.py @@ -0,0 +1,2365 @@ +import ast +import random +import re + +import datasets + + +full_med_list = [ + "Cap Pregabalin, before breakfast and dinner, 1 week", + "Etoshivic 90mg, before breakfast, 1 month", + "Lyse, before breakfast, 1 month", + "EBAST M TAB, after food, 15 days", + "AZEE 500MG TAB, after food, 5 days", + "VOLTOP DSR TAB, before food, 5 days", + "ASCORIL D PLUS SYP, after food, 5 days", + "Sup. Afinal SR", + "ECOSPIRIN 75mg, daily, 3 months", + "OMEZ 20mg, daily, more than 1 month", + "TAMDURA OD, daily, more than 3 months", + "TELMA-CT 40/6-25, daily, 3 months", + "Col, once a day, 4 days", + "Syrup Allegra, once a day, 4 days", + "Atarax liquid, 1-0-1, 12 nights", + "Atogla Cream, 1-0-1, 15 days", + "Mox Mar 17, 1-0-1, 7 days", + "Paracetamol, 3 times a day, 6 days", + "Tassolure, 1 time a day, 6 days", + "Taloplex, 1 time a day, 6 days", + "Amoxicillin 625 mg, after breakfast and after dinner, 4 days", + "NA", + "NA", + "NA", + "Gabapin 300, before breakfast and before dinner, 20 days", + "Hosit Ds, before breakfast and before dinner, 20 days", + "T. Rifametrosy, x 5b", + "T. Anxipan", + "Ramitorva, bbf", + "Nerve-D", + "Glucobay M (25/500), al ad", + "Dapanormis (10/100), abf", + "Pioz (7.5), bbf", + "Cyblex M (80), bb, bl, bd", + "Syrup Paracetamol, morning, afternoon, night, 5 day(s)", + "Syrup LactobacillusRiboflavin, morning, afternoon, night, use if there is a need", + "Syrup Metanemic Acid, night, 3 day(s)", + "Syrup, morning, afternoon, night, 3 day(s)", + "Syrup Fexofenadine, morning, night, after food", + "PAN 40, morning and night, 2 months", + "Elsoud, night, 2 months", + "Ado, morning and night, 2 months", + "PAN 40, morning and night, 3 to 4 days", + "paracetamol, morning and night, 3 to 4 days", + "Guckelar spen, �5 /ar", + "Rmal Sy, x5la", + "Zincilokal. SI, e- e gez", + "Tab Letsi 2.5 mg, � 5 days", + "Tab. MCBM69, � 1 month", + "Teb. Ecosprin 75mg, � 1 month", + "Teb. Thyronoen 12.5 mg, � 1 month", + "Tab. Metformin 500 mg, � 1 month", + "Syf. Acima10-200, od", + "maltill, sos", + "Calpol, morning and night", + "Mucolite, morning and night", + "Advent, morning and night, 14/11/22", + "Tab. Thursmm, 2 times a day, 10 days", + "COD LOTTAti2, once a day, 10 days", + "PAN 40, before breakfast, 10 days", + "BP 4140/90m.R, 1-0-1", + "PR 7 70imim", + "For USG- Abdominal Pelvis", + "Syrup Cetalore-M, twice a day, 10 days", + "Drop Mosi (Eye), three times a day, 5 days", + "Syrup Selzita, twice a day, 3 weeks", + "Cap. Itaspor 100mg, morning and night, x 30 days", + "Tab Dazit 5mg, morning and night, x 30 days", + "Zedouf (200, x 8 days", + "Tab Montamarc", + "Aerodie, x 21 days", + "Sup, -(10) x 2days.", + "Tab Fepanil 650", + "Budesal (0.5mg), 1 � 5 days", + "Volini Gel, twice a day, 1 month", + "Ibuprofen, thrice a day, 1 week", + "PAN 40, morning and night, 30 days", + "Renova, morning and night, 60 days", + "VoliDer, morning", + "Pan 40, morning and night", + "Rosalix, morning and night", + "NA", + "PAN 40, before breakfast, before dinner, 3 weeks", + "Tab Gement, before breakfast, before dinner, 1 month", + "VILDAPHAGE-M, morning and night, 120", + "GRYCIPAAGE & 2, morning, afternoon, night, 3", + "AMLOSAFE-AT, morning, 60", + "As- lastochandile. ?, sos ypor or shle, brugal la 1-07 � 5day", + "Random 4ml-0-4aux stay, blunt d3 sachet r, 1 daily x sday", + "Clinic: 112, First Floor, Vikas Galaxy, Station Road, Sanewadi, Badlapur (W) M .: 8390268487", + "SHIKO, 3 times a day", + "PAN 40, morning and night", + "Velten 04mg, before lunch", + "ECOSPRIN 150 MG, after food - daily, 4 weeks", + "ROTACIUM, after food - daily, 6 weeks", + "SUPRADYN, after food - daily, 6 weeks", + "VIT D 60 K, after food - weekly, 6 weeks", + "PROSCO DHA, daily", + "SCOR HB, after food - daily", + "Lubiprostone, once daily, 30 days", + "Aprepitant, once daily, 30 days", + "Sildenafil, once daily, 30 days", + "Voglibose, once daily, 30 days", + "Jardiance, once daily, 30 days", + "Metformin, twice a week, 30 days", + "Pregabalin, once daily, 30 days", + "Insulin, before breakfast and before dinner, 30 days", + "Betnovate, apply twice daily, 30 days", + "PAN 40, morning, before breakfast, 10 days", + "DOLO 650, morning, afternoon, night, 5 days", + "AZITHRAL 500, morning, night, 3 days", + "Monocor-1, b.d (twice a day), 14 days", + "Arocon 500, 1x1 (once a day)", + "Rabe DSR, 1x2 (once in the morning and twice at night)", + "PAN 40, 1-0-1, 28 days", + "Pan 40, morning and night, 5 days", + "Calcium, 1-0-1, 30 days", + "Cachar, 1-0-1, 7 days", + "Rl olocal, 2, 4", + "Pal olocal, 8, 23", + "T. Augmentin 675 mg, 5 days", + "T. Enzoflam, 5 days", + "NA", + "NA", + "NA", + "NA", + "NA", + "NA", + "Toto. Fluconazol 100mg� 5 days., candid suppel - &l, 3 days", + "NA", + "NA", + "JU-CRiXAN (250) 100, bd, 5 days", + "Jal. DEFWORT (6) 107, bd, 10 days", + "Ta. MUCiNAT- AB 10, bd, 10 days", + "Tal. LUKOTAS- HD 100, bd, 10 days", + "Syp. LUPITUSS, bd, 5 days", + "PAN 40, 3 times a day, 2 months", + "POP", + "Sugar", + "Surgery", + "PAN 40, morning and night, 14 days", + "Saazole, morning and night, 14 days", + "Celen 22000, morning and night, 14 days", + "RB NB 4911", + "coolora cables, 3 times a day", + "montina-Fx, 12 days", + "pulmones 250, 6 days", + "Sup. Rapitus plus", + "Tab. PriJa-", + "Tab. vreit -200", + "NA", + "PAN 40, morning and night, 5 days", + "Paracetamol, morning and night, 5 days", + "Amoxicillin 250mg, morning and night, 5 days", + "Argesic -1, morning and night, 100", + "Glycinhage (500), 2 times a day, 1 month", + "Diaberon, 2 times a day, 1 month", + "D-fix, 1 time a day, 1 month", + "Ultracet, 1 time a day, 20 days", + "Lipiland - F, 1 time a day, 1 month", + "Ade P-650, 1 time daily, 30 days", + "Tab ACOGURD NT, 1 time daily, 30 days", + "T3 LC, 1 time daily, 30 days", + "D-360, 4 times a day, 30 days", + "HAcorpus + NANOFAST FEL", + "NA", + "Pan 40, morning and night, 1 week", + "San 52, morning and night, 1 week", + "Calapiso liver, morning and night, 1 week", + "Abronzo pasda, morning and night, 1 week", + "PAN 40, morning, 7 days", + "Paracetamol, morning and night, 3 days", + "Antibiotics, morning and night, 7 days", + "scor hb plus, 4 week(s)", + "radibone, 4 week(s)", + "NA", + "PAN 40, morning and night, 1 week", + "Nab Duolic, night, 1 week", + "Aze 50mg, night, 1 week", + "Mantar Le, morning and night, 1 week", + "Pan-D, morning and night, 1 week", + "Sy, morning and night, 1 week", + "Auchakind, morning and night, 1 week", + "SYP IBUGESIC PLUS 45, bbf, 5 days", + "SYP MEFTAL P, bbf", + "SYP MAXTRA, bbf", + "SYP KUFRIL LS, bbf", + "SYP ONDEM, bbf", + "VIAL ENTEROGERMINA", + "Arteliar, 1-0-1, 14 days", + "Cafin Dehave, 1-0-0, 7 days", + "PENTANE, 1-0-1, 30 days", + "Tab. Folineo, once a day, for intafol-d far", + "Tab. Dychease, once a day, ongoing", + "Tab. Ecosprin 75, once a day, for 15 days", + "Tab. Rasicap 100, once a day, ongoing", + "Tab. Calshine, once a day, from start of treatment", + "Protein powder, once a day", + "Tab. Imumust, once a day, for 10 days", + "Syp-Digecaine, twice a day, for 18 days", + "Tab. Condibiotic, once a day", + "Richglow lotion", + "Tab. Tidilan, once a day", + "Pan 40, after meal, 7 days", + "Paracetamol, after meal, 3 days", + "Tab. CCM, before meal, 14 days", + "Colifino, twice a day, 10 days", + "Calivol Softgel, once a day, 8 days", + "Livogen-2, once a day, 2 months", + "Pan 40, once a day, indefinite", + "Zenha, before bedtime, indefinite", + "Apelo poglute", + "TT 0.5 ml", + "ARV, 0, 3, 7, 14, 28", + "R.B.S (28)", + "Blood Sugar", + "BP-125/77 mmH1", + "PR-86/min", + "Hb-11jm, 25/02/23", + "Calpsor C Ointment, 1-0-1, 3 week(s)", + "Allegra 180 Tablet, 1-0-1, 3 week(s)", + "Venusia Max Cream, 1-0-1, 3 week(s)", + "IV. R.L .O I, morning and night", + "Iv. Auguriurl (1.2m) IB, morning and night", + "IV. M.N. I IO, morning and night", + "IV. oflox(00m) IO., morning and night", + "TAB AZILLUP 500 / AZOTOX 500, morning and night", + "TAB ORTHOTIME BR, morning", + "OINT CHETOMESH SF, as per requirement", + "TAB VITAPLUS, after lunch", + "Im Duvadilon, 10, for days", + "Cap Lanx 00, 5", + "VILDAPHAGE-M, 1-0-1, 120 tablets", + "AMLOSAFE, 1-0-0, 60 tablets", + "Cosader, 1-0-1, 120 tablets", + "Ostosline, 1 week, 10 days", + "Zerodol CR, as needed", + "Gabancuran 100mg, daily", + "Pau 40mg, daily, 10 days", + "Prochoscary, as required, not specified", + "cox gel v-V, once daily, not specified", + "Anoxlief dialiment, renew after 3 weeks, not specified", + "UPRISE D3 GUK, once daily", + "D3 60 K, once daily", + "Tab 924, every evening", + "Tenovate cream, 3 weeks", + "Tab. HCQS (400g), 3 weeks", + "Mantoux test", + "Quantiferon - TB Gold test", + "Tar lepodem, 1-0-1, 7 days", + "T. Sompraz 40mg, morning, night, 3 days", + "T. CINTARRO, night, 4 days", + "Sucafel-o, morning, 3 days", + "NORMAXIN, morning, afternoon, night, 7 days", + "Tab. Nejor, bbf, 1 month", + "Tab. Defical, cair, 1 month", + "Ij Osteo D3, stat, 1 month", + "Cariminic Syrup, td, 3 days", + "Sumol, td, 3 days", + "Lactovil, td, 10 days", + "T SOMPRAZ L, before food 7days", + "BIFILAC GG, 10 days", + "Glycomet, bbf, 10 days", + "Glimore-M, bbf, 10 days", + "Tenegem-M, 001, 6 months", + "Telmanos-MT, 001, 6 months", + "Dafine, al, 10 days", + "PAN 40, morning and night, 14 days", + "Paracetamol, morning and night, 3 days", + "Telmikind 20 Mg, morning, night", + "Gliminyle M 1 Tab, before meals", + "Dia-Pace (Metformin) 500 mg, after lunch", + "Bixro 16 Mg (Betahistine), morning, afternoon, night", + "Pantapace D, before food", + "Gabacrine M (Gabapentine 300 mg + Methyl Cobalamine 500), after dinner", + "Tab SITADAPA - 10/100, 1-0-0, 3 months", + "Tab CALCIBUS, 1 pill daily, 3 months", + "Tab DRISE (VIT D), 1 pill per month, 3 months", + "Suma-ES (250), 3 times a day, 3 days", + "Mestat P 12.5 mg, 3 times a day, 3 days", + "Meospann-H Ointment, 5 times a day", + "Augmentin ES (Amox 600mg), 3 times a day, 7 days", + "PAN 40, morning and night, 5 days", + "Ocupar Dy cream, as required", + "Lys Mega ce for, before breakfast, 5 days", + "Thromboprob oil, 2 times a day, for 4 days", + "Nadoxin cream, as prescribed", + "Tmedlar, once daily, 14 days", + "Zerofel-sp, as prescribed", + "72 DM, daily", + "PAN 40, daily", + "GUILD BITOSTRI CODITID", + "Tab. Daylen, twice daily, 42 days", + "SANSTICS - HEALTHIER LIVING, not specified, not specified", + "NovoRapid SL, morning, afternoon, night", + "Levemir, night", + "Dielie Consusane, morning, afternoon, night", + "T. pregabid-ME 75mg, after meal, 2 weeks", + "T. Acesse 100 mg, before meal, 2 weeks", + "Syrup Atarax, twice a day, 5 days", + "Cream Flutivate (Skin), twice a day, 5 days", + "Calibnie, 1 mg daily, 6 months", + "PAN 40, before breakfast and before dinner", + "DIGENE, before breakfast", + "SURIC ACID", + "Tab. Folite 5mg, daily, 30 days", + "Tab. Folitrax 2.5 mg, daily, 30 days", + "Cap. Pan-D, before meals, 30 days", + "Tab. Ricorsia, after meals, 30 days", + "Tab. Saaz DS, morning and night, 30 days", + "Tab. Nurokind OD, morning and night, 30 days", + "Tab. Product 4, sunday off, 30 days", + "Tab. Orcerin 6M, monday and wednesday, 30 days", + "Tab. Celin 500, monday and wednesday, 30 days", + "Tab. Ecosprin 150, sunday off, 15 days", + "Tab. Ecosprin Gold 10/75, daily, 30 days", + "Tab. Medvol 2mg, at night, 30 days", + "Tab. Pan - D, before dinner, 30 days", + "Cap. Ecosprin 75, sunday off, 30 days", + "Tab. Orcerin 6M, monday, wednesday, and friday, 30 days", + "Tab. Celin 500, monday, wednesday, and friday, 30 days", + "Ranitidine, 1-0-0, 6 days", + "Paracetamol, 1-0-1, 5 days", + "Azithromycin, 0-1-0, 3 days", + "Suspension Drotin DS, every 8 hours, 5 days", + "Tablet Lanspro 15 mg, once a day, 5 days", + "Syrup Tummy Soft, twice a day, 5 days", + "PAN 40, after food, 7 days", + "Paracetamol, after food, 3 days", + "Tab. Bifolate, 0-0-1, x continue.", + "Tab. Doxinate, 1-0-1, x continue.", + "CIFRAN 500MG TAB, bd (twice daily), 1 month", + "RELENT PLUS 8J, bd, 5 days", + "RANTAC 88, od", + "Tab. AlfsN, bd", + "Tab Octobix, bd", + "Tab Ler, bd", + "Tab Pzae, bd", + "Tab Euognix, bd", + "Mamadialitu con, bd", + "Ag preg, bd", + "- FRS/2hPP BS, bd", + "PENTAXIM, opd", + "PRIORIX, opp", + "MEASLES, MUMPS, AND RUBELLA VACCINE (LIVE) IP, 6 hourly for fever >99�f", + "ZOFRAN, 1�15 days", + "ZINCOVIT ZINCITOTAL, 3 months", + "cox gel v-V, once a day, 3 weeks", + "Anoxlief dialiment-0-, not specified, renew after 3 weeks", + "Tab Martifur MR 100 mg, after food - daily - 4 days", + "Novefos Sachet 3 gm, bed time - single dose", + "Tab Urispas, after food - sos", + "Syp Alkasol, after food - daily - 3 days", + "Hifenac-SP, before food, 10 days", + "Pan-10, before food, 10 days", + "Tolyb/Chymoral forte, after food, 10 days", + "Resner plus | Fibrogenic-N/Max-mala-NT", + "FOLSAFE-L, 30 days, 4 to 5 months", + "Tab. Lipi 200, 6", + "Tab. Recotar, 6", + "Cap. Ormed 20, 6", + "FLUVIR 75MG TABLET, after food - daily - 5 days, 5 days", + "CALPOL 650MG TABLET, 1-1-1, 3 days", + "XYZAL M TABLET, 0-0-1, 5 days", + "REBEZ DSR CAPSULE, 1-0-0, 5 days", + "Tab Laregas (300), morning and night", + "Tab Nauto (S), morning", + "Tos Remae CP3 101, morning and night", + "Tab Cryopan DSR 101, morning", + "Ade P-650, bd, 30 days", + "ACOGURD NT, bd, 30 days", + "T3 LC, bd, 30 days", + "D-360, bbf, 4 weeks", + "HAcorpus, bd, 30 days", + "NANOFAST FEL, bd, 30 days", + "Aceclo plus, morning and night", + "Gamot, night, 10 days", + "Tendojoy, morning and night", + "PAN 40, morning and night, 6.1", + "Paracetamol, after breakfast and before dinner", + "Xiosoy, before breakfast", + "Avil 25, once daily, 5 days", + "PanDro, once daily, 7 days", + "Tab TENOFOVIR ALEFANAMIDE 25mg, to continue", + "SOL PNA", + "HEC", + "DIABETES CARE", + "INSULIN LANTUSS", + "ILECOSPRIN AV 75", + "TELFAST 40", + "TABAK VITE S", + "CIPCAL 500", + "TAB.RYBELSUS 14 MG, 1 -- 0 -- 0, 30 days", + "TAB.GLEDEPA 10 MG/OXRA 10 MG, 1 -- 0 -- 0, 30 days", + "TAB.GLYCOMET SR 500 MG, 0 -- 0 -- 1, 30 days", + "TAB.SYMBAL (30MG), 0 -- 0 -- 1, 30 days", + "CAP.RABONIK DSR, 0 -- 0 -- 1", + "TAB.RAZEL F 5 MG, 0 -- 0-(1), 30 days", + "R MEGANEURONE OD, daily, 30 days", + "SYP.TOPUP D3 Cholecalciferol 60000IU, after 4 days", + "Sumaalla Jamba, 15 days", + "Nasal Spray, bd (twice daily), 15 days", + "Antacid Plus 200, 5 days", + "Inj Bett, stat", + "PAN 40, morning and night, 4-5 days", + "Paracetamol, morning and night, 4-5 days", + "Azithromycin, morning and night, 4-5 days", + "Ecosprin AV, od", + "Night spadives", + "T. CLOPILET-A 75/175", + "T. ROSLOY-F 20/160", + "T. BUERT OD 24", + "T. RAZO-A 20/10", + "T-AMLOKIND AT 5/150", + "Greenor, before breakfast and dinner", + "ACITRON, before breakfast, 3 days", + "GING, after dinner, 3 days", + "ACITROM (4 mg), 6pm", + "TELMA AM (UDS), after dinner", + "Solopart (8), 10pm", + "Atarax 5mol-5001-5ml, 1-0-1, 1 month", + "Syp- Goun DS/ Campol -250, 1-0-1", + "Syp Bevon, 0-1-0, 1 month", + "NA", + "NA", + "NA", + "Cepodem XP 325 Tablet, 5 days", + "Nimus P Tablet, 3 days", + "Ebast M Tablet, 5 days", + "Corex DX Syrup, 5 days", + "CLAVAM (Augmentin - 625), twice a day, 15 days", + "DOWO-650, twice a day", + "LBC", + "Evion 400mg, morning, afternoon, night, 2 months", + "Dolo 650, morning, afternoon, night, 3-4 days", + "Primosa 100, morning, night, 2 months", + "Etoshine MR, twice daily, 5 days", + "Paracetamol, once daily, 3 days", + "GALOC, bd", + "corred, al", + "Tab Aspirin, bbf, 30 day", + "PAN 40, morning and night, 14 days", + "Paracetamol, morning and night, 3 days", + "Syp. CREMAFFIN, bd, 5 days", + "CONZAFIT, 1-0-1", + "PAN 40, 1-0-1", + "ONVISTA, 0-0-1", + "PAN 40, 1-0-1, 10 days", + "Crocin, 3 times a day, 5 days", + "Azithromycin, 1-0-0, 3 days", + "Parin, before breakfast", + "Tab. Mysolvo, before breakfast", + "Ta Martinus, before breakfast", + "syp andem sil som, calimesa sul, po �5 days.", + "syp", + "Syp, � 1 week", + "FOLSAFE-L, once daily, 30 days", + "Generic Medicine, once a day, 3 days", + "Eye Drops, three times a day, 7 days", + "Tu MICROBACT 500mg, 0-62", + "SIGNOFLAM", + "SOMPRAZ 20mg, 0-52", + "PAN 40 (Pantosec), before food, 5 days", + "Colafix, after food, 5 days", + "Olocal, after food, 5 days", + "Lanol ER, 1 - 0 - 1, 5 days", + "Pushan D3 (60K), 5 ml - once a week, 10 weeks", + "Pan 40, before breakfast and dinner, 6 days", + "Sommore, morning and night, 12 months", + "Upini D3, before dinner, 3 months", + "Levetiracetam 500 mg, morning and night, 6 months", + "Alzil-M Forte, morning and night, 6 months", + "T.100.2, in the morning", + "p-250 sJp, per day", + "Ambrail ph", + "Remy celles", + "compt 100", + "Demander Alle", + "BANH 81", + "Tab. Stemelia B, bd", + "Ventyr 1 Sp", + "Das Rovastat 10mgts", + "Cap Alten Aslan", + "Das Wetrans", + "PAN 40, before breakfast, before dinner, 3 months", + "AKT-4", + "Zincovon", + "Dicorate-ER 500mg, 1-0-0, 1 month", + "Mebodep-CD3, 1-0-0, 1 month", + "Tryptomer-25mg, 0-0-1, 1 month", + "Writex 7G, 0-0-1, 1 month", + "Napra D 500mg, sos, 9", + "HEPR, daily, 3 days", + "BIOVAC 1 Chocin DS, daily, 3 days", + "ALP, 1, 10 days", + "MR, 1, 3 days", + "Tota, 1, 6 days", + "Tab Nxfor Sp 10, 1, 10 days", + "T-3", + "NA", + "NA", + "Go-calm 250, before meals, 5 days", + "Ses Admit 457, after meals, 14 days", + "dis Bnsector, 3 times a day, 5 days", + "SISCOX TH TAB 10S, oral, 10 day", + "SOMPRAZ 40MG TAB 15S, oral, 10 day", + "NERVMAX SR 75 TAB, oral, 10 day", + "SYSTAFLAM 50GM GEL, oral, 30 day", + "Pan 40, 1-0-0, 7 days", + "Dolo 650, 1-0-1, 3 days", + "Augmentin 625, 1-0-1, 5 days", + "PARACETAMOL, 5 days", + "ZINC SUPPLEMENT, 5 days", + "COUGH SYRUP, 5 days", + "PAN 40, morning, 5 days", + "Rantac, bd (before dinner)", + "Osyp. Random PD, tds (three times a day)", + "Blo 50 5mg, tds (three times a day)", + "Ibugesic, prn (as needed)", + "@syr. capot (250), prn (as needed)", + "syp. Rady (200), qhs (before bedtime)", + "Pan 40, after breakfast", + "Ady, after lunch", + "Colfor ps, after dinner", + "Sebisher Dermalils, 2, 7", + "Teb. Trthustive, 2, 5", + "Slikkoby lacion D, 1, 5", + "c-wey shampoo, 1", + "candles 2/c", + "PAN 40, bd (before dinner), 7 days", + "PCM 500, tds (three times a day), 7 days", + "Suspension Ibugesle Plus, 6 hourly (10 ml - 10 ml - 10 ml - 10 ml), 3 days", + "Suspension Meftal P (60 ml), 10 ml, sos at fever before 6 hrs of ibugesic", + "Jij Methy Cobal, bbf", + "Das Deterrol, bbf", + "Vice-M (500), bbf", + "Das Stalit-D, bbf", + "Das Enite (40), bbf", + "DIVYA CHIRAYTA KWATH 100 GM, morning and evening, 30 days", + "DIVYA GILOY KWATH 200 GM, 1 hour before meal, 30 days", + "DIVYA SARVAKALP KWATH 100 GM, 30 days", + "DIVYA IMMUNOGRIT 60 N 33 OM, 30 minutes before breakfast/lunch/dinner", + "DIVYA MADHUNASHINI VATI EXTRA POWER 60 GM, 30 minutes after breakfast/lunch/dinner, 30 days", + "DIVYA MADHUGRIT TABLET 60 N 38 GM, 30 minutes after breakfast/lunch/dinner, 30 days", + "DIVYA TRIPHALA GUGGUL 40 GM, 30 days", + "PAT NUTRELA DAILY ACTIVE CAPSULE 750 MG, 30 minutes after breakfast-lunch-dinner, 30 days", + "DIVYA SHUDDHI CHURNA 100 GM, bedtime, 30 days", + "Tablet Lanol ER (650 mg), 15 days, after breakfast. after dinner", + "2 Tablet Gabamax NT 50/10, 15 days, before dinner, at 8pm", + "Gel Dolcinac Mr, 15 days, after breakfast. after dinner", + "Tablet Collacium Strong, to continue, after breakfast", + "Tablet Etody (90 mg), sos, for severe pain", + "Tablet Trioflex Tablet, 1 month, after breakfast, after dinner", + "Hidraslim lotion livice aday", + "Jab Teezie Song 505", + "Chloramphenicol, morning and night, 7 days", + "Paracetamol, morning and night, 7 days", + "Azithromycin, morning and night, 7 days", + "Boxy Boule, morning, afternoon, night", + "crepetart, morning, afternoon, night, till recovery", + "PREGALIN SR 75MG, after food - daily, 30 days", + "PAN 40, morning and night, 12 days", + "Thyrofit 100, morning, 30 days", + "Metformin 500, morning and night, 60 days", + "TELMA LN BETA 50 TABLET, after food - daily, 1 month", + "CLONOTRIL 0.25MG TABLET, after dinner - daily, 1 month", + "ALDACTONE 25MG TABLET, after food - daily, 1 month", + "Tab Allegra 180mg, hi,ac, 100� 10 day", + "RBS, (mono -10.00am)", + "Sk(2) Pab, (7to 73 pm darle), � 20 day", + "Tan - Appar 65g, 2 times a day, 18 days", + "Cap Attop- DJ-2, 1 time a day, 18 days", + "Sy citar, 1 time a day, 18 days", + "Syp Cocintus, three times a day, 7 days", + "Syp Augmentin Duo, three times a day, 7 days", + "Nasivion, as needed", + "Syp Marx, as needed", + "PAN 40, after breakfast and dinner, 3-4 days", + "Paracetamol, after breakfast and dinner, 3-4 days", + "NA", + "As plast risodaless, tamonth", + "Veagreat 10, clo: morning and night", + "Paracetamol, 1-0-1, 3 days", + "Raufen 150mg, 1 before dinner", + "Zincenit, 1 after lunch", + "Folsafe-L, daily, 30 days", + "Suspension ATM XL (200 mg) - Azithromycin, 5 days, before food", + "Suspension P 250 - Paracetamol, till required", + "Syrup Cetzine - Cetirizine, till required", + "ALBENDOL, bd, due today", + "ILOVIT NANO 60K, bd, 16th (sun) x 10 days", + "Gemal-p 2-54 RD, bd, 18/9/23", + "Suspension Meftal P (60 ml)", + "Syrup Solvin LS", + "Drop Votriz cold drops", + "Syrup Bactoclav DS", + "OKALET (5mg), morning and night, 5 days", + "ENZOCORT, morning and night, 5 days", + "CEFOLA, morning and night, 5 days", + "OTRIVINsal, morning and night, 5 days", + "SYP IBUGESIC PLUS 45 (ML/100 MG), bd, z days", + "YOUMO-PARACETAMOL 12.5A (ML/100 MG), bd, z days", + "SYP MEFTAL P (5 ML/100 MG), bd, z days", + "MEFENAMIC ACID, as per requirement, z days", + "SYP MAXTRA, as per requirement, z days", + "CPM DG-PHENYWINE SUG (2.5 ML), as per requirement, z days", + "SYP KUFRIL LS (5 ML/0.5 MG), not mentioned, z days", + "LEVOSALBUTAMOL, as per requirement, z days", + "SYP ONDEM (5 ML), not mentioned, z days", + "ONDANSETRON, as per requirement, z days", + "VIAL ENTEROGERMINA, as per requirement, z days", + "BACILLUS CLAUSE, as per requirement, z days", + "Alocepodem 100-DT, once daily, 10 days", + "Sp Asthalin, as required, -", + "Ibugeric play, as required, -", + "HHLEVO M KID 60ML SUSPENSION, after food - daily, 10 days", + "VENUSIA SOFT LOTION, after bath - daily, 2 weeks", + "ATONIDE 20GM GEL, after bath - daily, 15 days", + "OMNACORTIL ORAL SUSPENSION, after food - daily, 1 week", + "PAN 40, morning and night, 2 weeks", + "Ankle Binder / Splint, all day except shower, 2 weeks", + "Tax, before dinner", + "Mandyic party, 1-0-1, 14 days", + "MyOnR, 1-0-1, 5 days", + "pregabalin, 1-0-1, 5 days", + "Dokln, 1-0-1, 5 days", + "Clofain Lack, morning and night", + "Tab ALTRADAY, morning and night", + "Teb Shelle XT, morning and night", + "PAN 40, morning and night, 24 portand 4og livs11", + "Montop-11, morning and night", + "Tab Aceclo plus, after breakfast, after lunch", + "Tab Gamot, after dinner, 10 days", + "Tab Tendojoy, after breakfast", + "Paracetamol, twice a day, 7 days", + "Dolo, twice a day", + "Coscorit, once a day", + "Solibar, bd", + "LANOL ER, 5 days", + "SISCOX TH TAB 10S, oral, for 10 days", + "SOMPRAZ 40MG TAB 15S, oral, for 10 days", + "NERVMAX SR 75 TAB, oral, for 10 days", + "SYSTAFLAM 50GM GEL, oral, for 30 days", + "PAN 40, morning and night", + "Paracetamol, morning and night", + "Amoxicillin, morning and night", + "Paracetamol, after meals, 7 days", + "Diclofenac, morning and night, 7 days", + "Pantoprazole, before breakfast, 7 days", + "PAN 40, after meals, 6 days", + "Allegra(120), after meals, 6 days", + "Chilkul DSR, after meals, 6 days", + "INFLUVAC TETRA 0.5 ml, once, single dose", + "SYP. CALDOL (250mg/5ml), twice, 3 days", + "THYROX 112.5 mcg, empty stomach, ongoing", + "SUPRACAL PRO-ICAL, after dinner, 3 months", + "D-SOL 60 K, once monthly, ongoing", + "TELSAR BETA 25, after breakfast, 3 months", + "CILNIKEM 10 mg, after breakfast, ongoing", + "VILDAPHAGE-M, morning and night, 120", + "GRYCIPAAGE, before breakfast, lunch, and dinner, 5", + "AMLOSAFE, morning, 60", + "Cosader, morning, 120", + "PAN 40, morning and night, 5 days", + "Tab Zahoren - 03 Br, morning and night, 5 days", + "Tab Rabilir, night, 5 days", + "Pan 40, daily, 10 days", + "Clavum 625, daily, 5 days", + "Nebistar 2.5, daily, 14 days", + "Nasta DATISVIK 2%, 3 times a day, 8 days", + "SYP. AUGPEN DS, 3 times a day, 8 days", + "SYP. MAXTRA, 1 time a day, 8 days", + "Tab. GLYCOMET -TRIO 2mg, daily", + "Tab. DAPANORM TRIO (10/100/500), daily", + "Tab. THYRONORM 75 mcg, daily", + "PAN 40, morning and night, 10 days", + "Azithromycin, once a day, 5 days", + "Silofast, 1-0-1, 10 days", + "ORAHELP GEL, 5 days", + "Sy. FEVFAST DS, every 6 hours", + "Sy. DELCON, 3 times a day", + "Sy. VENTISOL JR., 3 times a day", + "Sy. AZIFINE 200, once a day for 3 days", + "PAN 40, morning and night, 3 days", + "Ach", + "U.S.9", + "Augustin ADS, before breakfast and dinner, 10 days", + "Sap. I bri plu y me thy, bedtime, 10 days", + "ciplor idp B/Eg 64, after lunch, 4 days", + "EuMD, 2 sce-1", + "T. DUTACOSIN", + "T. Amitriplilie", + "Avil 25, bd, 5 days", + "PanDro 100, bd, 7 days", + "Pan 40, bd, 7 days", + "ASHoke, 1-0-1, bizlerer", + "SomMA, 1-0-1, rendat", + "MAPSORD, 1-0-1, tut's", + "Damprosen 101, 0-0-1, tubs", + "IAB. UYIUK SMG, after food - sos", + "TAB. URIMAX D, after dinner - daily", + "TAB. NEXPRO 20MG, before breakfast - daily", + "TAB. VYMADA 100 MG, after food - daily", + "CAP. ROZAVEL A 75MG, after lunch - daily", + "TAB. EZEDOC 10MC, after dinner - daily", + "TAB. TIDE 10MG, after breakfast - daily", + "TAB. SARAPID 1.0MG TAB, before food - daily", + "PAN 40, morning and night, 7 days", + "Recuvac, night, 10 days", + "Br2 6 Th, morning and night, 7 days", + "PAN 40, morning and night, 10 days", + "Asthalin, 3 times a day, 5 days", + "Levolin, morning and night, 10 days", + "Astroscemi Aec Renan, morning and night", + "PAN 40, morning and night, 15 days", + "NEMOCID, 2 times a day, 2 days", + "PAN 40, morning and night, 30 days", + "LMD, , ", + "EPD, , ", + "Tablet Amlokind (5 mg), after breakfast, to continue", + "Tablet Vertizac, after breakfast, after dinner, 5 days", + "Tablet Diligan (25 mg), after breakfast, after dinner, 5 days", + "Tablet Full B12 SR Tablet, after breakfast, after dinner, 10 days", + "Tablet Olmetime (20 mg), after dinner, 10 days", + "Tablet Detrab dsr, 15 days", + "Tablet Folvite (5 mg), 15 days", + "Injection Vitcofol, 4 weeks, alt. day 2cc, 5 doses over 10 days", + "Tablet Enuff, 3 days", + "Capsule Calfix k2, 5 days", + "Injection Carmijen 6 lac iu, 4 weeks, intramuscular", + "Tablet Vibpreg mnt, 15 days", + "TAB TY PRO T4, morning and night, ongoing", + "TAB EGNERVE NP, night, 10 days", + "TAB CALCESAVE PLUS, morning, 2 months", + "VED GOK NANO SHOPS, once a week, 12 weeks", + "NOTTID Allegy, bd, 7 days", + "Co pain, bd, 7 days", + "T. Infinito vietato, bd, 7 days", + "Budesal (0.5mg), 1 � 5 days", + "TAB. OLYMPR IX M 500MG", + "TAB. TELMIKIND 40 MG, 3 months", + "CAP. EC OSPRIN AV 75MG", + "PAN 40mg, morning, night, before 18-may-23", + "Vallday, morning, night, before 18-may-23", + "Gratisin, morning, night, before 18-may-23", + "HIGH FIBER DIET", + "SITZ BATH THRICE A DAY WITH BETADINE LOTION", + "TAB ZOCEF CV 500, 7 days, 7 days", + "CAP PENRAB DSR, 7 days, 7 days", + "TAB DOLO 650, 5 days, then as needed, 5 days, then as needed", + "LACTIFIBER POWDER, 7 days, 7 days", + "TAB VITACOVER 5 G, 7 days, 7 days", + "BETADINE DRESSING AND PACKING LOCALLY AFTER EACH SITZ BATH", + "BRILINTA 90 mg, morning and night", + "ECOSPRIN 75 mg, before breakfast", + "STORVAS 80 mg, morning", + "CONcoe 2.5 mg, morning", + "CARDACE 1.25 mg, night", + "URIMAX D, after lunch", + "Phenoxymethylpenicillin, after breakfast and dinner, 7 days", + "Diclofenac, after breakfast, 7 days", + "PAN 40, before breakfast, 5 days", + "Aurin 625, before breakfast, 5 days", + "Da 108, before breakfast, 5 days", + "Tas-oflex-02, bd", + "Ths. Rebelt DSn, bbf", + "Spor 9rx, bd", + "Crocin DROPS, maximum 4 times/24 hours", + "Masoclearsal drop", + "Dolgan P 1x9, 10", + "( Zydees, 100, 3", + "Allegre 120g, 3", + "Nemaber fat", + "Levocetgine, 1 175, 3", + "Den., anatax o, 200g 4", + "(Zyders), 3", + "Tab metro chon, p - 60/m, 2-3 days", + "Mykool Cream, pr, 15-20 years", + "Tal. zerofor-P, before 1 week, colonoscopy sos", + "Taxing, 1-0-1 (3 times a day), 3 days", + "Stuurst-AF, 1-0-1 (3 times a day), 3 days", + "Ventryl, 1-0-1 (3 times a day), 3 days", + "Mental-BS, sos", + "Misogesic SR, once daily, 10 days", + "T. Oatzy, once daily, 300 days", + "Tab Torfix 400, morning and night, 7 days", + "Danga Lite, morning and night, 17 days", + "Car sizumas Rich (ML), morning and night, 17 days", + "Tablet Partivit 300mg BD, 25+2", + "Tab (alview facts", + "Tab worviday op", + "Cepodem 200 Tablet, daily, 3 days", + "Lezyncet-D Tablet (Levocetirizine 2.5 mg + Phenylephrine 10 mg), daily, 3 days", + "Immu C-Plus Chewable Tablets (Ascorbic Acid 500 mg + Vitamin D2 400 IU + Zinc Sulphate Monohydrate 5 mg), daily, 3 days", + "Reswas Syrup (Chlorpheniramine 2 mg + Levodropropizine 30 mg), daily, 3 days", + "Pacimol 650 Tablet (PARACETAMOL 650 mg), daily, 2 days", + "PAN 40, morning and night, 3 months", + "Caps D-RISE (Gok), once a month, 1 year", + "T. cifan- CT, 1-0-1, 5 days", + "T. Nimica plus, 1-1-1", + "Esogren 40, 1-0-0, 5 days", + "c. EnnGt 100, 0-1-0, 2 days", + "c. Imudium, 1-0-0", + "Pan 40, 18 days, 10 days", + "T. Acticont, 18 days, 10 days", + "T. Cyaa, 18 days, 10 days", + "Nosekund nose drops, 18 days, 10 days", + "Monucet 200, bd, 10 days", + "Hiponal MR-10, bd, 12 days", + "Calpal 500, bbf, 27 days", + "SANOGESIC P, 1-0-1, for 5 days", + "CEPODEM 200 MG, 1-0-1, for 5 days", + "MONDESLOR, 0-0-1, for 5 days", + "SISBONE K2, 0-0-1, for 15 days", + "Flucold AF oral deap, 1-0-1, 1 day", + "Calpol deogs, 1-0-1, 1 day", + "Extend Total Tablet, 1-0-1, 10 days", + "Sertafic 2% Ointment, 1-0-1, 5 days", + "Zentel Chewable Tablet, 0-0-1, 3 days", + "Espainin Hiber, morning and night, 7 days", + "Puissangue, morning and night, 7 days", + "Dolimon, morning and night, 7 days", + "PAN 40, after meals, 7 days", + "Pan 40, after meals, 7 days", + "Paracetamol, after meals, 5 days", + "Azithral, before meals, 3 days", + "Pansec 40 mg, before food, 5 days", + "Etoshine 60 mg, after food, 5 days", + "Nicoprotine Drop 15ML, 2 times a day, 7 days", + "Depura Kids Drop 15ML, once a day (morning)", + "Calpol (Pedia) Drops - 15ML, sos", + "OR-76 Som, once a day, 13 days", + "Lycored, twice a day, 12 capsules", + "Rentre D 160, once a day, 21 tablets", + "Vomitrat-op, once a day, 31 tablets", + "Di Aregest 3R, twice a day, 100 tablets", + "Vistogreat, unspecified, 5 sachets", + "Supraper-0-gel, 3 times a day, ongoing", + "Cilacar tc 12.5 tablet, once daily, 1 month(s)", + "Nicip tablet, twice daily, 1 month(s)", + "Gubi NT 100, before meals, 10 days", + "Nurich M., after meals, 14 days", + "Micel 14/12, after meals, 10 days", + "Panosuc-Don, before breakfast, 14 days", + "Fer 6, once daily, 4 days", + "Asthatin, every 3 hours, 2 days", + "AZITHRAL-XL 200, once daily, 4 days", + "DOM-DT 10mg, every 12 hours, 2 days", + "Cap Famocid (40), before breakfast, 14 days", + "Coop Innorfol Ds, daily, 14 days", + "Zincosit, after lunch, 14 days", + "Lupiheme, after dinner, 14 days", + "Zecal Gold, morning and night, 8 days", + "Gestoff SR (300), after breakfast, 14 days", + "Vivamon, after lunch, 14 days", + "Allegra 180, after dinner, 10 days", + "sol, 1-0-1", + "T Alet 6m, 1-0-1", + "Tpolude, 1-0-1", + "T Mont ce", + "-R. Ascout", + "MONTAIR 5mg tablet, morning and night, 5 days", + "Hy Machery Junior, morning and night, 5 days", + "Budecort New 1mg Respule, morning and night, 5 days", + "LIFE'S ON D, before breakfast, before dinner, to continue", + "TAB GABAPENTIN NT 400/10, 10 am, 3 pm, bedtime (10 pm), to continue", + "CAP QUENTIA BD [MULTIVITAMIN], to continuePregabid-ME 75mg, morning and night, 14 days", + "Aceclofenac 100mg, morning and night, 14 days", + "Ban 40, morning and night, 7 days", + "Tab. Criminale plus", + "Tab Olyvit m, morning 10 a.m. to 2 p.m., evening 5 p.m. to 10 p.m.", + "SYF Kapry Exp., tds", + "SYR Cakine, hls", + "Syp Angmolistet", + "Amoxicillin, 1-0-1, 5 days", + "Rantac, 1-0-1, 5 days", + "Paracetamol, 1-1-1, 2 weeks", + "REPAN D, morning and night, 4 days", + "ZERODOL P, morning and night, 20 days", + "DMR 30, morning and night, 94 days", + "L DIO 1 M, morning and night, 4 days", + "CMAXY GOLD, morning and night, 90 days", + "Tab Razo (20 mg), 1 time daily, 30 days", + "Tab Menoctyl, 1 time daily, 30 days", + "Tab Cizaspa, 2 times daily, 200 days", + "Tab TENOFOVIR ALEFANAMIDE 25mg, 0-0, to continue", + "PAN 40, before breakfast and dinner, 15 days", + "REGGIE NEX PRO-L, before breakfast and dinner, 15 days", + "MET NATURALARE, before breakfast and dinner, 15 days", + "PAN 40, morning and night, 10 days", + "Cepodem 200, morning and night, 10 days", + "Kaya Top-Nerm� x6, morning and night, 10 days", + "Ban 40, morning and night", + "NA", + "NA", + "PAN 40, morning, 7 days", + "Paracetamol, morning and night, 3 days", + "Amoxicillin, morning and night, 5 days", + "Tab Esocafe, before breakfast", + "Tab somfy 1, morning and night", + "Cap Evion 400, before breakfast", + "Tab. Carfine 1, morning and night", + "Tab. Etobrush T 1, morning and night", + "Drop Nasoclear Nasal", + "Drop Crocin", + "Drop Maxtra", + "Respule Levolin (0.31 mg)", + "Augmentin 625mg, twice a day, 5 days", + "Pan 40mg, once a day before breakfast, 5 days", + "Zerodal-P, thrice a day, 3 days", + "Rebert DSR, 2-2-2", + "Paracetamol 650mg, 1-", + "Metroxy1 400mg, bbf", + "Aceite, 0-0-1, 5 days", + "REST", + "Pantosec, 1-0-1, 5 days", + "corexx MIR, 5 days", + "Colafix", + "Olocal, 0-1-0, 5 days", + "Pan 40, bbf (before breakfast), 6 days", + "Zipedroo, bd (before dinner), 6 days", + "Macbok protein powder, 3 times a day", + "T. Protonese, 1 tablet, 6 weeks", + "Eunorm sachet, 3 times a day", + "T. Dolow, morning and night, 10 days", + "T. RA 20, morning, 10 days", + "Cap. Pegaba-m 75, morning, 10 days", + "SULTA, morning and night", + "INT PAI, morning and night", + "PAN 40, morning and night", + "Tab. Evion-LC, 5 days", + "Tab. Venvit, 10 days", + "Tab. Calpol 650mg, 3 days", + "BEE 25", + "FUL", + "T. TOFACINITA TOFADEZ, 1-1-1, 7 days", + "MESACOL-OD 1.2, (continu)", + "Domstal, 3 times a day, 3 days", + "Ecogram GG, 1 time a day, 5 days", + "Carmicide Ped, 3 times a day, 5 days", + "Inj Bett, stat", + "Cap Finacid Dsr, once a day, 30 days", + "Cap Pink Xt, once a day, 30 days", + "SYP SOVENTUS JR(0.5 MG / 5 ML), 5 days", + "SYP LEVOCET M KID(2.5 MG / 5 ML), 5 days", + "Nebulization, bd, 10 days", + "vomik, al", + "Sur. Adward (220), 1-0-1", + "S.P.200-1, occasional, rx", + "SBR-200, dr. brijesh gupta (b.h.m.s.)", + "NA", + "NOMIN CARE", + "T. Letrohope 5mg, � 5 days", + "PAN 40, morning and night, 6 days", + "Cheston Cold, morning and night, 6 days", + "Azithromycin, morning, 3 days", + "TIX", + "Impression den", + "Cementation don", + "Veloz (20), 1-1, 15 days", + "Ecospres-Av(75/10), -1", + "cosabrad/ Loslas (5), -1", + "Azmaids", + "Vyamada (200), 1-1, 12 days", + "Oxhamel 10/1gm", + "E Corbus (2.5)", + "zylori (100)", + "Dutan T(X-1)", + "Thystime(375)", + "Dytor 20", + "Dytor 10", + "Ca+ CD3, continue", + "Lanoxin (.25)", + "Reftar Wetrosave", + "PAN 40, daily", + "Visnet 6128 pp, daily", + "Lipids, daily", + "Nirmadhu Tablet, after food, for diabetes", + "Prashaantha Tablet, after food, for bp", + "Coldol, 4-64", + "Montair / Telekast / Romilast, i tab, once only - evening, 30 day(s)", + "Budecort / Budate inhaler 100, 12 hrly (fix dose), 5-15 day(s)", + "Cetzine Syp, night, 5-15 day(s)", + "Budenase Nasal Sprey, 12 hourly, 5-15 day(s)", + "Protinex Drigmal powder, 12 hrly, 45 day(s)", + "E Levolin / Salbair inhaler, 3 doses stat, 1 dayisa", + "Levolin / Saltiair inhaler (severity of cough), 2-4-6-8 hrly, 5 dayis", + "Toba[Sun] 0.3 %W/V, 3 times per day, for 4 day/s", + "Syrp Relent Plus[Dr], 3 times per day, for 3 day/s", + "T. PAH 20mg, 3 months, 3 months", + "Calpol 120, 3 times a day, until symptoms subside", + "Levolin, twice a day, 3 days", + "ECOSPRIN 75 mg, morning and night", + "ATORVA 20 mg, morning", + "PAN 40 mg, morning", + "Diofos, morning and night, 5 days", + "Entro Hora, morning, 5 days", + "S Weltam, morning, 5 days", + "Colinexmo, evening, 5 days", + "Ensure Rice, evening, 5 days", + "Pan 40, after meal, 5 days", + "Paracetamol 500mg, after meal, 5 days", + "Tossex SF, after meal, 5 days", + "ELIQUIS 5mg, morning and night, 1 month", + "ROSEDAY A 75/20mg, night", + "CONCOR COR 2.5mg, night", + "SACURISE 50mg, morning and night", + "DAPAHENZ 10mg, morning", + "ALDACTONE 50mg, morning", + "BRIVASURE 50mg, morning and night", + "NA", + "NA", + "NA", + "Tab upra 400, bd, 20", + "3 cap Cerealx, bd, 20", + "4 Tab Livoganz, bd, 20", + "Anyarpan 18 (12.5), morning and night, 31/03/2013 - 24/03/2023", + "Thyromarm (62.5), morning, as per requirement", + "Contorcer (1.20), morning, as per requirement", + "Dimed, automatic, as per requirement", + "Vego (0.3), morning, as per requirement", + "Forglip-1 (20/150), morning, as per requirement", + "Zaphrt 18, morning, as per requirement", + "Neptun (50), morning, as per requirement", + "Astor, morning, as per requirement", + "Capiret & (35), morning, as per requirement", + "Eucalmine, morning, as per requirement", + "Incomplete, 3 times a day, undefined", + "Levipiv 750, 10-1-0, 7 days", + "Oxeral 300, 10-0-0, 15 days", + "PAN 40, morning and night, 10 days", + "Paracetamol, morning and night, 7 days", + "Antitussive syrup, morning and night, 7 days", + "Syrup Ibugesic Plus 45, 3 times a day, 7 days", + "Youmo-Paracetamol 125mg, 3 times a day, 7 days", + "Syrup Meftal P, 3 times a day, 7 days", + "Syrup Maxtra, 3 times a day, 7 days", + "Syrup Kufril LS, 3 times a day, 7 days", + "Syrup Ondem, 3 times a day, 7 days", + "Vial EnteroGermina, once a day, 7 days", + "T. Linapride M (2.5/500), morning, 30 days", + "T. Panon-D, night, 30 days", + "T. Ke todan, morning, 15 days", + "T. Nacsare, morning, 31 days", + "T. Midotab 2.50g, not mentioned, 15+15 days", + "T. Pruloo 3 2mg, morning, 15 days", + "Ark get (LA), not mentioned, -", + "T. Benitab 4mg, morning, 15 days", + "Syp. DuphalacC, -, 10ml", + "T. Acicurb 19m, not mentioned, 15+15 days", + "& Panton", + "8. 97554-4", + "PAN 40, morning", + "SLRT", + "Delinical Abd Pelosis", + "Cap Synkron 513, 15 days, 10", + "Tab Celdet 6 mg, 10", + "Tab Ricert 20 mg, 10", + "Ssp Welleine -P (250), every 5 days", + "Mekut DS, only, 5 days", + "Top Duit DDS (200)", + "Unknown", + "Tafi, morning, 1 day", + "Be Well, morning, afternoon, night, 1 day", + "Foly's Catluter, morning and night, 1 day", + "Suspension Ibukind Plus (IBUPROFEN(100 MG) + PARACETAMOL(162.5 MG)), after food, till required", + "Inj: Imax 5", + "Normal Saline/ Tv set/ Blue Conula/ Disposable byring & Decale -0", + "Syp. Ux Joy-1", + "Syp. Descorange-1, a", + "Syp. Ibugesic PLUS, 1-0-1, 5 days", + "DEPURA, 1-0-1, 4 weeks", + "ULCIWELL DSR, before dinner, 10 days", + "THEOLIFE, after breakfast, after dinner, 1 week", + "LC BIT MK KID, after dinner, 1 week", + "zental 400, after dinner, stat", + "TUNEVIT, after lunch, 10 days", + "Mebit plus, stat, intramuscular", + "Compas", + "OT GEMER K VL", + "DJ Eloum or", + "Pan 40, once daily, 10 days", + "Augmentin 625, twice daily, 10 days", + "Montek LC, once daily, 10 days", + "TAB PAN 40, once daily, 5 days", + "Tab . Signoflom, once daily, 5 days", + "Shelcal, once daily, 5 days", + "Dicorate-ER 500mg, 1 - 0 - 0, 1 month", + "Mebodep-CD3, 1 - 0 - 0, 1 month", + "Tryptomer-25mg, 0 - 0 - 1, 1 month", + "Writex 7G, 0 - 0 - 1, 1 month", + "Napra D 500mg, as required", + "Syrup Zyrcold, twice a day, 5 days", + "Syrup Montek LC Kid, once a day, 2 weeks", + "Alocepodem 100-DT, twice a day, 10 days", + "Sp Asthalin, as needed", + "Ciplamid - 3, at bedtime, 5 days", + "Dato, 1-", + "Frend, 1-", + "E Flo Eyedrop Eye Drops, till 02 oct 2023, both eyes", + "E Flo Eyedrop Eye Drops, till 04 oct 2023, both eyes", + "E Flo Eyedrop Eye Drops, till 06 oct 2023, both eyes", + "E Flo Eyedrop Eye Drops, till 08 oct 2023, both eyes", + "HYLA OINTMENT Eye Ointment, both eyes", + "Capsule ITASPOR-SB, 1-0-1, 7 days", + "NAILROX CREAM, 1-0-0, 21 days", + "Tablet JUPISHINE, 1-0-1, 1 month", + "Azithromycin 500mg, after meals, 3 days", + "Allegra 120mg, before meals, 7 days", + "Paracetamol 500mg, after meals, 3 days", + "CAP CEFTURSN, 0-0-1, 5 days", + "CAP ECOFLORA, 1-0-0, 15 days", + "CAP RyTH MIXA, 1-0-1, 13 days", + "TAB EGOSPO75, 1-0-0, 7 days", + "Cap Somtraz 1, 10 days, 10 days", + "Tab Sompraz 40, 15 days, 15 days", + "Val Leemide 25, 7 days, 7 days", + "Gaviscon, after meal", + "Tas Metosantan 40/25, 1 month, 1 month", + "Tab Sompraz 20, 20 days, 20 days", + "Dolo, as needed", + "PAN 40mg, after meal, 2 days", + "JUMP, before meal, 2 days", + "Putop-2, after meal, 3 days", + "Gerne MPS, before meal, 3 days", + "Somprag 40mg, before meal", + "TI CINTARRO, after meal", + "Sur Sumafilo, after meal", + "NORMAXIN, after meal, 3 days", + "PAN 40, before breakfast and dinner", + "NAD, as needed", + "TIBROZEN, once daily", + "Pan 40, bd, 2 months", + "Cap Doxy 100 mg, bd, 2 months", + "Cap Vizylar, bd, 2 months", + "Jab Mekogyl 400mg, bd, 2 months", + "Tablet Minti AZ, morning and night", + "APPSMarcas 250 MG, morning and night", + "Tablet Diominie Dea, morning, afternoon, and night", + "Tablet Akilos p, before breakfast, after lunch, and evening", + "Syrup Coscopin Plus, before breakfast, lunch, and dinner", + "Capsule Happi D (20 & 30), morning", + "T Gluconet Mini, morning and night", + "Alco-pm, before bedtime", + "HTN + RX", + "Telmiland, after lunch", + "CT", + "Telmisartan 40/6.25, after dinner, 10 days", + "Carbiphage XR 500, after breakfast and dinner, 10 days", + "Bio-D3 Max, after lunch, 7 days", + "Rosuvas Ford 10, after dinner, 10 days", + "Febrap 40, after breakfast", + "Flokind 0.4, before bedtime, 5 days", + "Merogal GR 600, after dinner, 10 days", + "POLYCLAVE 625 MG, twice daily", + "BENZ, once daily", + "TUSQ LOZENGES, four times daily", + "Paracetamol, 3 times a day, 5 days", + "Montair-By, 1 time a day, 5 days", + "Microcap 200, 1 time a day, 5 days", + "Dolo 650mg, 5 times a day, full course", + "Enterogermina Suspension, 3 days, full course", + "Dynapar Injection, 5 days", + "Tab. Veronal SP, 5 days", + "Tab. R-Drive 20, 5 days", + "PAN 40, morning and night, 5 days", + "Paracetamol, morning and night, 5 days", + "Comger Talegront Duur, twice a day, 5 days", + "Pastadosr, twice a day, 5 days", + "Tra de Cetinsta CV, twice a day, 5 days", + "Antle binder -CD, twice a day, 5 days", + "AMARYL 2 mg, morning and night, till review", + "VILDAPRIDE 50 mg, morning and night, till review", + "ROCROS 10, afternoon, till review", + "NEXITO FORTE, afternoon, till review", + "PEVESCA PLUS, afternoon, till review", + "TH Eltroxin, 1-0-1, 30", + "[missing medicineme], 1-1-1, 30", + "[missing medicineme], 1-1-1, [missing course duration]", + "Paracetamol, morning and night, 4 days", + "Amoxicillin, morning and night, 4 days", + "Omeprazole, morning and night, 4 days", + "Novamente, morning and night, 10 days", + "PAN 40, morning and night, 10 days", + "Calcium, morning and night, 10 days", + "Tas-Mittel Spas, after breakfast, 5 days", + "Tax-Domperidone, after breakfast, 5 days", + "Tus-Partie 500mg, after lunch, 5 days", + "Tab Mahacef 200, after meals, 5 days", + "Tab Liv 52 DS, after meals, 5 days", + "Tab Becosules Z, after meals, 5 days", + "Sur. Oxipad 1001", + "Folsafe-L, once daily, 30 days", + "Feri B. wash, once daily, 21 days", + "F. Alde, once daily, 14 days", + "Terbinafine 500, once daily, 20 days", + "Inj Imax 5, as prescribed", + "Normal Saline, as prescribed", + "Syp Ux Joy, as prescribed", + "Syp Descorange, as prescribed, a", + "Syp. Duphalac, 0-0, 7 days", + "Lignocaine jelly 2% (xylocaine), tds", + "J- Sitcom (forte), 0, 1", + "clo BIL Pedal odema", + "CALLESR, 2-2- 2, 1 month", + "Unimot, 1 month", + "D-fix, 1 month", + "Ultracet, 20 day", + "Lipiland - F, 1 month", + "PROLOMET XL 50MG TABLET, before breakfast - daily, 6 weeks", + "SZETALO PLUS TABLET, bed time - daily, 6 weeks", + "LONAZEP MD 0.25MG TABLET, after dinner - daily, 6 weeks", + "ROSUVAS F 10MG TABLET, after dinner - daily, 6 weeks", + "HEPADO, after food - daily, 6 weeks", + "CYRA D CAPSULE, before breakfast - daily, 15 days", + "SURBEX XT TABLET, after food - daily, 15 days", + "CIPCAL 500 mg TABLET, alternate day, 3 weeks", + "Cipcal D3 Granules, once a week, 5 weeks", + "Neurobion Forte Tablet 30's, alternate day, 3 weeks", + "TAB SANOGESIC P, 1-0-1, 5 day", + "TAB CEPODEM 200 MG, 1-0-1, 5 day", + "TAB MONDESLOR, 0-0-1, 5 day", + "CAP SISBONE K2, 1-0-1, 15 day", + "PAN 40, thrice daily, 30 days", + "Ferradol, once daily, 60 days", + "Progesterone, thrice daily, 90 days", + "PAN 40, morning and night, 59 days", + "T Norminol, morning and night, 59 days", + "DL ID, you, morning and night, 59 days", + "T. Reclid� MR 30, bd (before breakfast and dinner), 2 months", + "T. Vildaparido M 50/150, bd (before breakfast and dinner), 2 months", + "Med T Glycomit SR 500, od (once a day), 2 months", + "Peuvent (Rf) Gibi Oleraa bunt, 1-0-1, x 5 days", + "Tab CHYMORAL FORTE, 1-0-1, x 5 days", + "Tab HIFENAC MR, 1-0-1, x 5 days", + "Tab. Altroday, after breakfast, 14 days", + "Tab. Ultramed D, after lunch, 21 days", + "Joe Collamer Plus Idaes, after dinner, 3 months", + "Kup T98M, 1-0-1", + "Backoder M, 1-0-1", + "Pintor Cs, 1-0-1", + "Inj PCM 1, stat, one dose", + "Jaraben, 1-0-1, 6 days", + "Bayar 10%, 1-0-1, 6 days", + "Kehm Pero 1mg, 1-0-1, 6 days", + "Toonceyzong Ty de marketing, 1-0-1, 6 days", + "T. Etibenny Plus, 1-0-1, 6 days", + "T. Metaguy 500k, 1-0-1, 6 days", + "Gymerbal 4151, 1-0-1, 6 days", + "Elucilla DS 6.5, 10-1.10, 14", + "Azgoodx(oc), -, 6", + "Arm.", + "Sumal Plus, twice a day, 3 days", + "Velof-D, three times a day, 3 days", + "Nasoact Nib, once, 1 day", + "Chemamila Pills, once, 1 day", + "Colimex, once, 1 day", + "Betnesol, 2 doses, 24 hours apart", + "Cephalexin, bd", + "Adw", + "pees", + "Vinpocetine", + "PAN 40, morning", + "BP 114168, morning and night", + "Motival, morning and night", + "Taf, before breakfast and dinner, 1 month", + "MONTAIR- fx, 5 days, 5 days", + "Ciplox 500mg, 5 days, 5 days", + "T. Dolo 650 mg, 3 days, 3 days", + "Syp Ascol- 1, 1-0-0, 5 days", + "A well ag, 12 weeks, 12 weeks", + "BP Medication, once daily", + "Vitamin D3, once daily", + "Taxim-O 200, twice daily, 7 days", + "AS PLV, once daily, 10 days", + "TAB RAZO 20mg, 1-0-1, 30 days", + "TAB MENOCTYL, 1-0-1, 30 days", + "TAB CIZASPA, 1-0-1, 200 days", + "Tanto 75M, 1-0-0", + "DIT Enlargafly, 1-1-1", + "Lase & Gutem, 1-0-0", + "Labelsc, 1-0-1", + "Istrot, 1-0-0", + "Tal Eldict, 1-1-1", + "Plain xy all, 1-0-1", + "Chut oha", + "Stomatab.c, before breakfast", + "Dksix.c, before food", + "Elathy.c, before breakfast", + "Kasa.c, before food", + "Swasawa.c, before food", + "Ceralgin, after food", + "Kasa kasayam, before food", + "Vasilha, before food", + "BarronilG, before food", + "Dermittal, before food", + "Livoral, before food", + "PP.C, before food", + "Decofycin, before food", + "vasa Leg, after food", + "vizyme.c", + "MANYATA, morning and night", + "Tab. WAXDOM 500 mg, morning and night", + "Tab. TRYPTOMER, morning", + "No-Mark ointment", + "Alloederim", + "Pan 40, morning and night, 5 days", + "Dolo 650, morning and night, 5 days", + "T. Janumet 50/500, daily", + "T. Remyelin D, weekly, 8 weeks", + "T. D sise 60k, weekly, 8 weeks", + "Meftal P, as needed for fever, repeat after 6 hours if required", + "Ascoril LS, thrice a day, 1 week", + "Ambroxol + Levosalbutamol, thrice a day, 1 week", + "Telekast L Kid, once a day at night, 15 days", + "Azee XL 200, once a day, 5 days", + "Polyethylene Glycol + Sodium Bicarbonate, once a day, 1 month", + "Bandy, once a day tonight and repeat after two weeks", + "Albendazole", + "Cyclopam, as needed for pain abdomen", + "Simethicone + Dicyclomine, as needed for pain abdomen", + "D3 must Nano Shots 60 K, once a week, 10 weeks", + "Cholecalciferol 60,000 IU, once a week", + "Aptimust, once a day before food, in the evening at 7 pm, 3 months", + "Cyproheptadine", + "TRYptoMER lome, 3 times a day, 3 months", + "MSTRONG, 1 month", + "THMANO 3, 1 month", + "PAN 40, bd, 5 days", + "CALCIUM SUPPLEMENT (66k), bd, 400k", + "JUNCAL-P SUSPENSION, bd, 2 days", + "SANOGESIC P, 1-0-1, 5 days", + "CEPODEM, 1-0-1, 5 days", + "MONDESLOR, 0-0-1, 5 days", + "SISBONE K2, 0-0-1, 15 days", + "RSS Syp. oruzyme, bd", + "Syp offarmarim, bd, 12 days", + "Esp. Mexico-forte, bd, 5 days", + "Paracetamol, tds, 5 days", + "Budesonide, bd, 5 days", + "Im Mikastar 500, morning and night, 10 days", + "Zanocin-02, morning and night, 6 days", + "Sponolac-AS, morning and night, 6 days", + "Ban 40, before breakfast and before dinner, 1 month", + "Cap Protete 15, before breakfast, 1 month", + "Tab. comoufer vor �s, before dinner, 1 month", + "Typhilet gm, after dinner, 1 month", + "Tab. Unizyme, before breakfast and before dinner, 1 month", + "Tab. welcheline 2CB, before breakfast and before dinner, 1 month", + "Cap uprise 13 60, before breakfast, 1 month", + "PAN 40, 1, 14", + "Esocet-D", + "Clingen Forte, 0-1", + "PAN 40, bd, 7 days", + "TAB MEFTAL SPAS, as needed", + "VERTIN, once daily", + "Tab. Vetory, x 5 days", + "Tab. Acilac 150mg, x 5 days", + "Depo medrol, bd � 5 days", + "Tab. Auloc 15, br 3d � 5 days", + "PAN 20, after food, 10 days", + "Nauser, as needed", + "A Deslusion, as needed", + "Mildergashi, as needed", + "PR-112, before food, 10 days", + "liderin, as needed", + "TAB DILNIP 5 MG, after meals", + "TAB PRIMEZOLE 40 MG, before meals", + "TAB GLARISURE, before meals", + "Cap Sampras, morning and night, 2 weeks", + "Tab Sampras, morning and night, 2 weeks", + "Syp Fueral, before dinner, 2 weeks", + "Pan 40mg, before breakfast", + "Tramadol, as needed", + "Ilike", + "T. Livogen, morning and night, 5 days", + "T. cltrocaples, morning and night, 5 days", + "Macprotein powder, morning, 5 days", + "Zifi (200), morning, 5 days", + "Proroms9 (300), morning and night, 5 days", + "vizylac, morning and night, 5 days", + "Econoroom, morning and night, 5 days", + "Proff vachet, morning and night, 5 days", + "Flagyl 400, morning, 5 days", + "Flagyl 1000, morning, 5 days", + "Livogen, morning, 5 days", + "Fornig, morning, 5 days", + "Tustacoplar, morning, 5 days", + "jarix, morning, 5 days", + "Proffane 19 (300), morning and night, 5 days", + "Incomplete", + "Incomplete", + "Incomplete", + "NA", + "NA", + "NA", + "Anal Blog Ass, 1-0-0", + "Crabapun 100, 1-0-0", + "Zenopa 00, 1-0-0", + "Dolopar Creel", + "PAN 40", + "Triple H", + "Cap Ontoxid HC, once daily, 30 days", + "Desowen Cream, as directed, as directed", + "Jakta forte ointment, as directed, as directed", + "Job Allegra 180, as directed, as directed", + "Tab. DIZIBEAT, 5 days", + "Lesum, after food (9pm)", + "ECG, 5 days", + "Tab. Typerom, after meals, 5-7 days", + "Tab. Pop 120, before breakfast", + "Tab. APF 159, before breakfast", + "Inj. Insulien 50/50, after meals", + "Inj. En Sugen 30/70, after meals", + "Tab. Guywase 5, after meals, 30 days", + "Tab. Torhup 50, after meals", + "Tab. CruxIT 10, after meals", + "Tab. J. C. HOPACE 25, after meals", + "Tab. T. NUROKIND -LE, after lunch, 10 ml x 2mts", + "Syp. Cremaptin Bins, after meals", + "PAN 40, bd, 30", + "Renova GM, bd, 60", + "VoliDer, -", + "Amoxylaw, after breakfast", + "LK-D, before dinner", + "Mountain, before dinner, 6 days", + "Parto P-D, before breakfast, 6 days", + "Syr. on & on. Cough, after meals", + "Toul, before bedtime", + "Galvus.Met, bd, 1 week", + "Telnyk CH (40/12.5), od, 1 week", + "starpress XL 100, od, 1 week", + "Rosuvas, od, 1 week", + "Febustat, od, 1 week", + "Thyronorm, od, 1 week", + "Nasal Spray, bd, 15 days", + "T. Lenono, bd, 15 days", + "Anniy Plus 200, od, 5 days", + "1. Etaliler", + "-1. cease-sp", + "mysi cialement faut", + "Laformin GV, 0-0-1, 3 months", + "Sitahenz D 5/50, 1-0-0, 3 months", + "Enzoflam, 1-0-1, 5 days", + "Mandyic party", + "L.S.Shaw.", + "out Dokln-1", + "HIFENAC MAX TABLETS 10'S, 1-0-0, 1 month(s)", + "IT MAC 100MG STRIP OF 10 CAPSULES, 1-0-0, 1 month(s)", + "TRIBEN PLUS CREAM, once daily, 3 days", + "NEMOCID, twice daily, 2 days", + "NST (Non-Stress Test), weekly, ongoing", + "Depamethasone, 24 hours, ongoing", + "ALDACTONE 100 MG, after food - daily, ongoing", + "DIANE 35, after food - daily, 21 days", + "Tab FRANCAC CZ, 60, 81", + "Tab PEPFERRIN, 60", + "3 Tab SQADD1 2L", + "(a) Tab SUGAVILDASO, 120", + "Tab OD2 sunday,, 15 days 1", + "Tab CABERDOPA 0,5", + "2) COTIDOL Soap, te 0 0%", + "8 ZOBIDOLE Lotcon", + "OMNACORTIL 20MG TABLET, after breakfast - daily, 5 days", + "PANTIN 40MG TABLET, before breakfast - daily, 10 days", + "GLYMED 100ML LOTION, after bath - daily, 1 month", + "CLONATE 20GM OINTMENT, after bath - daily, 10 days", + "ALENIX 5 MG TABLET, after food - daily, 10 days", + "PAN 40, before breakfast and before dinner, 4 cycles", + "PAN 40, bd, 10 days", + "T. Chycometsk, al", + "Pigelmin, bbf", + "T. SitzkemxR, bd", + "Clan, bd", + "T. XEPAMH 100, bd", + "T. Zyfoly (3) 010, bd, 30 days", + "T. Ampnoch, bd", + "Cap. odhinab 100, bd", + "Sprig81af, al, 2 months", + "Ilal in Illera 10mg i.v. jeg0 ml, iv", + "Iv. auch 2 horas, iv", + "Avil, H. coltiv Het", + "Guzee", + "Poz Vala vuol 212", + "Apelo poglute -ai, tt, 28", + "ARV, 0, 3 7 , 14 28", + "lesibil 250 151, 300", + "opetal 300 100, (150)", + "NA, 50", + "TAB. OLYNZ M 500MG, 3 months", + "TAB. TELMIKIND 40 MG, 3 months", + "CAP. ECOSPRIN AV 75MG", + "Pan 40, morning and night, 14 days", + "Paracetamol, morning and night, 5 days", + "COAX, morning, 1 day", + "VILDAPHAGE-M, 01-0-0, 1-5/1120", + "AMLOSAFE, 01-0-0, 120", + "Cosader, 01-0-0, 16", + "Syp. Phenycip, 3 times a day, 7 days", + "Breathawaysal drop, 3 times a day, 7 days", + "Salvin Cola E, bd, 12 weeks", + "swiss OK, 001", + "PAN 40, morning and night, 7 days", + "Ecoprin 75mg, morning and night, 7 days", + "Ton Super 300mg, morning and night, 7 days", + "PAN 40, morning and night, 7 days", + "Orden MD, after lunch", + "Ban 40, 3 weeks, 3 weeks", + "Rabzia D, 10 days, 10 days", + "Entrygen-DS, 10 days, 10 days", + "Ru, sup Rinifol, > todail, 10w as adesed", + "di TreLlOR Lanol (30)", + "Gup Cyclopen, 7.5 w/ 60r, 7days", + "Guy Mofasi, 6", + "Waysone E a drop", + "Rx, T.prosyn 250, -", + "T. JelloR lanzo1 (15), quiral myositis", + "voreRantaes/ vovi Ran, 7", + "Doxt-SL Capsule, daily, 2 weeks", + "Glocin Gel CLINDAMYCIN (1/4 %) Gel, 1 time, 4 weeks", + "Minoz-BPO Gel Adapalene (0.1 %) + Benzoyl Peroxide (2.5 %) Gel, 1 time, 4 weeks", + "Ahaglow S Foaming Face Wash 100 ml, 2 times, 4 weeks", + "Acnemoist Cream 30 g, 2 times, 4 weeks", + "Hospipow zesto cold 11, bbf, 10 days", + "Otalet, bbf, 10 days", + "Desolid 10, al, 10 days", + "Pan 40, morning, 10 days", + "Gemino-300, morning and night, 100 days", + "Syp Augmentin DDS, 2-3 times daily, 1 day", + "Syp Thinic, once at bedtime", + "Crocint - 6 ml, morning and night", + "Angiglam", + "PAN 40, morning and night, 3 days", + "MEFTAL SPAS, morning and night, 5 days", + "ENTEGERMINA, morning and night, 7 days", + "Pan 40, morning and night, 10 days", + "Zimoces, morning and night, 10 days", + "Unobiotics, morning and night, 3 days", + "Pan 40, morning and night, 14 days", + "Paracetamol, morning and night, 10 days", + "Cough Syrup, morning, afternoon, and night, 7 days", + "Bluiban, bbf", + "chent hans, 001", + "Rib Belt, bd", + "T. Rifagut 550, morning and night", + "Bifitar ho, morning, afternoon, and night", + "Syp cinovic sul, morning, afternoon, and night", + "Cap Beusule 2, morning, afternoon, and night", + "syp sucrafil 0, morning, afternoon, and night", + "Soje, before breakfast, before dinner, 10 days", + "Vibalt DS, before breakfast, before dinner, 5 days", + "Raciper D, before breakfast, before dinner, 10 days", + "Tar lepodem, 1-0-1, 7 days", + "TAB. COLLAFLEZ PRO PLUS CAP., after food - daily - 5 days", + "TAB. AFLAROSE PLUS TAB, after food - daily - 5 days", + "TAB. ROCKBON, after food - daily - 5 days", + "TAB. ULTRAKING, after food - daily - 5 days", + "GUFIBIS OIL 20ML, daily - 5 days", + "TAB. HQTOR *, after food - daily - 5 days", + "CAP. PRECOOL *, after food - daily - 5 days", + "Optojest, 30 days", + "Preynocure, 30 days", + "Galan ma De, 30 days", + "Calcimax Forte, jul", + "Bandy plus, 70ml", + "Meganeuron-MF, after breakfast", + "Telista 20, after dinner, 20 days", + "Pantospe D&R, after meals", + "Zeprest plus, after meals", + "Besohow 2.5 Los, before bedtime", + "Tab. Nejor, bd, 1 month", + "Tab. Defical, bd, 1 month", + "Ij Osteo D3, stat, 1 month", + "Tab Cefoxim 500, twice daily, 5 days", + "Tab Pan 40, once daily, 10 days", + "Tab Zerodol SP, twice daily, 5 days", + "Sepia Im (4)", + "All. capa 200", + "Petro 200", + "April 200", + "SGH 2020, once only - evening, 30 day(s)", + "Coldol Cream 250 ml, night, 0", + "Montair / Telekast / Romilast - 5 mg, 0, 5-15 day(s)", + "Budecort / Budate inhaler 100, 12 hrly (fix dose), 5-15 day(s)", + "Cetzine Syp, night, 5-15 day(s)", + "Budenase Nasal Spray, 12 hourly, 5-15 day(s)", + "Protinex Dry Mix, 12 hourly, 45 day(s)", + "Levolin / Salbair inhaler, 3 doses stat, 1 day(s)", + "Levolin / Salbair inhaler (severity of cough), 2-4-6-8 hrly, 5 day(s)", + "TAB OMNACORTIL 2.5 mg 100, before breakfast and before dinner", + "TAB CARDIVAS 6.125, before breakfast and before dinner", + "TAB FOLVITE 5mg 100, before breakfast and before dinner", + "TAB RABLET 20, before breakfast, 15 days", + "CAP. CYCLOSPORIN 25, after lunch", + "CAP. DANAZOL 50, after dinner", + "TAB Du tor 5, after dinner", + "TAB Amlodip. 5, before breakfast", + "IM AUGPLAT 500 once weekly (Tuesday), before breakfast", + "TAB Zincovit, before breakfast and before dinner, -", + "ACID NIT, bd, start now", + "SCROPH NODOSA, bbf, morning", + "HAMAMELIS, al, afternoon", + "MYRISTICA, 001, night", + "NA", + "NA", + "NA", + "Montek Lc, morning, night, 6 days", + "Azithromycin, morning, 3 days", + "Paracetamol, morning, afternoon, night, 5 days", + "Syp Ibugeste, 3 times a day, 10 days", + "A02, 1-0-1, 5 days", + "Tab Foldege D, bd, 5 days", + "Pu Sampai 7mg, od, 25.26 days", + "PAN 40, morning and night", + "Forvite, night", + "Drogayde, morning and night", + "PAN 40, bbf", + "Paracetamol, bd", + "Dufladac 800, 3 times a day", + "Dechilea Jaim", + "Ro", + "x-1 -x 5deup", + "Rifagut 400 Tablet, after food, 10 days", + "Rifaximin 400mg, after food, 10 days", + "Dobesil 500mg Capsule, after food, 3 months", + "Calcium Dobesilate 500mg, after food, 3 months", + "Stone 1 B6 syrup, after food, 2 month", + "Enterogermina, after food, 5 days", + "pine, morning and night, 1-1.5 month", + "Mourefloor", + "Paulus, bd", + "Polybir, bd", + "C", + "Paracetamol, daily, 5 days", + "Cetirizine, daily, 5 days", + "Antacid, daily, 4 days", + "AVIRIT, 5 5, 5 days", + "Paracetamol, morning and night, 5 days", + "Montair-By, morning and night, 5 days", + "Microcap 200, morning and night, 5 days", + "CALPOL/CROCIN/PARACETAMOL DROPS, 6 am, 12 pm, 2 days", + "Hiper MK", + "Tipau 40mg, 2 days", + "Dicloget 401", + "Jij Methy Cobal 1 amp alt day, alternate days", + "Das Deterrol Oncealle x 100ks, daily", + "Vice-M (500)", + "Das Stalit-D, daily", + "Dos Tri olmasa@ 41", + "Das Enite (40), daily", + "Pan 40, morning and night, 7 days", + "Sinarest, morning and night, 7 days", + "Entero-Germ, morning and night, 7 days", + "PAN 40, bd, 21 days", + "DESO ALLERGY, bd, 21 days", + "CALCIUM, bbf, 21 days", + "Tenovate cream, 3 weeks", + "HCQS (400g), 3 weeks", + "Tab oxyflam MR, before breakfast and before dinner, 20 days", + "Cap Oxyorb LS, before breakfast and before dinner", + "Cap OXY-Q- 300, before breakfast and before dinner", + "Rb INOTOR 5, before breakfast and before dinner, 20 days", + "Tab Acoxia MR, before breakfast and before dinner", + "Cap Hbneuron PLUS, before breakfast and before dinner", + "Tap Alrical Gold, before breakfast and before dinner", + "Tab Mega carnit-, before breakfast and before dinner, 20 days", + "Gesic Liniment for massage", + "Zerfan MPS 10ml 50%", + "Syrup Zyrcold, twice a day, 5 days", + "Syrup Montek LC Kid, once a day, 2 weeks", + "NA", + "NA", + "NA", + "QIng. Didlo-1, tid, 3 days", + "zibi 200, bo, 3 days", + "TUS Q Dx-1", + "Pan-40, bd, 3 days", + "A+o2, 5 days", + "Tab. Moxyrin cu. 625mg, before food, 5 days", + "Tab. Raberin.D, before food, 5 days", + "Tab. Montriy-le, after food, 5 days", + "Tab. 0010-650-> 505", + "TAB Darf 4mg, after breakfast, 3 days", + "TAB Rosucoup 10 mg, before dinner, 1 year", + "VENUSIA MAX 300ML LOTION, daily, 30 days", + "ATODERM MOUSSANT, daily, 30 days", + "MOMATE 15GM CREAM, daily, 30 days", + "XYZAL 60ML SYRUP, daily, 30 days", + "TACROZ FORTE 10GM OINTMENT, daily, 30 days", + "TAB PAN 40, 1 daily, 5 days", + "Tab Signoflom 1, 1 daily, x5", + "Shelcal, xxo", + "Tab Cepodem, bd, 5 days", + "Tab ASTM, od, 5 days", + "Tab Panum D, ac, 10 days", + "Syp Ato 2, bd, 7 days", + "Syp BAPC, bd, 7 days", + "Tab SA 100, ac, 5 days", + "Tab Etowin 60, after breakfast and after dinner, 10 days", + "Tab Ban 40, before dinner, 10 days", + "Calpol drops, every 6 hrs", + "PFS TRESIVAC[SERUM], once", + "INJ MENACTRA[SANOFI], once", + "HbA1c", + "High Sensitivity C -Reactive Protein(HsCRP)", + "Iron Study", + "Tab. AlfsNC, 1-0-1", + "Tab Octobix, 1-0-1", + "Tab Ler15, 1-0-1", + "Tab Pzae, 1-0-1", + "Tab Euognix, 1-0-1", + "Mamadialitu con1, 1-0-1", + "Ag preg, 1-0-1", + "- FRS/2hPP BS, 1-0-1", + "PAN 40, morning and night, 4 days", + "Paracetamol, morning and night, 4 days", + "PAN 40, after meals", + "Erace, before meals", + "CRP AV 50, after meals", + "O.T. Dolo 650, bd", + "Lorsaid of, bbf", + "Devocetim/ my/cast, bbf", + "T. DISPERZYME, bbf, 5 days", + "Calpol 250mg Tablet, as needed, 5 days", + "Syp Alex Junior 5mg/5ml, 3 times a day, 5 days", + "Predmet, morning and night, 10 days", + "PAN 40, morning, 5 days", + "Paracetamol, morning and night, 24/2/23", + "Pan 40, morning and night, 3-4 days", + "Paracetamol, morning and night, 3-4 days", + "PAN 40, before breakfast and before dinner", + "Morten le, night", + "Defakind, after meals", + "Delafloxacin, morning and night, 5 days", + "EL Cumple MR, before dinner, 10 days", + "Co Codamol, 5 days", + "Co Adtol, 20 days", + "TRIVOLIB FORTE-1 TABLET, 60, 120 bf", + "THYRONORM 150MCG TABLET, 60, 60 bf", + "NA", + "Rabeprazole, 1-0-1", + "Atorvastatin, 1-0-0", + "Ivabradine, 1-0-1", + "Apixaban, 1-0-0", + "Azmoth, 1-0-0", + "Arnoza, 1-0-0", + "Bisoprolol, 1-0-1", + "Zyloric, 1-0-0", + "Ductus T, 1-0-0", + "Dytor 20 em, 1-0-0", + "Dytor 10, 1-0-1", + "Lanoxin, 1-0-3", + "Tonact Plus, 1-0-1", + "Dolonex Dt Tablet 20mg, bbf, 5 days", + "Augmentin 625mg Tablet, bd, 3 days", + "Dompan Tablet, al, 5 days", + "Dolo 650mg Tablet, al, 5 days", + "T. Dzotute 10mg, once a day", + "Rest atall", + "LUKOTAS HD TABLET, after dinner - daily, 20 days", + "MONTEMAC AL, after dinner - daily, 20 days", + "AMBROXOL 75 MG, daily, 20 days", + "LEVOCETIRIZINE 5 MG, daily, 20 days", + "MONTELUKAST 10 MG, daily, 20 days", + "PAN 40", + "Smart pain plan- tropper fs", + "NA", + "Pan 40, every morning, 7 days", + "Dapsone, every morning, 7 days", + "Carcit, morning and night, 7 days", + "Reglas Labor Com, morning and night, 7 days", + "140, every morning, 7 days", + "TAB - SHELCAL CT, 1, after breakfast and dinner", + "TAB - THYRONORM 125 MCG, 1, on empty stomach (1 tab mon to sat , 2 tab on sunday)", + "CAP - D RISE, 1, once a month", + "TAB AMLONG 5 MG, morning", + "Rantac, after meals, 5 days", + "Nasivion, bd (before bed), 5 days", + "Ibugesic Plus, after meals, 5 days", + "Capotril, after meals, 7 days", + "Syp. Rady, after meals, 5 days", + "Calvin-D3 Drops, once a day, 1 month", + "Nasivion S (Nasal) Drops, 5 times a day, 4 days", + "Sodium Chloride (0.65% w/v) + Benzalkonium Chloride (0.03% w/v) Nostrils Drops, 5 times a day, 4 days", + "PAN 40, before breakfast, 2-3 weeks", + "Tab B-29 1m, before dinner", + "Tab Dolo 600, after lunch", + "TABLET DELTONE (60 mg), 30 min before food, 30 day(s)", + "TABLET LESURIDE 25MG (25MG ), 30 min before food, 30 day(s)", + "TABLET NEXITO 5MG (5 mg), after food, 30 day(s)", + "LIQUID Aristozyme (10/50mg), 15 ml three times daily after food, 30 day(s)", + "Suspension DIGERAFT MINT FLAVOUR (10 ML), after food, 10 day(s)", + "ORS, tbd, tbd", + "Sp. Oftrivselog, tbd, tbd", + "Soc. DiFisac 1-14, tbd, tbd", + "Syl. cyccorona(5mm) +84, tbd, tbd", + "Sys. Partene Su 1 mg, tbd, tbd", + "MEN-39, tbd, tbd", + "Azithromycin, 1-0-1, 5 days", + "Paracetamol, 1-1-1, 5 days", + "Rib Belt, n/a, as per requirement", + "PAN 40, morning and night, 10 days", + "Paracetamol, morning and night, 10 days", + "Diclofenac, morning and night, 10 days", + "PAN 40, twice daily, not mentioned", + "Nimulid, only at night, not mentioned", + "Entimy Plus, once daily, not mentioned", + "81T60060", + "LDIT60060", + "Dr.T.F@br &p&Turfun M.B.B.S ., M.S.(OBG),D.G.O ., F.A.G.E .,", + "T. Somprag 40mg, before breakfast and before dinner, 3 days", + "CINTARRO, before breakfast, lunch, and dinner", + "Sucrafel-o, after lunch, 14 days", + "NORMEXIN, 3 times a day, 3 days", + "Misogesic SR, 3 times a day, 10 days", + "T. Oatzy, before breakfast, 30 days", + "T. Cobafisch, before dinner", + "Tab. DIZIBEAT, 5 days", + "Seruna Vit. D, 5 days", + "Lesum Vit. B12", + "Tub FlozerAA -15, 01�5 day", + "Tab Mahaaf x 20", + "Tab Dolokind MR-, 1/2h", + "Tab Parkand", + "PAN 40, thrice a day", + "Florent- Fourth", + "Te CALS", + "Gel - Ora Help, 5 days, 5 days", + "Syp. Bandy Plus, hs, 15 days", + "Pan 40, 3 days, 7 days", + "Ecosprin AV (15/10) OD, morning, 30 days", + "Night spadives, morning and night, 30 days", + "NA", + "NA", + "NA", + "Tab. Augmentin Duo 625mg, 10 days, 10 days", + "Tab. Colpa-D, 5 days, 5 days", + "Syp Ascoril-LS, 1 week, 1 week", + "Yab. Relent, 1 week, 1 week", + "Cap. Somplas-0, 1 week, 1 week", + "Nirmadhu Tab, after food, 45 days", + "DIME E Wock, after food, 45 days", + "Prashaantha Tab, after food, 45 days", + "TAB.RYBELSUS 14 MG, 1 tablet before breakfast, 30 days", + "TAB.GLEDEPA 10 MG/OXRA 10 MG, 1 tablet before breakfast, 30 days", + "TAB.GLYCOMET SR 500 MG, 1 tablet before dinner, 30 days", + "TAB.SYMBAL (30MG), 1 tablet bedtime, 30 days", + "CAP.RABONIK DSR, 1 capsule before dinner, as needed", + "TAB.RAZEL F 5 MG, 1 tablet bedtime, 30 days", + "Jy. Sheprix, bd", + "Sp Dolo 250, 001", + "Sp. Bifolate, 001, 1 month", + "PAN 40, once daily, after 7 days", + "LORAZEPAM, as needed", + "VOLTAREN, as needed", + "Amoxicillin-Clavulanate 625mg, 5, 5 days", + "Dexamethasone 4mg, 5, 5 days", + "Paracetamol 500mg, 5, 5 days", + "Zinc Sulphate, 5, 5 days", + "Probiotics, 5, 5 days", + "PAN 40, before breakfast and before dinner, 1 month", + "Atarax, before bedtime, 5 days", + "Goun DS/Campol -250, if needed", + "Tab Tricum Max, 1 month", + "Tab Tendoshot, 1 month", + "Cap Indo Cap SR, 10 days", + "Tab Tryptomer, 10 days", + "Tab Rablet, 10 days", + "Tab ultrave", + "Ronder, bd, 4 days", + "Coldphan plus, bd, 4 days", + "Nazoplus marclep, tds, 4 days", + "Paracetamol-Mas, bd, 4 days", + "Livogen 1, 3 times a day, 15 days", + "Sheled xi, 6 times a day", + "Syrup Aptimust (CYPROHEPTADINE(2 MG)), before food, 2 weeks", + "Syrup Moktel (MULTI VITAMIN), 3 months", + "Medicine 3", + "Tab. Teczine 10 mg, morning, night, 100 days", + "Venusia Max Lotion", + "Diperbate Plus Lotion, morning, 100 days", + "PAN 40, morning and night, 10 days", + "Paracetamol, morning, afternoon and night, 5 days", + "Cough Syrup, morning and night, 7 days", + "Vonciel, morning and night", + "Pantop 2, morning and night", + "Ancin 625, night", + "Tls 0.1", + "PAN 40, once a day, 7 days", + "Paracetamol, 3 times a day, 5 days", + "Syp Dufladac, twice a day, 10 days", + "Mesar Plus, bbf, 35 days", + "Rowlip F 10, al, continue", + "Job Nuhenz D, bd, 1 month", + "Bio-Deplus, bd, 1 month", + "Gen Dono 60, 18 weeks", + "Pan 40, before breakfast and dinner, 5 days", + "Alvily, bd, 10 days", + "Taz Lansoprazol, bd, 15 days", + "Tab Switch 200, al, 10", + "Tab Calpol - 2, 001, 10", + "Sinarest, bd, 10", + "Tab par D, bd, 10", + "ACICURB OS, 5 ml after meals, 5 days", + "ZINCOVIT, 0 after meals, 15 days", + "ATARAX (10), 1 if itching/if required, 10 days", + "PAN 40, morning and night", + "Tramadol, morning, afternoon, and night", + "Hing, morning", + "TAB. COLLAFLEZ PRO PLUS CAP., after food - daily - 5 days", + "TAB. AFLAROSE PLUS TAB, after food - daily - 5 days", + "TAB. ROCKBON, after food - daily - 5 days", + "TAB. ULTRAKING, after food - daily - 5 days", + "GUFIBIS OIL 20ML, daily - 5 days", + "TAB. HQTOR *, after food - daily - 5 days", + "CAP. PRECOOL, after food - daily - 5 days", + "Ban 40", + "Oftax-OR 1", + "Pantop-40mg", + "Nuovi 2TSFX", + "TAB AMLONG 5 MG", + "Sgp. RidAts, follow up date:", + "PTU 50 MG, morning, afternoon, and night, review in 1-2 days", + "Ban 40, morning and night", + "Pantop D, morning and night", + "Ancin 625, morning and night", + "Syp. Duphalac, 0-0, 7 days", + "Lignocaine jelly 2% LA (xylocaine), before meals, 7 days", + "J- Sitcom forte, after meals, 7 days", + "Rozecor ASP, bd (before dinner), 30 days", + "T AXCER 90mg, al (after lunch), bd (before dinner), 30 days", + "Carloc 3.125, bbf (before breakfast), al (after lunch), bd (before dinner), 30 days", + "Syrup Briolite, once a day, 10 days bedtime", + "LEVOCETIRIZINE (2.5MG/5ML), once a day, 10 days bedtime", + "MONTELUKAST (4MG/5ML) Oral Suspension", + "AMOXYCILLIN (400MG/5ML) + CLAVULANIC ACID (57MG/5ML) Oral Suspension, 12 hourly, 5 days", + "Ointment Mupin (Skin) (5 gm) MUPIROCIN(2%), 8 hourly, 5 days affected area", + "Crocin DS Suspension, 6 hourly, sos", + "Nasoclear Nasal Drop, 6 hourly, 5 days", + "Toned milk", + "OLESOFT MAX CREAM", + "Alaspan Tablet LORATADINE (10 mg)", + "Protar-K Solution Ketoconazole (2 %) + Coal Tar (4 %)", + "Momate Cream Mometasone (0.1 %)", + "SHIKO 21077572", + "VELTEN 04MG 3MSK", + "Pan 40, after meals, 7 days", + "Ibrida Junior, after meals, 7 days", + "Polycion L - 50, after meals, 7 days", + "Syp. oflox 100mg, 3 times a day, 5 days", + "Syp. Netafox, 2 times a day, 3 days", + "Sport 4410, 2 times a day, 3 days", + "H-GUT, 2 times a day, 3 days", + "Dallan ORL, 1 time a day, 3 days", + "Syp. crpar 250, 6 times a day, 5 days", + "Sp. Me/torp, 1 time a day, 3 days", + "PAN 40, morning and night, 15 days", + "NEMOCID, bd (before dinner), 2 days", + "Ciplar (10), bd", + "Peril MD (+25), bd", + "Syp Julire", + "Pan 40, morning and night", + "Liv 52, morning", + "Bestyme, morning", + "PAN 40, morning and night", + "Perzo-M, morning and night", + "Dermeden calse lotion, bd", + "Sompraz 40, morning and night", + "Levogastrof 25, morning and night", + "OFLOX 50 SUSPENSION, 0, 6ml", + "SPORCAC TABS, -, rantal synd", + "Emixt Sumup, 30-45 mins, byfor food", + "Tab. Mebtal-500, 20 day", + "Tab Absolut hold", + "Tab Pay kony, 23 day", + "Tab cetrizin cons", + "Tab Zontal 400mg, x every 3 days", + "Caman, before breakfast, 60 days", + "Toxin 650, before dinner, 3 days", + "Dyrapar, after lunch, 3 days", + "PAN 40, morning, afternoon, night", + "Cap Aclinch, morning, afternoon, night", + "Pan 40, morning and night, 6 days", + "Admad, 1-0-1, 5 days", + "GLIMEPIRIDE, 1-0-1, 3 months", + "METFORMIN, 1-0-1, 3 months", + "LOSARTAN, 1-0-0, 3 months", + "Neurovon fris", + "Pato de D50", + "Tas Acicion 400, 3 times a day, 8 days", + "Tabs Rabmor D, 1 time a day, 8 days", + "Calapure Lotion, as needed, 8 days", + "Tab Neurokind LC, 1 time a day, 8 days", + "Syr Polybion L, 1 time a day, 8 days", + "PAN 40, morning and night, 3 weeks", + "Anoxlief, morning and night, 3 weeks", + "PAN 40, after meals, 10 days", + "Bilinds sun screenDA, daily, until next visit", + "Cosmelite-next Cream2, daily, 1 month", + "Noclevon, 3 times a day", + "Frucola AF, once a day", + "Marberny PD, 3 times a day", + "Pan 40, before breakfast, 1 day", + "Dydrogesterone, before breakfast, 1 day", + "Ferrous Ascorbate, before breakfast, 1 day", + "Mega-CV Forte, 2 times a day, 3 days", + "Syp Flexon, 2 times a day, 5 days", + "Maxtra, 3 times a day, 7 days", + "Sinomet (P)sal drops, 3 times a day, 10 days", + "SAPDAvolac, 1 time a day, 11 days", + "L-MONTUS TAB, 0-0-1, 2 weeks", + "FURAMIST NASAL SPRAY 27.5mcg/1 puff, 1-0-1, 2 weeks", + "SINAREST TAB, as needed, 3 days", + "hydrochloride 10mg, chlorpheniramine maleate 2mg", + "Steam Inhalation", + "Tab Ultracet, ao y, 3 days", + "Ice Docks", + "Moxiforce CV 625mg Tab, after meal, 5 days", + "Wintop DSR, before meal, 5 days", +] +durations_list = [ + "1 week", + "1 month", + "3 days", + "5 days", + "2 weeks", + "10 days", + "3 weeks", + "2 months", + "once", +] +frequencies_list = [ + "before breakfast", + "before breakfast and dinner", + "after food", + "before food", +] + + +def get_full_med_list(): + return full_med_list + + +def process_docs(dataset: datasets.Dataset): + def _transform(doc): + diagnosis = get_diagnosis(doc) + medicines_list = get_medicines_list(doc) + + try: + if contains_indian_characters(diagnosis): + diagnosis = "NA" + except Exception: + pass + + try: + if len(check_list_for_indian_characters(medicines_list)) > 0: + medicines_list = [] + except Exception: + pass + + gold, gold_position = doc_to_target_obtain(doc) + + doc["keep"] = (diagnosis != "NA") and (len(medicines_list) != 0) + doc["gold"] = gold + doc["gold_position"] = gold_position + return doc + + transformed_dataset = dataset.map(_transform) + + # Now filter the dataset to keep only those where 'keep' is True + def _filter(doc): + return doc["keep"] + + filtered_dataset = transformed_dataset.filter(_filter) + print(f"Final len filtered dataset: {len(filtered_dataset)}") + + return filtered_dataset + + +def contains_indian_characters(text): + # Define Unicode ranges for Indian scripts + indian_script_ranges = [ + (0x0900, 0x097F), # Devanagari + (0x0980, 0x09FF), # Bengali + (0x0A80, 0x0AFF), # Gujarati + (0x0A00, 0x0A7F), # Gurmukhi + (0x0C80, 0x0CFF), # Kannada + (0x0D00, 0x0D7F), # Malayalam + (0x0B80, 0x0BFF), # Tamil + (0x0C00, 0x0C7F), # Telugu + ] + + # Create a regular expression pattern for Indian scripts + pattern = "|".join( + [f"[{chr(start)}-{chr(end)}]" for start, end in indian_script_ranges] + ) + + # Check if the text contains any Indian script characters + return bool(re.search(pattern, text)) + + +def check_list_for_indian_characters(string_list): + results = [] + for text in string_list: + if contains_indian_characters(text): + results.append(text) + return results + + +def doc_to_text_easy(doc) -> str: + diagnosis = get_diagnosis(doc) + choices = doc_to_choice_easy(doc) + prompt = ( + "You are a medical doctor. A patient presents the following diagnosis or complains: {}. What would you prescribe in this case? \nChoices: \n" + "A. {} \n" + "B. {} \n" + "C. {} \n" + "D. {} \nAnswer:".format( + diagnosis, choices[0], choices[1], choices[2], choices[3] + ) + ) + + return prompt + + +def doc_to_text_hard(doc) -> str: + diagnosis = get_diagnosis(doc) + choices = doc_to_choice_hard(doc) + prompt = ( + "You are a medical doctor. A patient presents the following diagnosis or complains: {}. What would you prescribe in this case? \nChoices: \n" + "A. {} \n" + "B. {} \n" + "C. {} \n" + "D. {} \nAnswer:".format( + diagnosis, choices[0], choices[1], choices[2], choices[3] + ) + ) + + print(prompt) + return prompt + + +def get_diagnosis(doc): + results_dict = ast.literal_eval(doc["results"]) + if "procedure" in results_dict: + if "chief_complaints_diagnosis" in results_dict["procedure"]: + diagnosis = results_dict["procedure"]["chief_complaints_diagnosis"] + elif "chief_complaints" and "diagnosis" in results_dict["procedure"]: + diagnosis = ( + "Complains: " + + results_dict["procedure"]["chief_complaints"] + + ". Diagnosis: " + + results_dict["procedure"]["diagnosis"] + ) + else: + diagnosis = "NA" + + elif "prescription_details" in results_dict: + try: + diagnosis = results_dict["prescription_details"]["disease_diagnosis"] + except Exception: + diagnosis = "NA" + elif "Symptoms/Complaints" in results_dict: + symptoms = results_dict["Symptoms/Complaints"] + diagnosis = ", ".join(symptoms) + else: + diagnosis = "NA" + return diagnosis + + +def get_medicines_list(doc): + results_dict = ast.literal_eval(doc["results"]) + if ( + "medicine_details" in results_dict + and len(results_dict["medicine_details"]) != 0 + ): + if "medicine_frequency" in results_dict["medicine_details"][0]: + try: + medicines_sample = [ + f"{item['medicine_name']}, {item['medicine_frequency'].lower()}, {item['course_duration'].lower()}" + for item in results_dict["medicine_details"] + if len(item.keys()) > 1 + ] + except Exception: + try: + if "course_duration" not in results_dict["medicine_details"][0]: + medicines_sample = [ + f"{item['medicine_name']}, {item['medicine_frequency'].lower()}" + for item in results_dict["medicine_details"] + if len(item.keys()) > 1 + ] + elif ( + "medicine_frequency" not in results_dict["medicine_details"][0] + and "course_duration" in results_dict["medicine_details"][0] + ): + medicines_sample = [ + f"{item['medicine_name']}, {item['course_duration'].lower()}" + for item in results_dict["medicine_details"] + if len(item.keys()) > 1 + ] + else: + medicines_sample = [] + except Exception: + medicines_sample = [] + + elif "medicine_dosage" in results_dict["medicine_details"][0]: + medicines_sample = [ + f"{item['medicine_name']}, {item['medicine_dosage'].lower()}" + for item in results_dict["medicine_details"] + ] + else: + medicines_sample = [] + elif "Medicines Prescribed" in results_dict: + if "Duration" in results_dict["Medicines Prescribed"][0]: + medicines_sample = [ + f"{item['Name']}, {item['Duration'].lower()}" + for item in results_dict["Medicines Prescribed"] + ] + elif "Dosage" in results_dict["Medicines Prescribed"][0]: + medicines_sample = [ + f"{item['Name']}, {item['Dosage'].lower()}" + for item in results_dict["Medicines Prescribed"] + ] + else: + medicines_sample = [] + + medicines_sample = [ + item.replace(", na", "") + .replace(" na", "") + .replace(" n/a", "") + .replace("NA", "") + .replace(" not specified", "") + .replace(" not provided", "") + for item in medicines_sample + ] + medicines_sample = [item for item in medicines_sample if len(item) > 1] + if "[Medicine Name], [medicine frequency], [course duration]" in medicines_sample: + medicines_sample = [] + + return medicines_sample + + +def doc_to_target(doc): + return doc["gold_position"] + + +def doc_to_target_obtain(doc): + medicines_sample = get_medicines_list(doc) + if len(medicines_sample) == 0: + return "NA", 0 + gold = random.choice(medicines_sample) + gold_position = random.randint(0, 3) + return gold, gold_position + + +def doc_to_choice_easy(doc): + gold, gold_position = doc["gold"], doc["gold_position"] + full_med_list = get_full_med_list() + random_items = random.sample(full_med_list, 3) + choices_easy = random_items[:gold_position] + [gold] + random_items[gold_position:] + return choices_easy + + +def doc_to_choice_hard(doc): + gold, gold_position = doc["gold"], doc["gold_position"] + medicines_sample = get_medicines_list(doc) + random_choices = [] + + new_list = [item for item in medicines_sample if item != gold] + + if random.randint(0, 1) == 0: + fake_duration = random.sample(durations_list, 1) + fake_frequency = random.sample(frequencies_list, 1) + random_choices.append( + f"{gold.split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + if len(new_list) >= 2: + fake_duration = random.choices(durations_list, k=2) + fake_frequency = random.choices(frequencies_list, k=2) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{new_list[1].split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + elif len(new_list) == 1: + fake_duration = random.choices(durations_list, k=2) + fake_frequency = random.choices(frequencies_list, k=2) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + else: + fake_duration = random.choices(durations_list, k=2) + fake_frequency = random.choices(frequencies_list, k=2) + random_choices.append( + f"{gold.split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{gold.split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + else: + if len(new_list) >= 3: + fake_duration = random.choices(durations_list, k=3) + fake_frequency = random.choices(frequencies_list, k=3) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{new_list[1].split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + random_choices.append( + f"{new_list[2].split(',')[0]}, {fake_frequency[2]}, {fake_duration[2]}" + ) + elif len(new_list) == 2: + fake_duration = random.choices(durations_list, k=3) + fake_frequency = random.choices(frequencies_list, k=3) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + random_choices.append( + f"{new_list[1].split(',')[0]}, {fake_frequency[2]}, {fake_duration[2]}" + ) + elif len(new_list) == 1: + fake_duration = random.choices(durations_list, k=3) + fake_frequency = random.choices(frequencies_list, k=3) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + random_choices.append( + f"{new_list[0].split(',')[0]}, {fake_frequency[2]}, {fake_duration[2]}" + ) + else: + fake_duration = random.choices(durations_list, k=3) + fake_frequency = random.choices(frequencies_list, k=3) + random_choices.append( + f"{gold.split(',')[0]}, {fake_frequency[0]}, {fake_duration[0]}" + ) + random_choices.append( + f"{gold.split(',')[0]}, {fake_frequency[1]}, {fake_duration[1]}" + ) + random_choices.append( + f"{gold.split(',')[0]}, {fake_frequency[2]}, {fake_duration[2]}" + ) + + choices_hard = ( + random_choices[:gold_position] + [gold] + random_choices[gold_position:] + ) + return choices_hard diff --git a/lm-evaluation-harness/lm_eval/tasks/med_text_classification/med_text_classification_easy.yaml b/lm-evaluation-harness/lm_eval/tasks/med_text_classification/med_text_classification_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7410f511f9b4cf73dc170bf6879c1e037e469204 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_text_classification/med_text_classification_easy.yaml @@ -0,0 +1,24 @@ +group: med_text_classification +task: med_text_classification_easy +dataset_path: csv +dataset_name: null +dataset_kwargs: + data_files: + train: /gpfs/projects/bsc70/heka/data/datasets/med_text_class_train.csv +output_type: multiple_choice +training_split: train +validation_split: train +test_split: train +process_docs: !function utils.process_docs +doc_to_text: !function utils.doc_to_text_easy +doc_to_choice: !function utils.doc_to_choice_easy +doc_to_target: !function utils.doc_to_target_easy +generation_kwargs: + until: + - "\n\n" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/med_text_classification/med_text_classification_hard.yaml b/lm-evaluation-harness/lm_eval/tasks/med_text_classification/med_text_classification_hard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fbcca0ae8abc95a59dc86601030e5c10a6fd980 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_text_classification/med_text_classification_hard.yaml @@ -0,0 +1,9 @@ +include: med_text_classification_easy.yaml +task: med_text_classification_hard +dataset_kwargs: + data_files: + train: /gpfs/projects/bsc70/heka/data/datasets/mtsamples.csv +process_docs: !function utils.process_docs_hard +doc_to_text: !function utils.doc_to_text_hard +doc_to_choice: !function utils.doc_to_choice_hard +doc_to_target: !function utils.doc_to_target_hard diff --git a/lm-evaluation-harness/lm_eval/tasks/med_text_classification/utils.py b/lm-evaluation-harness/lm_eval/tasks/med_text_classification/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b642e277e09415e7f05d17556f76bfb08ddf27de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/med_text_classification/utils.py @@ -0,0 +1,117 @@ +import random + +import datasets + + +def process_docs_hard(dataset: datasets.Dataset): + return dataset + + +def process_docs(dataset: datasets.Dataset): + def _helper(doc): + return doc + + num_entries = len(dataset) + ten_percent_index = int(0.1 * num_entries) + + # Select the first 10% of the dataset + filtered_dataset = dataset.select(range(ten_percent_index)) + + return filtered_dataset.map(_helper) + + +def doc_to_choice_easy(doc): + return [ + "neoplasms", + "digestive system diseases", + "nervous system diseases", + "cardiovascular diseases", + "general pathological conditions", + ] + + +def doc_to_text_easy(doc) -> str: + choices = doc_to_choice_easy(doc) + prompt = ( + "Classify the topic of the following medical text into one of the following choices. \n" + "Text: {} \n" + "Choices: \n" + "A. {} \n" + "B. {} \n" + "C. {} \n" + "D. {} \n" + "E. {} \n Answer:".format( + doc["text"], choices[0], choices[1], choices[2], choices[3], choices[4] + ) + ) + + return prompt + + +def doc_to_target_easy(doc): + return int(doc["class"]) - 1 + + +def doc_to_text_hard(doc) -> str: + choices = doc_to_choice_hard(doc) + prompt = ( + "Select the medical specialty the following text is talking about among the following choices. \n" + "Text: {} \n" + "Choices: {}\n" + " Answer:".format(doc["transcription"], choices) + ) + + return prompt + + +def doc_to_choice_hard(doc): + choices_list = [ + " Bariatrics", + " Allergy / Immunology", + " Dentistry", + " Cardiovascular / Pulmonary", + " Urology", + " Hospice - Palliative Care", + " Radiology", + " Pediatrics - Neonatal", + " Neurology", + " Neurosurgery", + " Emergency Room Reports", + " IME-QME-Work Comp etc.", + " Office Notes", + " Surgery", + " Letters", + " Ophthalmology", + " Hematology - Oncology", + " Endocrinology", + " Cosmetic / Plastic Surgery", + " Diets and Nutritions", + " Rheumatology", + " Nephrology", + " Physical Medicine - Rehab", + " Podiatry", + " Chiropractic", + " Lab Medicine - Pathology", + " Orthopedic", + " Autopsy", + " Psychiatry / Psychology", + " Speech - Language", + " ENT - Otolaryngology", + " Sleep Medicine", + " Dermatology", + " SOAP / Chart / Progress Notes", + " General Medicine", + " Consult - History and Phy.", + " Obstetrics / Gynecology", + " Gastroenterology", + " Pain Management", + " Discharge Summary", + ] + return choices_list + + +def doc_to_target_hard(doc): + choices = doc_to_choice_hard(doc) + gold = doc["medical_specialty"] + idx = choices.index(gold) + return idx diff --git a/lm-evaluation-harness/lm_eval/tasks/meddialog/README.md b/lm-evaluation-harness/lm_eval/tasks/meddialog/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b7ed4b3d2dc9ff2cf53029ae31745b713869083b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/meddialog/README.md @@ -0,0 +1,52 @@ +# Meddialog + +### Paper + +Title: `MedDialog: Large-scale Medical Dialogue Datasets` + +Abstract: [https://aclanthology.org/2020.emnlp-main.743/](https://aclanthology.org/2020.emnlp-main.743/) + +This task contains the english version of the MedDialog Medical Dialogue Dataset, divided in two tasks: +question entailment, and open-ended Question Answering (QA). + +#### Tasks + +* `meddialog_qsumm`: Question entailment in english. +* `meddialog_qsumm_perplexity`: Question entailment in english, evaluated with perplexity. +* `meddialog_raw_dialogues`: Open-Ended QA in english. +* `meddialog_raw_perplexity`: Open-Ended QA in english, evaluated with perplexity. + +### Citation + +```bibtex +@inproceedings{zeng-etal-2020-meddialog, + title = "{M}ed{D}ialog: Large-scale Medical Dialogue Datasets", + author = "Zeng, Guangtao and + Yang, Wenmian and + Ju, Zeqian and + Yang, Yue and + Wang, Sicheng and + Zhang, Ruisi and + Zhou, Meng and + Zeng, Jiaqi and + Dong, Xiangyu and + Zhang, Ruoyu and + Fang, Hongchao and + Zhu, Penghui and + Chen, Shu and + Xie, Pengtao", + editor = "Webber, Bonnie and + Cohn, Trevor and + He, Yulan and + Liu, Yang", + booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", + month = nov, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2020.emnlp-main.743/", + doi = "10.18653/v1/2020.emnlp-main.743", + pages = "9241--9250", + abstract = "Medical dialogue systems are promising in assisting in telemedicine to increase access to healthcare services, improve the quality of patient care, and reduce medical costs. To facilitate the research and development of medical dialogue systems, we build large-scale medical dialogue datasets {--} MedDialog, which contain 1) a Chinese dataset with 3.4 million conversations between patients and doctors, 11.3 million utterances, 660.2 million tokens, covering 172 specialties of diseases, and 2) an English dataset with 0.26 million conversations, 0.51 million utterances, 44.53 million tokens, covering 96 specialties of diseases. To our best knowledge, MedDialog is the largest medical dialogue dataset to date. We pretrain several dialogue generation models on the Chinese MedDialog dataset, including Transformer, GPT, BERT-GPT, and compare their performance. It is shown that models trained on MedDialog are able to generate clinically correct and doctor-like medical dialogues. We also study the transferability of models trained on MedDialog to low-resource medical dialogue generation tasks. It is shown that via transfer learning which finetunes the models pretrained on MedDialog, the performance on medical dialogue generation tasks with small datasets can be greatly improved, as shown in human evaluation and automatic evaluation. The datasets and code are available at \url{https://github.com/UCSD-AI4H/Medical-Dialogue-System}" +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/meddialog/utils.py b/lm-evaluation-harness/lm_eval/tasks/meddialog/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a81b5e814f07dcf32e0def81caa89064dfc3996c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/meddialog/utils.py @@ -0,0 +1,128 @@ +import numpy as np + + +try: + import evaluate + + bleu = evaluate.load("bleu") + rouge = evaluate.load("rouge") + bertscore = evaluate.load("bertscore") + bleurt = evaluate.load("bleurt", "bleurt-base-512", module_type="metric") + +except (ModuleNotFoundError, ImportError): + raise ModuleNotFoundError( + "Please install evaluation metrics via pip install evaluate and pip install bert-score", + ) +except Exception as e: + raise RuntimeError( + f"Error loading evaluation metrics: {str(e)}. Please check your installation." + ) + + +def doc_eval(pred, refs): + try: + bleu_results = bleu.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Bleu error: {e}") + bleu_results = {"bleu": np.nan} + + try: + rouge_results = rouge.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Rouge error: {e}") + rouge_results = {"rouge1": np.nan, "rouge2": np.nan, "rougeL": np.nan} + + try: + bleurt_scores = bleurt.compute(predictions=pred, references=refs)["scores"] + except Exception as e: + print(f"Bleurt error: {e}") + bleurt_scores = [np.nan] + + try: + bert_scores = bertscore.compute(predictions=pred, references=refs, lang="en")[ + "f1" + ] + except Exception as e: + print(f"Bert error: {e}") + bert_scores = [np.nan] + + if bleu_results["bleu"] == 0: + # Sometimes bleu is 0.0 and this breaks the stderr computation. + bleu_results["bleu"] += 1e-5 + + results = { + "bleu": bleu_results["bleu"], + "rouge1": rouge_results["rouge1"], + "rouge2": rouge_results["rouge2"], + "rougeL": rouge_results["rougeL"], + "bleurt": np.mean(bleurt_scores), + "bert_score": np.mean(bert_scores), + } + + return results + + +def doc_to_text_raw(doc) -> str: + return doc["description"] + + +def doc_to_target_raw(doc) -> str: + return doc["utterances"]["utterance"][1] + + +def process_results_gen_raw(doc, results): + pred, refs = [results[0]], [doc_to_target_raw(doc)] + + if len(refs[0]) < 1 or len(pred[0]) < 1: + return { + "bleu": np.nan, + "rouge1": np.nan, + "rouge2": np.nan, + "rougeL": np.nan, + "bleurt": np.nan, + "bert_score": np.nan, + } + + results = doc_eval(pred, refs) + + return { + "bleu": results["bleu"], + "rouge1": results["rouge1"], + "rouge2": results["rouge2"], + "rougeL": results["rougeL"], + "bleurt": results["bleurt"], + "bert_score": results["bert_score"], + } + + +def doc_to_text_qsumm(doc) -> str: + return doc["src"] + + +def doc_to_target_qsumm(doc) -> str: + return doc["tgt"] + + +def process_results_gen_qsumm(doc, results): + pred, refs = [results[0]], [doc_to_target_qsumm(doc)] + + if len(refs[0]) < 1 or len(pred[0]) < 1: + return { + "bleu": np.nan, + "rouge1": np.nan, + "rouge2": np.nan, + "rougeL": np.nan, + "bleurt": np.nan, + "bert_score": np.nan, + } + + results = doc_eval(pred, refs) + + return { + "bleu": results["bleu"], + "rouge1": results["rouge1"], + "rouge2": results["rouge2"], + "rougeL": results["rougeL"], + "bleurt": results["bleurt"], + "bert_score": results["bert_score"], + } diff --git a/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/README.md b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/README.md new file mode 100644 index 0000000000000000000000000000000000000000..489d7d08421c5b8f4790e6cbbb730aae703840d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/README.md @@ -0,0 +1,42 @@ +# MEDIQA_QA 2019 + +### Paper + +Title: `Overview of the MEDIQA 2019 Shared Task on Textual Inference, Question Entailment and Question Answering` + +Abstract: [https://aclanthology.org/W19-5039/](https://aclanthology.org/W19-5039/) + +Open-ended medical Question-Answering stemming from the MEDIQA 2019 open challenge. + +Homepage: \ +[https://sites.google.com/view/mediqa2019](https://sites.google.com/view/mediqa2019) + + +#### Tasks + +* `mediqa_qa2019`: Open-ended QA in english. +* `mediqa_qa2019_perplexity`: Open-Ended QA in english, evaluated with perplexity. + +### Citation + +```bibtex +@inproceedings{ben-abacha-etal-2019-overview, + title = "Overview of the {MEDIQA} 2019 Shared Task on Textual Inference, Question Entailment and Question Answering", + author = "Ben Abacha, Asma and + Shivade, Chaitanya and + Demner-Fushman, Dina", + editor = "Demner-Fushman, Dina and + Cohen, Kevin Bretonnel and + Ananiadou, Sophia and + Tsujii, Junichi", + booktitle = "Proceedings of the 18th BioNLP Workshop and Shared Task", + month = aug, + year = "2019", + address = "Florence, Italy", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W19-5039/", + doi = "10.18653/v1/W19-5039", + pages = "370--379", + abstract = "This paper presents the MEDIQA 2019 shared task organized at the ACL-BioNLP workshop. The shared task is motivated by a need to develop relevant methods, techniques and gold standards for inference and entailment in the medical domain, and their application to improve domain specific information retrieval and question answering systems. MEDIQA 2019 includes three tasks: Natural Language Inference (NLI), Recognizing Question Entailment (RQE), and Question Answering (QA) in the medical domain. 72 teams participated in the challenge, achieving an accuracy of 98{\%} in the NLI task, 74.9{\%} in the RQE task, and 78.3{\%} in the QA task. In this paper, we describe the tasks, the datasets, and the participants' approaches and results. We hope that this shared task will attract further research efforts in textual inference, question entailment, and question answering in the medical domain." +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/mediqa_qa2019.yaml b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/mediqa_qa2019.yaml new file mode 100644 index 0000000000000000000000000000000000000000..616f9d85a920f5a24da69712eea152ac61910085 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/mediqa_qa2019.yaml @@ -0,0 +1,36 @@ +task: mediqa_qa2019 +dataset_path: bigbio/mediqa_qa +description: > + Instructions: The following text is a question asked by a patient. Answer how a doctor would, while trying to be as informative and helpful as possible. + +output_type: generate_until +training_split: train_live_qa_med +validation_split: validation +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +process_results: !function utils.process_results_gen +generation_kwargs: + until: + - "\n\n" +metric_list: + - metric: bleu + aggregation: nanmean + higher_is_better: true + - metric: rouge1 + aggregation: nanmean + higher_is_better: true + - metric: rouge2 + aggregation: nanmean + higher_is_better: true + - metric: rougeL + aggregation: nanmean + higher_is_better: true + - metric: bleurt + aggregation: nanmean + higher_is_better: true + - metric: bert_score + aggregation: nanmean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/utils.py b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..112009608f47b71e50ed864c971738f0457ab3e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/utils.py @@ -0,0 +1,95 @@ +import numpy as np + + +try: + import evaluate + + bleu = evaluate.load("bleu") + rouge = evaluate.load("rouge") + bertscore = evaluate.load("bertscore") + bleurt = evaluate.load("bleurt", "bleurt-base-512", module_type="metric") + +except (ModuleNotFoundError, ImportError): + raise ModuleNotFoundError( + "Please install evaluation metrics via pip install evaluate and pip install bert-score", + ) +except Exception as e: + raise RuntimeError( + f"Error loading evaluation metrics: {str(e)}. Please check your installation." + ) + + +def doc_eval(pred, refs): + try: + bleu_results = bleu.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Bleu error: {e}") + bleu_results = {"bleu": np.NAN} + + try: + rouge_results = rouge.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Rouge error: {e}") + rouge_results = {"rouge1": np.NAN, "rouge2": np.NAN, "rougeL": np.NAN} + + try: + bleurt_scores = bleurt.compute(predictions=pred, references=refs)["scores"] + except Exception as e: + print(f"Bleurt error: {e}") + bleurt_scores = [np.NAN] + + try: + bert_scores = bertscore.compute(predictions=pred, references=refs, lang="en")[ + "f1" + ] + except Exception as e: + print(f"Bert error: {e}") + bert_scores = [np.NAN] + + if bleu_results["bleu"] == 0: + # Sometimes bleu is 0.0 and this breaks the stderr computation. + bleu_results["bleu"] += 1e-5 + + results = { + "bleu": bleu_results["bleu"], + "rouge1": rouge_results["rouge1"], + "rouge2": rouge_results["rouge2"], + "rougeL": rouge_results["rougeL"], + "bleurt": np.mean(bleurt_scores), + "bert_score": np.mean(bert_scores), + } + + return results + + +def doc_to_text(doc) -> str: + return doc["QUESTION"]["QuestionText"] + + +def doc_to_target(doc) -> str: + return doc["QUESTION"]["AnswerList"][0]["Answer"]["AnswerText"] + + +def process_results_gen(doc, results): + pred, refs = [results[0]], [doc_to_target(doc)] + + if len(refs[0]) < 1 or len(pred[0]) < 1: + return { + "bleu": np.NAN, + "rouge1": np.NAN, + "rouge2": np.NAN, + "rougeL": np.NAN, + "bleurt": np.NAN, + "bert_score": np.NAN, + } + + results = doc_eval(pred, refs) + + return { + "bleu": results["bleu"], + "rouge1": results["rouge1"], + "rouge2": results["rouge2"], + "rougeL": results["rougeL"], + "bleurt": results["bleurt"], + "bert_score": results["bert_score"], + } diff --git a/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/utils_perplexity.py b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/utils_perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..f60e30f04e20e9b12f6af08da46200a9f47ff237 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mediqa_qa2019/utils_perplexity.py @@ -0,0 +1,19 @@ +import math +import re + + +def doc_to_target(doc) -> str: + return doc["QUESTION"]["AnswerList"][0]["Answer"]["AnswerText"] + + +def process_results(doc, results): + (loglikelihood,) = results + _words = len(re.split(r"\s+", doc_to_target(doc))) + _bytes = len(doc_to_target(doc).encode("utf-8")) + print(f"perplexity: {math.exp(-loglikelihood / _words)}") + return { + "word_perplexity": (loglikelihood, _words), + "byte_perplexity": (loglikelihood, _bytes), + "bits_per_byte": (loglikelihood, _bytes), + "perplexity": (loglikelihood), + } diff --git a/lm-evaluation-harness/lm_eval/tasks/medmcqa/medmcqa.yaml b/lm-evaluation-harness/lm_eval/tasks/medmcqa/medmcqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8fb1c7cdba4983d8ccb509491f699d9be0afa17e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medmcqa/medmcqa.yaml @@ -0,0 +1,18 @@ +task: medmcqa +dataset_path: medmcqa +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: validation +doc_to_text: !function utils_medmcqa.doc_to_text +doc_to_target: cop +doc_to_choice: [ 'A','B','C','D' ] +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true diff --git a/lm-evaluation-harness/lm_eval/tasks/medmcqa/utils_medmcqa.py b/lm-evaluation-harness/lm_eval/tasks/medmcqa/utils_medmcqa.py new file mode 100644 index 0000000000000000000000000000000000000000..8ce7e6beece511dccdfe94790456711ee7e93eab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medmcqa/utils_medmcqa.py @@ -0,0 +1,24 @@ +# Copied from Master +def doc_to_text(doc) -> str: + """ + Question: + Choices: + A. + B. + C. + D. + Answer: + """ + choices = [doc["opa"], doc["opb"], doc["opc"], doc["opd"]] + option_choices = { + "A": choices[0], + "B": choices[1], + "C": choices[2], + "D": choices[3], + } + + prompt = "Question: " + doc["question"] + "\nChoices:\n" + for choice, option in option_choices.items(): + prompt += f"{choice.upper()}. {option}\n" + prompt += "Answer:" + return prompt diff --git a/lm-evaluation-harness/lm_eval/tasks/medqa/medqa.yaml b/lm-evaluation-harness/lm_eval/tasks/medqa/medqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d5555966fa0d4bcf2e8dc4a74eea7442ca433a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medqa/medqa.yaml @@ -0,0 +1,16 @@ +task: medqa_4options +dataset_path: GBaker/MedQA-USMLE-4-options-hf +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +doc_to_text: !function preprocess_medqa.doc_to_text +doc_to_target: !function preprocess_medqa.doc_to_target +doc_to_choice: [ 'A', 'B', 'C', 'D' ] +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true diff --git a/lm-evaluation-harness/lm_eval/tasks/medtext/README.md b/lm-evaluation-harness/lm_eval/tasks/medtext/README.md new file mode 100644 index 0000000000000000000000000000000000000000..97c746f6aecab4e84037493757decc48866b8d67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medtext/README.md @@ -0,0 +1,33 @@ +# MedText + +### Paper + +Title: `Towards Automatic Generation of Shareable Synthetic Clinical Notes Using Neural Language Models` + +Abstract: [https://arxiv.org/abs/1905.07002](https://arxiv.org/abs/1905.07002) + +MedText is a medical diagnosis dataset containing over 1000 top notch textbook +quality patient presentations and diagnosis/treatments. The 100 most common diseases +and the 30 most common injuries people go to the hospital with, are, among others, +fully captured in the dataset, with multiple datapoints for each ranging from mild +to complicated to severe. + + +#### Tasks + +* `medtext`: Open-ended QA in english. +* `medtext_perplexity`: Open-ended QA in english, evaluated with perplexity. + +### Citation + +```bibtex +@misc{melamud2019automaticgenerationshareablesynthetic, + title={Towards Automatic Generation of Shareable Synthetic Clinical Notes Using Neural Language Models}, + author={Oren Melamud and Chaitanya Shivade}, + year={2019}, + eprint={1905.07002}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/1905.07002}, +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/medtext/medtext.yaml b/lm-evaluation-harness/lm_eval/tasks/medtext/medtext.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7756cc64d1060fcd90f49e07c042f3ad5731e5cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medtext/medtext.yaml @@ -0,0 +1,36 @@ +task: medtext +dataset_path: BI55/MedText +description: > + Instructions: The following text is from a collection of medical records. What follows is the patients record. Answer how a doctor would, what is the likely diagnosis, and what is the treatment?. Answer how a doctor would, what is the likely diagnosis, and what is the treatment? + +output_type: generate_until +training_split: train +validation_split: train +test_split: train +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n\n" +metric_list: + - metric: bleu + aggregation: nanmean + higher_is_better: true + - metric: rouge1 + aggregation: nanmean + higher_is_better: true + - metric: rouge2 + aggregation: nanmean + higher_is_better: true + - metric: rougeL + aggregation: nanmean + higher_is_better: true + - metric: bleurt + aggregation: nanmean + higher_is_better: true + - metric: bert_score + aggregation: nanmean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/medtext/medtext_perplexity.yaml b/lm-evaluation-harness/lm_eval/tasks/medtext/medtext_perplexity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a89e5f2be1d219dcabfda99a5f69d76c6ddef4e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medtext/medtext_perplexity.yaml @@ -0,0 +1,14 @@ +include: medtext.yaml +task: medtext_perplexity +output_type: loglikelihood_rolling +doc_to_text: "" +process_results: !function utils_perplexity.process_results +metric_list: + - metric: word_perplexity + higher_is_better: false + - metric: byte_perplexity + higher_is_better: false + - metric: bits_per_byte + higher_is_better: false +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/medtext/utils.py b/lm-evaluation-harness/lm_eval/tasks/medtext/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f9827f41178e7db1e2e8960a06b387659e9c83b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medtext/utils.py @@ -0,0 +1,95 @@ +import numpy as np + + +try: + import evaluate + + bleu = evaluate.load("bleu") + rouge = evaluate.load("rouge") + bertscore = evaluate.load("bertscore") + bleurt = evaluate.load("bleurt", "bleurt-base-512", module_type="metric") + +except (ModuleNotFoundError, ImportError): + raise ModuleNotFoundError( + "Please install evaluation metrics via pip install evaluate and pip install bert-score", + ) +except Exception as e: + raise RuntimeError( + f"Error loading evaluation metrics: {str(e)}. Please check your installation." + ) + + +def doc_eval(pred, refs): + try: + bleu_results = bleu.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Bleu error: {e}") + bleu_results = {"bleu": np.NAN} + + try: + rouge_results = rouge.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Rouge error: {e}") + rouge_results = {"rouge1": np.NAN, "rouge2": np.NAN, "rougeL": np.NAN} + + try: + bleurt_scores = bleurt.compute(predictions=pred, references=refs)["scores"] + except Exception as e: + print(f"Bleurt error: {e}") + bleurt_scores = [np.NAN] + + try: + bert_scores = bertscore.compute(predictions=pred, references=refs, lang="en")[ + "f1" + ] + except Exception as e: + print(f"Bert error: {e}") + bert_scores = [np.NAN] + + if bleu_results["bleu"] == 0: + # Sometimes bleu is 0.0 and this breaks the stderr computation. + bleu_results["bleu"] += 1e-5 + + results = { + "bleu": bleu_results["bleu"], + "rouge1": rouge_results["rouge1"], + "rouge2": rouge_results["rouge2"], + "rougeL": rouge_results["rougeL"], + "bleurt": np.mean(bleurt_scores), + "bert_score": np.mean(bert_scores), + } + + return results + + +def doc_to_text(doc) -> str: + return doc["Prompt"] + + +def doc_to_target(doc) -> str: + return doc["Completion"] + + +def process_results(doc, results): + pred, refs = [results[0]], [doc_to_target(doc)] + + if len(refs[0]) < 1 or len(pred[0]) < 1: + return { + "bleu": np.NAN, + "rouge1": np.NAN, + "rouge2": np.NAN, + "rougeL": np.NAN, + "bleurt": np.NAN, + "bert_score": np.NAN, + } + + results = doc_eval(pred, refs) + + return { + "bleu": results["bleu"], + "rouge1": results["rouge1"], + "rouge2": results["rouge2"], + "rougeL": results["rougeL"], + "bleurt": results["bleurt"], + "bert_score": results["bert_score"], + } diff --git a/lm-evaluation-harness/lm_eval/tasks/medtext/utils_perplexity.py b/lm-evaluation-harness/lm_eval/tasks/medtext/utils_perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..cc81a0b43627bd64f87985a9b560cb0deff9b3f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/medtext/utils_perplexity.py @@ -0,0 +1,14 @@ +import re + +from lm_eval.tasks.medtext.utils import doc_to_target + + +def process_results(doc, results): + (loglikelihood,) = results + _words = len(re.split(r"\s+", doc_to_target(doc))) + _bytes = len(doc_to_target(doc).encode("utf-8")) + return { + "word_perplexity": (loglikelihood, _words), + "byte_perplexity": (loglikelihood, _bytes), + "bits_per_byte": (loglikelihood, _bytes), + } diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/README.md b/lm-evaluation-harness/lm_eval/tasks/mela/README.md new file mode 100644 index 0000000000000000000000000000000000000000..de008e494dc0fa46d06d5208305bdadde60b6441 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/README.md @@ -0,0 +1,60 @@ +# Task-name + +### Paper + +Title: [MELA: Multilingual Evaluation of Linguistic Acceptability](https://arxiv.org/abs/2311.09033) + +**Abstract**: In this work, we present the largest benchmark to date on linguistic acceptability: Multilingual Evaluation of Linguistic Acceptability -- MELA, with 46K samples covering 10 languages from a diverse set of language families. We establish LLM baselines on this benchmark, and investigate cross-lingual transfer in acceptability judgements with XLM-R. In pursuit of multilingual interpretability, we conduct probing experiments with fine-tuned XLM-R to explore the process of syntax capability acquisition. Our results show that GPT-4o exhibits a strong multilingual ability, outperforming fine-tuned XLM-R, while open-source multilingual models lag behind by a noticeable gap. Cross-lingual transfer experiments show that transfer in acceptability judgment is non-trivial: 500 Icelandic fine-tuning examples lead to 23 MCC performance in a completely unrelated language -- Chinese. Results of our probing experiments indicate that training on MELA improves the performance of XLM-R on syntax-related tasks. + +Homepage: https://github.com/sjtu-compling/MELA + +### Citation + +``` +@inproceedings{zhang2023mela, + author = {Ziyin Zhang and + Yikang Liu and + Weifang Huang and + Junyu Mao and + Rui Wang and + Hai Hu}, + title = {{MELA:} Multilingual Evaluation of Linguistic Acceptability}, + booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2024, Bangkok, Thailand}, + publisher = {Association for Computational Linguistics}, + year = {2024}, + url = {https://doi.org/10.48550/arXiv.2311.09033} +} +``` + +### Groups and Tasks + +#### Groups + +- `mela`: multilingual evaluation of linguistic acceptability + +#### Tasks + +- `mela_en`: English +- `mela_zh`: Chinese +- `mela_it`: Italian +- `mela_ru`: Russian +- `mela_de`: Germany +- `mela_fr`: French +- `mela_es`: Spanish +- `mela_ja`: Japanese +- `mela_ar`: Arabic +- `mela_ar`: Icelandic + +### Checklist + +For adding novel benchmarks/datasets to the library: + +- [x] Is the task an existing benchmark in the literature? + - [x] Have you referenced the original paper that introduced the task? + - [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: + +- [ ] Is the "Main" variant of this task clearly denoted? +- [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +- [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/_mela.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/_mela.yaml new file mode 100644 index 0000000000000000000000000000000000000000..107498c0b23a233690b6591078c5e65fd20b37d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/_mela.yaml @@ -0,0 +1,17 @@ +group: mela +task: + - mela_en + - mela_zh + - mela_it + - mela_ru + - mela_de + - mela_fr + - mela_es + - mela_ja + - mela_ar + - mela_ar +aggregate_metric_list: + - metric: mcc + weight_by_size: False +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eea6026a6dec0baf3cb0dcb764b1334399f1311a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_ar.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_ar +dataset_name: ar +training_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3108142d36db4efdc00d254ae4e21fd7170bd334 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_de.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_de +dataset_name: de +training_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5727b785993c98fad6b2bd630c8dbd076ff60c45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_en.yaml @@ -0,0 +1,17 @@ +task: mela_en +dataset_path: Geralt-Targaryen/MELA +dataset_name: en +training_split: train +validation_split: dev +test_split: test +output_type: multiple_choice +doc_to_text: "Sentence: {{sentence}}\nDetermine whether this sentence is acceptable or unacceptable?\nA. Acceptable\nB. Unacceptable\nAnswer:" +doc_to_choice: ["A", "B"] +doc_to_target: "{{['B', 'A'][label]}}" +description: "Determine whether the following sentence(s) violate certain linguistic constraints. If yes, then it is \"unacceptable\"; otherwise, \"acceptable\".\n\n" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: mcc + higher_is_better: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8c6716243055222e4c07a37ee90d9bd0d33b058 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_es.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_es +dataset_name: es +training_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bf68ad6a43ba7866bf836663b6f58b558db8e68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_fr.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_fr +dataset_name: fr +training_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_is.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_is.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd312f227e35b62bd8438368c741295a2a4bdddd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_is.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_is +dataset_name: is +training_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_it.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1751d65a7d60921446db78c001b41a51739d9249 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_it.yaml @@ -0,0 +1,3 @@ +include: mela_en.yaml +task: mela_it +dataset_name: it diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb8477e06028d32741816b2a0deac04975fa3ffc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_ja.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_ja +dataset_name: ja +training_split: null diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_ru.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..191a9a1c881c665ca779e624bde1bd21d65d0062 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_ru.yaml @@ -0,0 +1,3 @@ +include: mela_en.yaml +task: mela_ru +dataset_name: ru diff --git a/lm-evaluation-harness/lm_eval/tasks/mela/mela_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mela/mela_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3456abcdda3dac1529c429159a36366509cab13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mela/mela_zh.yaml @@ -0,0 +1,3 @@ +include: mela_en.yaml +task: mela_zh +dataset_name: zh diff --git a/lm-evaluation-harness/lm_eval/tasks/meqsum/README.md b/lm-evaluation-harness/lm_eval/tasks/meqsum/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e34c13b654679c9571fe41393e5539992c29b90f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/meqsum/README.md @@ -0,0 +1,32 @@ +# MeqSum + +### Paper + +Title: `On the Summarization of Consumer Health Questions` + +Abstract: [https://aclanthology.org/P19-1215/](https://aclanthology.org/P19-1215/) + +Question understanding is one of the main challenges in question answering. In real world +applications, users often submit natural language questions that are longer than needed +and include peripheral information that increases the complexity of the question, leading +to substantially more false positives in answer retrieval. In this paper, we study neural +abstractive models for medical question summarization. We introduce the MeQSum corpus of +1,000 summarized consumer health questions. + + +### Citation + +```bibtex +@inproceedings{ben-abacha-demner-fushman-2019-summarization, + title = "On the Summarization of Consumer Health Questions", + author = "Ben Abacha, Asma and + Demner-Fushman, Dina", + booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics", + month = jul, + year = "2019", + address = "Florence, Italy", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/P19-1215", + doi = "10.18653/v1/P19-1215", + pages = "2228--2234"} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/meqsum/meqsum.yaml b/lm-evaluation-harness/lm_eval/tasks/meqsum/meqsum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9de006cb751e700b6c92426bbd04920d74095ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/meqsum/meqsum.yaml @@ -0,0 +1,37 @@ +task: meqsum +dataset_path: bigbio/meqsum +dataset_name: meqsum_source +description: > + Instructions: The following text is contains a medical question. Extract and summarize the question. + +output_type: generate_until +training_split: train +validation_split: train +test_split: train +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +process_results: !function utils.process_results_gen +generation_kwargs: + until: + - "\n\n" +metric_list: + - metric: bleu + aggregation: nanmean + higher_is_better: true + - metric: rouge1 + aggregation: nanmean + higher_is_better: true + - metric: rouge2 + aggregation: nanmean + higher_is_better: true + - metric: rougeL + aggregation: nanmean + higher_is_better: true + - metric: bert_score + aggregation: nanmean + higher_is_better: true + - metric: bleurt + aggregation: nanmean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/meqsum/utils.py b/lm-evaluation-harness/lm_eval/tasks/meqsum/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d8a55df161eadf493fec26d22adcea76defdce1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/meqsum/utils.py @@ -0,0 +1,85 @@ +import numpy as np + + +try: + import evaluate + + bleu = evaluate.load("bleu") + rouge = evaluate.load("rouge") + bertscore = evaluate.load("bertscore") + bleurt = evaluate.load("bleurt", "bleurt-base-512", module_type="metric") + +except (ModuleNotFoundError, ImportError): + raise ModuleNotFoundError( + "Please install evaluation metrics via pip install evaluate and pip install bert-score", + ) +except Exception as e: + raise RuntimeError( + f"Error loading evaluation metrics: {str(e)}. Please check your installation." + ) + + +def doc_to_text(doc) -> str: + text = doc["CHQ"] + idx = text.find("MESSAGE") + if idx != -1: + return text[idx + 9 :] + else: + return text + + +def doc_to_target(doc) -> str: + return doc["Summary"] + + +def process_results_gen(doc, results): + pred, refs = [results[0]], [doc_to_target(doc)] + + if len(refs[0]) < 1 or len(pred[0]) < 1: + return { + "bleu": np.NAN, + "rouge1": np.NAN, + "rouge2": np.NAN, + "rougeL": np.NAN, + "bleurt": np.NAN, + "bert_score": np.NAN, + } + + try: + bleu_results = bleu.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Bleu error: {e}") + bleu_results = {"bleu": np.NAN} + + try: + rouge_results = rouge.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Rouge error: {e}") + rouge_results = {"rouge1": np.NAN, "rouge2": np.NAN, "rougeL": np.NAN} + + try: + bleurt_scores = bleurt.compute(predictions=pred, references=refs)["scores"] + except Exception as e: + print(f"Bleurt error: {e}") + bleurt_scores = [np.NAN] + + try: + bert_scores = bertscore.compute(predictions=pred, references=refs, lang="en")[ + "f1" + ] + except Exception as e: + print(f"Bert error: {e}") + bert_scores = [np.NAN] + + if bleu_results["bleu"] == 0: + # Sometimes bleu is 0.0 and this breaks the stderr computation. + bleu_results["bleu"] += 1e-5 + + return { + "bleu": bleu_results["bleu"], + "rouge1": rouge_results["rouge1"], + "rouge2": rouge_results["rouge2"], + "rougeL": rouge_results["rougeL"], + "bleurt": np.mean(bleurt_scores), + "bert_score": np.mean(bert_scores), + } diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/README.md b/lm-evaluation-harness/lm_eval/tasks/metabench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6e9ac427915eaedd498865d76af18df8b146ddbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/README.md @@ -0,0 +1,84 @@ +# Metabench + +### Paper + +Title: `metabench` -- A Sparse Benchmark to Measure General Ability in Large Language Models + +Abstract: https://arxiv.org/abs/2407.12844 + +Large Language Models (LLMs) vary in their abilities on a range of tasks. Initiatives such as the 𝙾𝚙𝚎𝚗 𝙻𝙻𝙼 𝙻𝚎𝚊𝚍𝚎𝚛𝚋𝚘𝚊𝚛𝚍 aim to quantify these differences with several large benchmarks (sets of test items to which an LLM can respond either correctly or incorrectly). However, high correlations within and between benchmark scores suggest that (1) there exists a small set of common underlying abilities that these benchmarks measure, and (2) items tap into redundant information and the benchmarks may thus be considerably compressed. We use data from $n> 5000$ LLMs to identify the most informative items of six benchmarks, ARC, GSM8K, HellaSwag, MMLU, TruthfulQA and WinoGrande (with d=28,632 items in total). From them we distill a sparse benchmark, `metabench`, that has less than $3%$ of the original size of all six benchmarks combined. This new sparse benchmark goes beyond point scores by yielding estimators of the underlying benchmark-specific abilities. We show that these estimators (1) can be used to reconstruct each original individual benchmark score with, on average, $1.5%$ root mean square error (RMSE), (2) reconstruct the original total score with $0.8%$ RMSE, and (3) have a single underlying common factor whose Spearman correlation with the total score is $r=0.93$. + +Homepage: https://github.com/adkipnis/metabench + + +### Citation + +```bibtex +@article{metabench, + author = {Alex Kipnis and Konstantinos Voudouris and Luca M. Schulze Buschoff and Eric Schulz}, + title = {metabench - A Sparse Benchmark to Measure General Ability in Large Language Models}, + journal = {arXiv preprint arXiv:2407.12844}, + year = {2024}, +} +``` + +### Groups and Tasks + +#### Groups + +There are four groups. + +* `metabench` -- combines the six tasks covering the six reduced benchmarks, using the original data and transformations from the respective benchmarks, and produces an aggregated mean score. It contains a total of 858 items. +* `metabench_permute` -- combines five tasks covering five of the reduced benchmarks, permuting the multiple choice ordering, and produces an aggregated mean score. It contains a total of 858 items. For more details, see immediately below. +* `metabench_secondary` -- combines the six tasks covering the six reduced benchmarks, using the original data and transformations from the respective benchmarks, and produces an aggregated mean score. These items are distinct from the items in the `metabench` group, and offer similar (although slightly worse) predictability of overall benchmark performance. We include it as a secondary evaluation resource. It contains a total of 751 items. +* `metabench_secondary_permute` -- combines five tasks covering five of the reduced benchmarks used in `metabench_secondary`, permuting the multiple choice ordering, and produces an aggregated mean score. It contains a total of 751 items. For more details, see immediately below. + +#### Tasks + +We offer four sets of tasks. The first uses the original benchmark items straight out of the box. + +* `metabench_arc` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 145 most informative items. +* `metabench_gsm8k` -- a subset of the [GSM8K benchmark](https://huggingface.co/datasets/openai/gsm8k) containing the 237 most informative items. +* `metabench_hellaswag` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 93 most informative items. +* `metabench_mmlu` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 96 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)). +* `metabench_truthfulqa` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 154 most informative items. +* `metabench_winogrande` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 133 most informative items. + +Since the original benchmarks are open-source, there is a risk of contamination. To mitigate this risk, we also provide tasks in which the answers are shuffled. Since `GSM8K` is not a multiple-choice benchmark, it is excluded from this set. + +* `metabench_arc_permute` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 145 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_hellaswag_permute` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 93 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_mmlu_permute` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 96 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)). The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_truthfulqa_permute` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 154 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_winogrande_permute` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 133 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. + +We also offer a second reduced benchmark that offers similar (although slightly worse) predictability of overall benchmark performance. We include it as a secondary evaluation resource. The first set of tasks uses the original benchmark items straight out of the box. + +* `metabench_arc_secondary` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 100 most informative items. +* `metabench_gsm8k_secondary` -- a subset of the [GSM8K benchmark](https://huggingface.co/datasets/openai/gsm8k) containing the 249 most informative items. +* `metabench_hellaswag_secondary` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 58 most informative items. +* `metabench_mmlu_secondary` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 102 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)). +* `metabench_truthfulqa_secondary` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 136 most informative items. +* `metabench_winogrande_secondary` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 106 most informative items. + +The fourth set of tasks permute the choices in five of the above datasets. + +* `metabench_arc_secondary_permute` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 100 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_hellaswag_secondary_permute` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 58 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_mmlu_secondary_permute` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 102 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)). The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_truthfulqa_secondary_permute` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 136 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. +* `metabench_winogrande_secondary_permute` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 106 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark. + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [X] Is the task an existing benchmark in the literature? + * [X] Have you referenced the original paper that introduced the task? + * [X] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [X] Is the "Main" variant of this task clearly denoted? +* [X] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [X] Have you noted which, if any, published evaluation setups are matched by this variant? +* diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e276d2e90e57d569c7fc8e21de7e2e715e4aa1d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench.yaml @@ -0,0 +1,14 @@ +group: metabench +task: + - metabench_arc + - metabench_gsm8k + - metabench_hellaswag + - metabench_mmlu + - metabench_truthfulqa + - metabench_winogrande +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: false +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bae54ae5bcb15f5b7bcec9662f51713b1e2918b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc.yaml @@ -0,0 +1,23 @@ +task: metabench_arc +tag: + - metabench_arc_subset +dataset_path: HCAI/metabench +dataset_name: ARC +process_docs: !function process_docs.process_arc +output_type: multiple_choice +training_split: null +validation_split: null +test_split: primary +num_fewshot: 0 +doc_to_text: "{{twentyfive_shot_preprompt}}Question: {{question}}\nAnswer:" +doc_to_target: "{{choices.label.index(answerKey)}}" +doc_to_choice: "{{choices.text}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82c2d68b08ef300a4a298c46ae112b412a21d653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_arc.yaml +task: metabench_arc_permute +process_docs: !function process_docs_permute.process_arc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a33bf3661c124ec28c2c6243a1b327f047511b14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary.yaml @@ -0,0 +1,5 @@ +include: metabench_arc.yaml +task: metabench_arc_secondary +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eadbd7e53bf86a744703eb9921ca11db90635fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_arc_permute.yaml +task: metabench_arc_secondary_permute +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_gsm8k.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_gsm8k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c72eddc8ce822ef4327fdd1c3c9146cdc37b8e3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_gsm8k.yaml @@ -0,0 +1,46 @@ +task: metabench_gsm8k +tag: + - metabench_gsm8k_subset +dataset_path: HCAI/metabench +dataset_name: GSM8K +process_docs: !function process_docs.process_gsm8k +output_type: generate_until +training_split: null +validation_split: null +test_split: primary +doc_to_text: "{{five_shot_preprompt}}Question: {{question}}\nAnswer:" +doc_to_target: "{{answer}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" + - "(?s).*#### " + - "\\.$" +generation_kwargs: + until: + - "Question:" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +repeats: 1 +num_fewshot: 0 +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "#### (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: "take_first" +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_gsm8k_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_gsm8k_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..263b932a703c7120c448a611f214819b7e825180 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_gsm8k_secondary.yaml @@ -0,0 +1,5 @@ +include: metabench_gsm8k.yaml +task: metabench_gsm8k_secondary +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66e20228098d46b3bb0fe94ba91ac5dd519d39b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag.yaml @@ -0,0 +1,23 @@ +task: metabench_hellaswag +tag: + - metabench_hellaswag_subset +dataset_path: HCAI/metabench +dataset_name: HellaSwag +process_docs: !function process_docs.process_hellaswag +output_type: multiple_choice +training_split: null +validation_split: null +test_split: primary +num_fewshot: 0 +doc_to_text: "{{ten_shot_preprompt}}{{query}}" +doc_to_target: "{{label}}" +doc_to_choice: "choices" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e45d21618a741d57bd41404110eeafd84488a34a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_hellaswag.yaml +task: metabench_hellaswag_permute +process_docs: !function process_docs_permute.process_hellaswag +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01241bfa860a899bdc40a0c3515309468d165f0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary.yaml @@ -0,0 +1,5 @@ +include: metabench_hellaswag.yaml +task: metabench_hellaswag_secondary +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12620a099c445930ede856a9963184703520d54d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_hellaswag_permute.yaml +task: metabench_hellaswag_secondary_permute +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8e7295320b3651fc72891ea7e496614974e7d32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu.yaml @@ -0,0 +1,20 @@ +task: metabench_mmlu +tag: + - metabench_mmlu_subset +dataset_path: HCAI/metabench +dataset_name: MMLU +process_docs: !function process_docs.process_mmlu +output_type: multiple_choice +training_split: null +validation_split: null +test_split: primary +num_fewshot: 0 +doc_to_text: "{{five_shot_preprompt}}{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26dc5263a9d97b392cd7e36f328c90ed01c25e3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_mmlu.yaml +task: metabench_mmlu_permute +process_docs: !function process_docs_permute.process_mmlu +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e40f446afc72b674a7b7d1b998e68fd0ccff2fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary.yaml @@ -0,0 +1,5 @@ +include: metabench_mmlu.yaml +task: metabench_mmlu_secondary +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f7b31b91f62941f33883cc0f44e0250796803cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_mmlu_permute.yaml +task: metabench_mmlu_secondary_permute +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5dc1206be88918cb624728b3d1e70c1d107c6ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_permute.yaml @@ -0,0 +1,13 @@ +group: metabench_permute +task: + - metabench_arc_permute + - metabench_hellaswag_permute + - metabench_mmlu_permute + - metabench_truthfulqa_permute + - metabench_winogrande_permute +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: false +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3139a59a630afa9ea1f255ca37a5c71e0707467c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary.yaml @@ -0,0 +1,14 @@ +group: metabench_secondary +task: + - metabench_arc_secondary + - metabench_gsm8k_secondary + - metabench_hellaswag_secondary + - metabench_mmlu_secondary + - metabench_truthfulqa_secondary + - metabench_winogrande_secondary +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: false +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3707bc2e8708e1494beb852fbc3337a8b9654161 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary_permute.yaml @@ -0,0 +1,13 @@ +group: metabench_secondary_permute +task: + - metabench_arc_secondary_permute + - metabench_hellaswag_secondary_permute + - metabench_mmlu_secondary_permute + - metabench_truthfulqa_secondary_permute + - metabench_winogrande_secondary_permute +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: false +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f269593ff18adbcbdc3c292c2826ec63094548e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa.yaml @@ -0,0 +1,35 @@ +task: metabench_truthfulqa +tag: + - metabench_truthfulqa_subset +dataset_path: HCAI/metabench +dataset_name: TruthfulQA +process_docs: !function process_docs.process_truthfulqa +output_type: multiple_choice +training_split: null +validation_split: null +test_split: primary +num_fewshot: 0 +doc_to_text: "\ + {% set prompt_qa = '\ + Q: What is human life expectancy in the United States?\n\ + A: Human life expectancy in the United States is 78 years.\n\n\ + Q: Who was president of the United States in 1955?\n\ + A: Dwight D. Eisenhower was president of the United States in 1955.\n\n\ + Q: Which party did he belong to?\n\ + A: He belonged to the Republican Party.\n\n\ + Q: What is the square root of banana?\n\ + A: I have no comment.\n\n\ + Q: How does a telescope work?\n\ + A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n\ + Q: Where were the 1992 Olympics held?\n\ + A: The 1992 Olympics were held in Barcelona, Spain.\ + '%}\ + {{prompt_qa + '\n\nQ: ' + question + '\nA:'}}" +doc_to_target: 0 +doc_to_choice: "{{mc1_targets.choices}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b51aadb3f9f1bfc31e2c17637a5f63c457ad573 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_permute.yaml @@ -0,0 +1,6 @@ +include: metabench_truthfulqa.yaml +task: metabench_truthfulqa_permute +process_docs: !function process_docs_permute.process_truthfulqa +doc_to_target: answer +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6109281f1459efc942a1a26d245673c4502a5da8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary.yaml @@ -0,0 +1,5 @@ +include: metabench_truthfulqa.yaml +task: metabench_truthfulqa_secondary +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dab0fb0135b1c9b2a1cc01cdb0210237e25dbaaa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_truthfulqa_permute.yaml +task: metabench_truthfulqa_secondary_permute +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a5a25536b00692635fb37b32f7f2df449e16dbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande.yaml @@ -0,0 +1,20 @@ +task: metabench_winogrande +tag: + - metabench_winogrande_subset +dataset_path: HCAI/metabench +dataset_name: Winogrande +process_docs: !function process_docs.process_winogrande +output_type: multiple_choice +training_split: null +validation_split: null +test_split: primary +num_fewshot: 0 +doc_to_text: !function process_docs.winogrande_doc_to_text +doc_to_target: !function process_docs.winogrande_doc_to_target +doc_to_choice: !function process_docs.winogrande_doc_to_choice +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b38196ae2bb60384df4b189e8b4dbc4c93ba04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_winogrande.yaml +task: metabench_winogrande_permute +process_docs: !function process_docs_permute.process_winogrande +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e5b2ac6f461ba4bbc59ebfdedb2e69ad25d87c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary.yaml @@ -0,0 +1,5 @@ +include: metabench_winogrande.yaml +task: metabench_winogrande_secondary +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary_permute.yaml b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary_permute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f4428712ce9cc29d7b2b910a51306c131526245 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary_permute.yaml @@ -0,0 +1,5 @@ +include: metabench_winogrande_permute.yaml +task: metabench_winogrande_secondary_permute +test_split: secondary +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs.py b/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..8f8b0c81329a3835d195b2049dcc2751e9b74f0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs.py @@ -0,0 +1,186 @@ +import hashlib +import re + +import datasets + + +def hash_string(string: str) -> str: + return hashlib.sha256(string.encode("utf-8")).hexdigest() + + +def process_arc(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + long_prompt = "" + for shot in range(1, 26): + question = doc[f"arc_question_shot_{shot}"] + doc.pop(f"arc_question_shot_{shot}") + answer_lab = doc[f"arc_answerKey_shot_{shot}"] + doc.pop(f"arc_answerKey_shot_{shot}") + answer_idx = doc[f"arc_choices_shot_{shot}"]["label"].index(answer_lab) + answer = doc[f"arc_choices_shot_{shot}"]["text"][answer_idx] + doc.pop(f"arc_choices_shot_{shot}") + doc.pop(f"arc_idx_shot_{shot}") + + long_prompt = f"{long_prompt}Question: {question}\nAnswer: {answer}\n\n" # no choices are provided in the few-shot setting (per lines 602-610 of lm_eval.api.task) + doc["twentyfive_shot_preprompt"] = long_prompt + doc["original_hash"] = hash_string(doc["question"]) + doc.pop("alltwentyfiveshot_longprompt") + return doc + + return dataset.map(_subprocess) + + +def process_gsm8k(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + long_prompt = "" + for shot in range(1, 6): + question = doc[f"gsm8k_prompt_shot_{shot}"] + doc.pop(f"gsm8k_prompt_shot_{shot}") + answer = doc[f"gsm8k_answer_shot_{shot}"] + doc.pop(f"gsm8k_answer_shot_{shot}") + doc.pop(f"gsm8k_idx_shot_{shot}") + + long_prompt = f"{long_prompt}Question: {question}\nAnswer: {answer}\n\n" # no choices are provided in the few-shot setting (per lines 602-610 of lm_eval.api.task) + doc["original_hash"] = hash_string(doc["question"]) + doc["five_shot_preprompt"] = long_prompt + doc.pop("allfiveshot_longprompt") + return doc + + return dataset.map(_subprocess) + + +def process_hellaswag(dataset: datasets.Dataset) -> datasets.Dataset: + def process_txt(text): # mirrored from hellaswag task + text = text.strip() + # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + def _preprocess(doc): + ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() + doc.pop("ctx_a") + doc.pop("ctx_b") + doc.pop("ctx") + doc["query"] = process_txt(doc["activity_label"] + ": " + ctx) + doc["choices"] = [process_txt(ending) for ending in doc["endings"]] + doc["gold"] = int(doc["label"]) + doc.pop("activity_label") + doc.pop("endings") + + long_prompt = "" + for shot in range(1, 11): + ctx = ( + doc[f"hellaswag_ctx_a_shot_{shot}"] + + " " + + doc[f"hellaswag_ctx_b_shot_{shot}"].capitalize() + ) + doc.pop(f"hellaswag_ctx_a_shot_{shot}") + doc.pop(f"hellaswag_ctx_b_shot_{shot}") + doc.pop(f"hellaswag_ctx_shot_{shot}") + question = process_txt( + doc[f"hellaswag_activity_labels_shot_{shot}"] + ": " + ctx + ) + ending = process_txt( + doc[f"hellaswag_endings_shot_{shot}"][ + int(doc[f"hellaswag_label_shot_{shot}"]) + ] + ) + doc.pop(f"hellaswag_activity_labels_shot_{shot}") + doc.pop(f"hellaswag_endings_shot_{shot}") + doc.pop(f"hellaswag_label_shot_{shot}") + + long_prompt = f"{long_prompt}{question} {ending}\n\n" + + doc.pop(f"hellaswag_ind_shot_{shot}") + doc.pop(f"hellaswag_source_id_shot_{shot}") + doc.pop(f"hellaswag_split_shot_{shot}") + doc.pop(f"hellaswag_split_type_shot_{shot}") + + doc["original_hash"] = hash_string(doc["query"]) + doc["ten_shot_preprompt"] = long_prompt + doc.pop("alltenshot_longprompt") + return doc + + return dataset.map(_preprocess) + + +def process_mmlu(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + choices = ["A", "B", "C", "D"] + long_prompt = f"The following are multiple choice questions (with answers) about {' '.join(doc['subject'].split('_'))}.\n\n" + for shot in range(1, 6): + question = doc[f"mmlu_question_shot_{shot}"].strip() + doc.pop(f"mmlu_question_shot_{shot}") + answer = choices[int(doc[f"mmlu_answers_shot_{shot}"])] + choice_A = doc[f"mmlu_choices_shot_{shot}"][0] + choice_B = doc[f"mmlu_choices_shot_{shot}"][1] + choice_C = doc[f"mmlu_choices_shot_{shot}"][2] + choice_D = doc[f"mmlu_choices_shot_{shot}"][3] + + doc.pop(f"mmlu_choices_shot_{shot}") + doc.pop(f"mmlu_answers_shot_{shot}") + doc.pop(f"mmlu_ind_shot_{shot}") + + long_prompt = f"{long_prompt}{question}\nA. {choice_A}\nB. {choice_B}\nC. {choice_C}\nD. {choice_D}\nAnswer: {answer}\n\n" # choices are provided in the mmlu few-shot regime, unlike other benchmarks. + + doc["original_hash"] = hash_string(doc["question"]) + doc["five_shot_preprompt"] = long_prompt + doc.pop("allfiveshot_longprompt") + return doc + + return dataset.map(_subprocess) + + +def process_truthfulqa(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + doc["original_hash"] = hash_string(doc["question"]) + return doc + + return dataset.map(_subprocess) + + +def process_winogrande(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + long_prompt = "" + for shot in range(1, 6): + if doc[f"winogrande_answer_shot_{shot}"] == "1": + answer = doc[f"winogrande_option1_shot_{shot}"] + elif doc[f"winogrande_answer_shot_{shot}"] == "2": + answer = doc[f"winogrande_option2_shot_{shot}"] + else: + raise ValueError("Answer not recognised.") + + question = doc[f"winogrande_prompt_shot_{shot}"].replace("_", answer) + + doc.pop(f"winogrande_prompt_shot_{shot}") + doc.pop(f"winogrande_answer_shot_{shot}") + doc.pop(f"winogrande_idx_shot_{shot}") + doc.pop(f"winogrande_option1_shot_{shot}") + doc.pop(f"winogrande_option2_shot_{shot}") + + long_prompt = f"{long_prompt}{question}\n\n" + sentence = doc["sentence"] + doc["original_hash"] = hash_string(doc["sentence"]) + doc["sentence"] = f"{long_prompt}{sentence}" + doc.pop("allfiveshot_longprompt") + return doc + + return dataset.map(_subprocess) + + +def winogrande_doc_to_text(doc): # Mirrored from the winogrande task + answer_to_num = {"1": 0, "2": 1} + return answer_to_num[doc["answer"]] + + +def winogrande_doc_to_target(doc): # Mirrored from the winogrande task + idx = doc["sentence"].index("_") + 1 + return doc["sentence"][idx:].strip() + + +def winogrande_doc_to_choice(doc): # Mirrored from the winogrande task + idx = doc["sentence"].index("_") + options = [doc["option1"], doc["option2"]] + return [doc["sentence"][:idx] + opt for opt in options] diff --git a/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs_permute.py b/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs_permute.py new file mode 100644 index 0000000000000000000000000000000000000000..cce323d457f7c4a951eeef2319361c8dd773e486 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs_permute.py @@ -0,0 +1,226 @@ +import hashlib +import random +import re + +import datasets + + +def hash_string(string: str) -> str: + return hashlib.sha256(string.encode("utf-8")).hexdigest() + + +def process_arc(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + long_prompt = "" + for shot in range(1, 26): + question = doc[f"arc_question_shot_{shot}"] + doc.pop(f"arc_question_shot_{shot}") + answer_lab = doc[f"arc_answerKey_shot_{shot}"] + doc.pop(f"arc_answerKey_shot_{shot}") + answer_idx = doc[f"arc_choices_shot_{shot}"]["label"].index(answer_lab) + answer = doc[f"arc_choices_shot_{shot}"]["text"][answer_idx] + doc.pop(f"arc_choices_shot_{shot}") + doc.pop(f"arc_idx_shot_{shot}") + long_prompt = f"{long_prompt}Question: {question}\nAnswer: {answer}\n\n" # no choices are provided in the few-shot setting (per lines 602-610 of lm_eval.api.task) + doc["twentyfive_shot_preprompt"] = long_prompt + doc.pop("alltwentyfiveshot_longprompt") + doc["original_hash"] = hash_string(doc["question"]) + + # permute choices randomly without replacement (the new answer label will never be the answer label recorded in the original benchmarks) + original_answer_idx = doc["choices"]["label"].index(doc["answerKey"]) + correct_answer_text = doc["choices"]["text"][original_answer_idx] + new_answer_idx = original_answer_idx + + while new_answer_idx is original_answer_idx: + random.shuffle(doc["choices"]["text"]) + new_answer_idx = doc["choices"]["text"].index(correct_answer_text) + doc["answerKey"] = doc["choices"]["label"][new_answer_idx] + + return doc + + return dataset.map(_subprocess) + + +def process_hellaswag(dataset: datasets.Dataset) -> datasets.Dataset: + def process_txt(text): # mirrored from hellaswag task + text = text.strip() + # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + def _preprocess(doc): + ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() + doc.pop("ctx_a") + doc.pop("ctx_b") + doc.pop("ctx") + doc["query"] = process_txt(doc["activity_label"] + ": " + ctx) + + # permute choices randomly without replacement (the new answer label will never be the answer label recorded in the original benchmarks) + original_answer_idx = int(doc["label"]) + correct_answer_text = doc["endings"][original_answer_idx] + new_answer_idx = original_answer_idx + while new_answer_idx is original_answer_idx: + random.shuffle(doc["endings"]) + new_answer_idx = doc["endings"].index(correct_answer_text) + doc["label"] = str(new_answer_idx) + + doc["choices"] = [process_txt(ending) for ending in doc["endings"]] + doc["gold"] = int(doc["label"]) + doc.pop("activity_label") + doc.pop("endings") + + long_prompt = "" + for shot in range(1, 11): + ctx = ( + doc[f"hellaswag_ctx_a_shot_{shot}"] + + " " + + doc[f"hellaswag_ctx_b_shot_{shot}"].capitalize() + ) + doc.pop(f"hellaswag_ctx_a_shot_{shot}") + doc.pop(f"hellaswag_ctx_b_shot_{shot}") + doc.pop(f"hellaswag_ctx_shot_{shot}") + question = process_txt( + doc[f"hellaswag_activity_labels_shot_{shot}"] + ": " + ctx + ) + ending = process_txt( + doc[f"hellaswag_endings_shot_{shot}"][ + int(doc[f"hellaswag_label_shot_{shot}"]) + ] + ) + doc.pop(f"hellaswag_activity_labels_shot_{shot}") + doc.pop(f"hellaswag_endings_shot_{shot}") + doc.pop(f"hellaswag_label_shot_{shot}") + long_prompt = f"{long_prompt}{question} {ending}\n\n" + doc.pop(f"hellaswag_ind_shot_{shot}") + doc.pop(f"hellaswag_source_id_shot_{shot}") + doc.pop(f"hellaswag_split_shot_{shot}") + doc.pop(f"hellaswag_split_type_shot_{shot}") + + doc["original_hash"] = hash_string(doc["query"]) + doc["ten_shot_preprompt"] = long_prompt + doc.pop("alltenshot_longprompt") + return doc + + return dataset.map(_preprocess) + + +def process_mmlu(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + choices = ["A", "B", "C", "D"] + long_prompt = f"The following are multiple choice questions (with answers) about {' '.join(doc['subject'].split('_'))}.\n\n" + for shot in range(1, 6): + question = doc[f"mmlu_question_shot_{shot}"].strip() + doc.pop(f"mmlu_question_shot_{shot}") + answer = choices[int(doc[f"mmlu_answers_shot_{shot}"])] + choice_A = doc[f"mmlu_choices_shot_{shot}"][0] + choice_B = doc[f"mmlu_choices_shot_{shot}"][1] + choice_C = doc[f"mmlu_choices_shot_{shot}"][2] + choice_D = doc[f"mmlu_choices_shot_{shot}"][3] + + doc.pop(f"mmlu_choices_shot_{shot}") + doc.pop(f"mmlu_answers_shot_{shot}") + doc.pop(f"mmlu_ind_shot_{shot}") + + long_prompt = f"{long_prompt}{question}\nA. {choice_A}\nB. {choice_B}\nC. {choice_C}\nD. {choice_D}\nAnswer: {answer}\n\n" # choices are provided in the mmlu few-shot regime, unlike other benchmarks. + + doc["original_hash"] = hash_string(doc["question"]) + doc["five_shot_preprompt"] = long_prompt + doc.pop("allfiveshot_longprompt") + + # permute choices randomly without replacement (the new answer label will never be the answer label recorded in the original benchmarks) + original_answer_idx = int(doc["answer"]) + correct_answer_text = doc["choices"][original_answer_idx] + new_answer_idx = original_answer_idx + + while new_answer_idx is original_answer_idx: + random.shuffle(doc["choices"]) + new_answer_idx = doc["choices"].index(correct_answer_text) + doc["answer"] = new_answer_idx + + return doc + + return dataset.map(_subprocess) + + +def process_truthfulqa(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess( + doc, + ): # currently only permuting the mc1 targets as metabench does not use mc2 targets. + original_answer_idx = 0 # always 0 in truthfulqa + correct_answer_text = doc["mc1_targets"]["choices"][original_answer_idx] + new_answer_idx = original_answer_idx + + while new_answer_idx is original_answer_idx: + random.shuffle(doc["mc1_targets"]["choices"]) + new_answer_idx = doc["mc1_targets"]["choices"].index(correct_answer_text) + + labels = [0] * len(doc["mc1_targets"]["labels"]) + labels[new_answer_idx] = 1 + doc["original_hash"] = hash_string(doc["question"]) + doc["mc1_targets"]["labels"] = labels + doc["answer"] = new_answer_idx + + return doc + + return dataset.map(_subprocess) + + +def process_winogrande(dataset: datasets.Dataset) -> datasets.Dataset: + def _subprocess(doc): + long_prompt = "" + for shot in range(1, 6): + if doc[f"winogrande_answer_shot_{shot}"] == "1": + answer = doc[f"winogrande_option1_shot_{shot}"] + elif doc[f"winogrande_answer_shot_{shot}"] == "2": + answer = doc[f"winogrande_option2_shot_{shot}"] + else: + raise ValueError("Answer not recognised.") + + question = doc[f"winogrande_prompt_shot_{shot}"].replace("_", answer) + + doc.pop(f"winogrande_prompt_shot_{shot}") + doc.pop(f"winogrande_answer_shot_{shot}") + doc.pop(f"winogrande_idx_shot_{shot}") + doc.pop(f"winogrande_option1_shot_{shot}") + doc.pop(f"winogrande_option2_shot_{shot}") + + long_prompt = f"{long_prompt}{question}\n\n" + sentence = doc["sentence"] + doc["original_hash"] = hash_string(doc["sentence"]) + doc["sentence"] = f"{long_prompt}{sentence}" + doc.pop("allfiveshot_longprompt") + + # permute choices by swapping them + option1 = doc["option1"] + option2 = doc["option2"] + answer = doc["answer"] + + doc["option1"] = option2 + doc["option2"] = option1 + + if answer == "1": + doc["answer"] = "2" + elif answer == "2": + doc["answer"] = "1" + + return doc + + return dataset.map(_subprocess) + + +def winogrande_doc_to_text(doc): # Mirrored from the winogrande task + answer_to_num = {"1": 0, "2": 1} + return answer_to_num[doc["answer"]] + + +def winogrande_doc_to_target(doc): # Mirrored from the winogrande task + idx = doc["sentence"].index("_") + 1 + return doc["sentence"][idx:].strip() + + +def winogrande_doc_to_choice(doc): # Mirrored from the winogrande task + idx = doc["sentence"].index("_") + options = [doc["option1"], doc["option2"]] + return [doc["sentence"][:idx] + opt for opt in options] diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/README.md b/lm-evaluation-harness/lm_eval/tasks/mgsm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3b62edf136e2e1482b4da45febf786b8f4fe4c7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/README.md @@ -0,0 +1,98 @@ +# MGSM + +### Paper + +Title: `Language Models are Multilingual Chain-of-Thought Reasoners` + +Abstract: https://arxiv.org/abs/2210.03057 + +Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems, proposed in the paper [Language models are multilingual chain-of-thought reasoners](http://arxiv.org/abs/2210.03057). + +The same 250 problems from [GSM8K](https://arxiv.org/abs/2110.14168) are each translated via human annotators in 10 languages. The 10 languages are: +- Spanish +- French +- German +- Russian +- Chinese +- Japanese +- Thai +- Swahili +- Bengali +- Telugu + +GSM8K (Grade School Math 8K) is a dataset of 8.5K high quality linguistically diverse grade school math word problems. The dataset was created to support the task of question answering on basic mathematical problems that require multi-step reasoning. + +You can find the input and targets for each of the ten languages (and English) as `.tsv` files. +We also include few-shot exemplars that are also manually translated from each language in `exemplars.py`. + +Homepage: https://github.com/google-research/url-nlp/tree/main/mgsm + + +### Citation + +``` +@misc{cobbe2021training, + title={Training Verifiers to Solve Math Word Problems}, + author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman}, + year={2021}, + eprint={2110.14168}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +@misc{shi2022language, + title={Language Models are Multilingual Chain-of-Thought Reasoners}, + author={Freda Shi and Mirac Suzgun and Markus Freitag and Xuezhi Wang and Suraj Srivats and Soroush Vosoughi and Hyung Won Chung and Yi Tay and Sebastian Ruder and Denny Zhou and Dipanjan Das and Jason Wei}, + year={2022}, + eprint={2210.03057}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` + +### Groups and Tasks + +#### Groups + +* `mgsm_direct`: Direct question + * `mgsm_direct_bn`: Bengali + * `mgsm_direct_de`: German + * `mgsm_direct_en`: English + * `mgsm_direct_es`: Spanish + * `mgsm_direct_fr`: French + * `mgsm_direct_ja`: Japanese + * `mgsm_direct_ru`: Russian + * `mgsm_direct_sw`: Swahili + * `mgsm_direct_te`: Telugu + * `mgsm_direct_th`: Thai + * `mgsm_direct_zh`: Chinese +* `mgsm_cot_native`: Question with Answer followed by CoT prompt in the same language as the dataset. + * `mgsm_cot_native_bn`: Bengali + * `mgsm_cot_native_de`: German + * `mgsm_cot_native_en`: English + * `mgsm_cot_native_es`: Spanish + * `mgsm_cot_native_fr`: French + * `mgsm_cot_native_ja`: Japanese + * `mgsm_cot_native_ru`: Russian + * `mgsm_cot_native_sw`: Swahili + * `mgsm_cot_native_te`: Telugu + * `mgsm_cot_native_th`: Thai + * `mgsm_cot_native_zh`: Chinese + +Examplar Samples: https://github.com/google-research/url-nlp/blob/main/mgsm/exemplars.py + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +# changelog +- (en_cot, direct) ver 3; (native_cot) ver 4: issue #2578; PR #2587 + - fix fewshot format: Changed inconsistent usage of ':' (ASCII) and '：' (Chinese) to use '：' consistently. diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/direct_yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dd83c0c92bad498e6b83f503ef136766d550223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/direct_yaml @@ -0,0 +1,35 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly +# by the harness. +tag: mgsm_direct +dataset_path: juletxara/mgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +target_delimiter: "" +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08e7125127eabeda6fdc08a6a3edd83c84ea277e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: bn +doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"প্রশ্ন: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'প্রশ্ন:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_bn diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24bc43eda3eaa1815919c9abc7d05697f53be309 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: de +doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAntwort:"}}{% else %}{{"Frage: "+question+"\nAntwort:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Frage:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7ef407d39f7addb0688366cfd98005ee7a8da6b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: en +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6c3c1fd7ed85050098cb4db48db2bdbb86c7db6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: es +doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Pregunta:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..993c181a97d59c71ee50b67d641995296d373e58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fr +doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRéponse :"}}{% else %}{{"Question : "+question+"\nRéponse :"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question :' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_fr diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9a1ce2bb1e9f07cdada7f95dd153fa31e70daf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ja +doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題： "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 問題： + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_ja diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30d1618faacf5712154132b200b333e519426b95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ru +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Задача: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Задача:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_ru diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0357902d4eea32b0f4619e32f6806599caac4ae5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sw +doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Swali: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Swali:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_sw diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4abdc7e78ec0ddd597d1ff2210a3474ad397a30a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: te +doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"ప్రశ్న: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'ప్రశ్న:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_te diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcf35a6721ab7faa221e023483c7630040b0e72f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: th +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"โจทย์: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'โจทย์:' + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_th diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..462a92c36a7b5d42b0d964538ac3a2a44bfb1c6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zh +doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题： "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 问题： + - + - <|im_end|> +include: direct_yaml +task: mgsm_direct_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/cot_yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/cot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f3fabaa767d67cc2d36e700e3f0081f3d5cde9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/cot_yaml @@ -0,0 +1,36 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly +# by the harness. +tag: mgsm_cot_native +dataset_path: juletxara/mgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 3.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1c3c2fcd75827bf0c574090bb2adbc3890bdaf4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: bn +doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"প্রশ্ন: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'প্রশ্ন:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_bn diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2362fb7ac0944da0eae570963603275d459a254 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: de +doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Frage: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Frage:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f27a616487aadcda9ac0f6f4e549d9bcd8e26dc1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: en +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc748306a473dd11beace7d35ac7453f187c7abb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: es +doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Pregunta: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Pregunta:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d36dd813a3b86b6300620ec5c74ad0154017edf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fr +doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question : "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question :' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_fr diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb324970fbba46b6b77c2a6fb397ad5bc60e5816 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ja +doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題： "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 問題： + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_ja diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bfeb1dafe3cbd989ba3999394b1ea9a294504f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ru +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Задача: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Задача:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_ru diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f37cd3b87eb3660a701eec29ca1d51cc3c630e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sw +doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Swali: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Swali:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_sw diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75da745da1b6c27350be39d9e7c535c1d3c93168 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: te +doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"ప్రశ్న: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'ప్రశ్న:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_te diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ff2177b782ef3c939dd649c484a9b5a83501333 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: th +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"โจทย์: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'โจทย์:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_th diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebc822d698cdacc90075b034ee37a44fd4027bed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zh +doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"问题： "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 问题： + - + - <|im_end|> +include: cot_yaml +task: mgsm_en_cot_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/gen_yaml.sh b/lm-evaluation-harness/lm_eval/tasks/mgsm/gen_yaml.sh new file mode 100644 index 0000000000000000000000000000000000000000..27cbbcfdc7ae6bddb463de0c7ceb8ec467ec9c3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/gen_yaml.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +python utils.py --overwrite --output-dir direct --mode direct +python utils.py --overwrite --output-dir en_cot --mode en-cot +python utils.py --overwrite --output-dir native_cot --mode native-cot diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/cot_yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/cot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..80e5f443e499e8534a56682e3eb20e692e622d00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/cot_yaml @@ -0,0 +1,31 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly +# by the harness. +tag: mgsm_cot_native +dataset_path: juletxara/mgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +# target_delimiter: "" +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "get-answer" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" +metadata: + version: 4.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb58c8753784c250ce24860fd21211b62ef0cc31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: bn +doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nধাপে ধাপে উত্তর:"}}{% else %}{{"প্রশ্ন: "+question+"\nধাপে ধাপে উত্তর:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: The answer is (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'প্রশ্ন:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_bn diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f4701796945b74fe884a73d931debdf2c7b5ce9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: de +doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nSchritt-für-Schritt-Antwort:"}}{% else %}{{"Frage: "+question+"\nSchritt-für-Schritt-Antwort:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: Die Antwort lautet (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Frage:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2033b335fb51ec1310f98b4e905f18231c1b68a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: en +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: The answer is (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c39fb9c4740ac571db8165a80fdd7efa108f56b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: es +doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta paso a paso:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta paso a paso:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: La respuesta es (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Pregunta:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b52b881f7a3f8b30d64ce8eb8ee6b308673626c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_fr.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: fr +doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRéponse étape par étape :"}}{% else %}{{"Question : "+question+"\nRéponse étape par étape :"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: La réponse est (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Question :' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_fr diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3715aca53b7bd9d4af3c95a95fa5ebdd8e7e000e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: ja +doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題： "+question+"\nステップごとの答え:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: 答えは(\-?[0-9\.\,]+)です。 + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 問題： + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_ja diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ru.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cff6267a067da1e9d10cfa66aaad7c06618f7ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ru.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: ru +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nПошаговоерешение:"}}{% else %}{{"Задача: "+question+"\nПошаговоерешение:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: Ответ — (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Задача:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_ru diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4da793dbc78485cb8167a6fc069b87f7590c960f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_sw.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: sw +doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nJibu la Hatua kwa Hatua:"}}{% else %}{{"Swali: "+question+"\nJibu la Hatua kwa Hatua:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: Jibu ni (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Swali:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_sw diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_te.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cdbaca8893b6ee626084135c7a64ccd02737b81 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_te.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: te +doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nదశలవారీగా సమాధానం:"}}{% else %}{{"ప్రశ్న: "+question+"\nదశలవారీగా సమాధానం:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: సమాధానం (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'ప్రశ్న:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_te diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_th.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6931d3a2ff44ab0de25a31a7624f2cd104c655c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_th.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: th +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nคำตอบทีละขั้นตอน:"}}{% else %}{{"โจทย์: "+question+"\nคำตอบทีละขั้นตอน:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: คำตอบคือ (\-?[0-9\.\,]+) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'โจทย์:' + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_th diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b45170ca33e27ad4208a876a24a3f8f2373676f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml @@ -0,0 +1,24 @@ +# Generated by utils.py +dataset_name: zh +doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题： "+question+"\n逐步解答:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: 答案是 (\-?[0-9\.\,]+)。 + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 问题： + - + - <|im_end|> +include: cot_yaml +task: mgsm_native_cot_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mgsm/utils.py b/lm-evaluation-harness/lm_eval/tasks/mgsm/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..54e39af9b90ae0a7da777da3ca0524467942b58e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mgsm/utils.py @@ -0,0 +1,228 @@ +import argparse + +import yaml + + +LANGUAGES = { + "bn": { # Bengali + # "QUESTION": "প্রশ্ন:", + "QUESTION": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8:", + # "ANSWER": "ধাপে ধাপে উত্তর:", + "ANSWER": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 \u0989\u09a4\u09cd\u09a4\u09b0:", + "DIRECT": "Answer:", + "REGEX": "The answer is (\\-?[0-9\\.\\,]+)", + }, + "de": { # German + "QUESTION": "Frage:", + # "ANSWER": "Schritt-für-Schritt-Antwort:", + "ANSWER": "Schritt-f\u00fcr-Schritt-Antwort:", + "DIRECT": "Antwort:", + "REGEX": "Die Antwort lautet (\\-?[0-9\\.\\,]+)", + }, + "en": { # English + "QUESTION": "Question:", + "ANSWER": "Step-by-Step Answer:", + "DIRECT": "Answer:", + "REGEX": "The answer is (\\-?[0-9\\.\\,]+)", + }, + "es": { # Spanish + "QUESTION": "Pregunta:", + "ANSWER": "Respuesta paso a paso:", + "DIRECT": "Respuesta:", + "REGEX": "La respuesta es (\\-?[0-9\\.\\,]+)", + }, + "fr": { # French + "QUESTION": "Question :", + # "ANSWER": "Réponse étape par étape :" + "ANSWER": "R\u00e9ponse \u00e9tape par \u00e9tape :", + # "DIRECT": "Réponse :", + "DIRECT": "R\u00e9ponse :", + # "REGEX": "La réponse est (\\-?[0-9\\.\\,]+)", + "REGEX": "La r\u00e9ponse est (\\-?[0-9\\.\\,]+)", + }, + "ru": { # Russian + # "QUESTION": "Задача:", + "QUESTION": "\u0417\u0430\u0434\u0430\u0447\u0430:", + # "ANSWER": "Пошаговоерешение:", + "ANSWER": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435\u0440\u0435\u0448\u0435\u043d\u0438\u0435:", + "DIRECT": "Answer:", + # "REGEX": "Ответ — (\\-?[0-9\\.\\,]+)", + "REGEX": "\u041e\u0442\u0432\u0435\u0442 \u2014 (\\-?[0-9\\.\\,]+)", + }, + "sw": { # Swahili + "QUESTION": "Swali:", + "ANSWER": "Jibu la Hatua kwa Hatua:", + "DIRECT": "Answer:", + "REGEX": "Jibu ni (\\-?[0-9\\.\\,]+)", + }, + "te": { # Telugu + # "QUESTION": "ప్రశ్న:", + "QUESTION": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28:", + # "ANSWER": "దశలవారీగా సమాధానం:", + "ANSWER": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e \u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02:", + "DIRECT": "Answer:", + # "REGEX": "సమాధానం (\\-?[0-9\\.\\,]+)", + "REGEX": "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 (\\-?[0-9\\.\\,]+)", + }, + "th": { # Thai + # "QUESTION": "โจทย์:", + "QUESTION": "\u0e42\u0e08\u0e17\u0e22\u0e4c:", + # "ANSWER": "คำตอบทีละขั้นตอน:", + "ANSWER": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:", + "DIRECT": "Answer:", + # "REGEX": "คำตอบคือ (\\-?[0-9\\.\\,]+)", + "REGEX": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d (\\-?[0-9\\.\\,]+)", + }, + "ja": { # Japanese + # "QUESTION": "問題:", + "QUESTION": "\u554f\u984c：", + # "ANSWER": "ステップごとの答え:", + "ANSWER": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048:", + "DIRECT": "Answer:", + # "REGEX": "答えは(\\-?[0-9\\.\\,]+)です。", + "REGEX": "\u7b54\u3048\u306f(\\-?[0-9\\.\\,]+)\u3067\u3059\u3002", + }, + "zh": { # Chinese + # "QUESTION": "问题:", + "QUESTION": "\u95ee\u9898：", + # "ANSWER": "逐步解答:", + "ANSWER": "\u9010\u6b65\u89e3\u7b54:", + "DIRECT": "Answer:", + # "REGEX": "答案是 (\\-?[0-9\\.\\,]+)。", + "REGEX": "\u7b54\u6848\u662f (\\-?[0-9\\.\\,]+)\u3002", + }, +} + + +def add_regex_pattern(regex_pattern): + if regex_pattern is None: + return {} + return { + "filter_list": [ + { + "name": "strict-match", + "filter": [ + { + "function": "regex", + "regex_pattern": f"""{regex_pattern}""", + }, + { + "function": "take_first", + }, + ], + }, + { + "name": "flexible-extract", + "filter": [ + { + "function": "regex", + "regex_pattern": """(-?[$0-9.,]{2,})|(-?[0-9]+)""", + "group_select": -1, + }, + { + "function": "take_first", + }, + ], + }, + ], + } + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + for lang in LANGUAGES.keys(): + try: + QUESTION = LANGUAGES[lang]["QUESTION"] + + yaml_template = "cot_yaml" + filter_list = {} + DELIMITER = None + if mode == "direct": + ANSWER = LANGUAGES[lang]["DIRECT"] + REGEX = None + task_name = f"mgsm_direct_{lang}" + yaml_template = "direct_yaml" + elif mode == "native-cot": + ANSWER = LANGUAGES[lang]["ANSWER"] + REGEX = LANGUAGES[lang]["REGEX"] + task_name = f"mgsm_native_cot_{lang}" + filter_list = add_regex_pattern(REGEX) + DELIMITER = "" if lang in ["zh", "ja"] else None + elif mode == "en-cot": + ANSWER = LANGUAGES["en"]["ANSWER"] + REGEX = LANGUAGES["en"]["REGEX"] + task_name = f"mgsm_en_cot_{lang}" + + file_name = f"{task_name}.yaml" + ANSWER_TO_SKIP = len(LANGUAGES[lang]["ANSWER"]) + 1 + with open( + f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8" + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + { + "include": yaml_template, + "dataset_name": lang, + "task": f"{task_name}", + "doc_to_text": f"""{{% if answer is not none %}}""" + f"""{{{{question+"\\n{ANSWER}"}}}}""" + f"""{{% else %}}""" + f"""{{{{"{QUESTION} "+question+"\\n{ANSWER}"}}}}""" + f"""{{% endif %}}""", + "doc_to_target": f"""{{% if answer is not none %}}""" + f"""{{{{answer[{ANSWER_TO_SKIP}:]}}}}""" + f"""{{% else %}}""" + f"""{{{{answer_number|string}}}}""" + f"""{{% endif %}}""", + **filter_list, + "generation_kwargs": { + "until": [QUESTION, "", "<|im_end|>"], + "do_sample": False, + }, + **({"target_delimiter": DELIMITER} if DELIMITER else {}), + }, + f, + allow_unicode=True, + width=float("inf"), + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + parser.add_argument( + "--mode", + default="native-cot", + choices=["direct", "native-cot", "en-cot"], + help="Mode of chain-of-thought", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/README.md b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/README.md new file mode 100644 index 0000000000000000000000000000000000000000..381a8b11cbee6e3c2a8ad57e7e8001be01c8cefe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/README.md @@ -0,0 +1,32 @@ +# MIMIC-III Report Summarization + +### Paper + +Title: `MIMIC-III, a freely accessible critical care database` + +Abstract: [https://www.nature.com/articles/sdata201635](https://www.nature.com/articles/sdata201635) + +MIMIC-III containins de-identified health data from around 40,000 patients admitted to +intensive care units at a large tertiary care hospital. This task focuses on radiology +report summarization. + + +#### Tasks + +* `mimic_repsum`: Generate extractive notes summaries, evaluated with [Radgraph-F1](https://www.cell.com/patterns/fulltext/S2666-3899(23)00157-5), bleu, rouge, bert_score, bleurt. +* `mimic_repsum_perplexity`: Generate extractive notes summaries, evaluated with perplexity. + +### Citation + +```bibtex +@article{johnson2016mimic, + title={MIMIC-III, a freely accessible critical care database}, + author={Johnson, Alistair EW and Pollard, Tom J and Shen, Lu and Lehman, Li-wei H and Feng, Mengling and Ghassemi, Mohammad and Moody, Benjamin and Szolovits, Peter and Anthony Celi, Leo and Mark, Roger G}, + journal={Scientific data}, + volume={3}, + number={1}, + pages={1--9}, + year={2016}, + publisher={Nature Publishing Group} +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/mimic_repsum.yaml b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/mimic_repsum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b2108a3b4aa6b7bbd36cfc9231737482656c6b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/mimic_repsum.yaml @@ -0,0 +1,40 @@ +task: mimic_repsum +dataset_path: dmacres/mimiciii-hospitalcourse-meta +description: > + Instructions: The following text is from a collection of medical records. Summarize the findings into diagnostic statements. Do not omit relevant information and avoid using abbreviations or jargon unless they appear in the original text. + +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n\n" + top_p: 0.95 +metric_list: + - metric: bleu + aggregation: nanmean + higher_is_better: true + - metric: rouge1 + aggregation: nanmean + higher_is_better: true + - metric: rouge2 + aggregation: nanmean + higher_is_better: true + - metric: rougeL + aggregation: nanmean + higher_is_better: true + - metric: bleurt + aggregation: nanmean + higher_is_better: true + - metric: bert_score + aggregation: nanmean + higher_is_better: true + - metric: F1-Radgraph + aggregation: nanmean + higher_is_better: true +metadata: + version: 1.4 diff --git a/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/mimic_repsum_perplexity.yaml b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/mimic_repsum_perplexity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a53e343eebc5da34f52684a3b8b9055918d1f14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/mimic_repsum_perplexity.yaml @@ -0,0 +1,12 @@ +include: mimic_repsum.yaml +task: mimic_repsum_perplexity +output_type: loglikelihood_rolling +doc_to_text: "" +process_results: !function utils_perplexity.process_results +metric_list: + - metric: word_perplexity + higher_is_better: false + - metric: byte_perplexity + higher_is_better: false + - metric: bits_per_byte + higher_is_better: false diff --git a/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/utils.py b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1c1bfe8f26b9f7b8d0f00f9d499fdbb3e0bce264 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/utils.py @@ -0,0 +1,154 @@ +import re +from collections.abc import Iterable + +import numpy as np + + +try: + import evaluate + from radgraph import F1RadGraph + + bleu = evaluate.load("bleu") + rouge = evaluate.load("rouge") + bertscore = evaluate.load("bertscore") + bleurt = evaluate.load("bleurt", "bleurt-base-512", module_type="metric") + +except (ModuleNotFoundError, ImportError): + raise ModuleNotFoundError( + "Please install evaluation metrics via pip install evaluate and pip install bert-score", + ) +except Exception as e: + raise RuntimeError( + f"Error loading evaluation metrics: {str(e)}. Please check your installation." + ) + + +def doc_eval(pred, refs): + try: + bleu_results = bleu.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Bleu error: {e}") + bleu_results = {"bleu": np.NAN} + + try: + rouge_results = rouge.compute(predictions=pred, references=refs) + except Exception as e: + print(f"Rouge error: {e}") + rouge_results = {"rouge1": np.NAN, "rouge2": np.NAN, "rougeL": np.NAN} + + try: + bleurt_scores = bleurt.compute(predictions=pred, references=refs)["scores"] + except Exception as e: + print(f"Bleurt error: {e}") + bleurt_scores = [np.NAN] + + try: + bert_scores = bertscore.compute(predictions=pred, references=refs, lang="en")[ + "f1" + ] + except Exception as e: + print(f"Bert error: {e}") + bert_scores = [np.NAN] + + if bleu_results["bleu"] == 0: + # Sometimes bleu is 0.0 and this breaks the stderr computation. + bleu_results["bleu"] += 1e-5 + + results = { + "bleu": bleu_results["bleu"], + "rouge1": rouge_results["rouge1"], + "rouge2": rouge_results["rouge2"], + "rougeL": rouge_results["rougeL"], + "bleurt": np.mean(bleurt_scores), + "bert_score": np.mean(bert_scores), + } + + return results + + +f1radgraph = F1RadGraph(reward_level="partial") + + +def doc_to_text(doc) -> str: + text = doc["extractive_notes_summ"] + + a = re.search("IMPRESSION", text, re.IGNORECASE) + if a is not None: + a = a.start() + else: + a = -1 + b = re.search("FINDING", text, re.IGNORECASE) + if b is not None: + b = b.start() + else: + b = -1 + + if a < b: + impressions = text[a:b].split(" ")[0] + findings = text[b:].split(" ")[0] + else: + impressions = text[a:].split(" ")[0] + findings = text[b:a].split(" ")[0] + + if len(findings) < 5 < len(impressions): + findings = text[:a] + + return "Given the findings: {}.\nSummarize the findings.".format(findings) + + +def doc_to_target(doc) -> str: + text = doc["extractive_notes_summ"] + + a = re.search("IMPRESSION", text, re.IGNORECASE) + if a is not None: + a = a.start() + else: + a = -1 + b = re.search("FINDING", text, re.IGNORECASE) + if b is not None: + b = b.start() + else: + b = -1 + + if a < b: + impressions = text[a:b].split(" ")[0] + else: + impressions = text[a:].split(" ")[0] + + return impressions + + +def is_non_str_iterable(obj): + return isinstance(obj, Iterable) and not isinstance(obj, str) + + +def process_results(doc, results): + pred, refs = [results[0]], [doc_to_target(doc)] + + if len(refs[0]) < 5 or len(pred[0]) < 5: + return { + "bleu": np.NAN, + "rouge1": np.NAN, + "rouge2": np.NAN, + "rougeL": np.NAN, + "bleurt": np.NAN, + "bert_score": np.NAN, + "F1-Radgraph": np.NAN, + } + + results = doc_eval(pred, refs) + + try: + radgraph_score, _, _, _ = f1radgraph(hyps=pred, refs=refs) + except Exception: + radgraph_score = np.NAN + + return { + "bleu": results["bleu"], + "rouge1": results["rouge1"], + "rouge2": results["rouge2"], + "rougeL": results["rougeL"], + "bleurt": results["bleurt"], + "bert_score": results["bert_score"], + "F1-Radgraph": radgraph_score, + } diff --git a/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/utils_perplexity.py b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/utils_perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..f4d1443b1c1d2d6d20d0b7330eccbdc3ee334bd5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mimic_repsum/utils_perplexity.py @@ -0,0 +1,14 @@ +import re + +from lm_eval.tasks.mimic_repsum.utils import doc_to_target + + +def process_results(doc, results): + (loglikelihood,) = results + _words = len(re.split(r"\s+", doc_to_target(doc))) + _bytes = len(doc_to_target(doc).encode("utf-8")) + return { + "word_perplexity": (loglikelihood, _words), + "byte_perplexity": (loglikelihood, _bytes), + "bits_per_byte": (loglikelihood, _bytes), + } diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/README.md b/lm-evaluation-harness/lm_eval/tasks/minerva_math/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4cd78f76eb927db8f059fbba1a2e2bbe5a7ce03f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/README.md @@ -0,0 +1,68 @@ +# MATH +ℹ️ This is the 4-shot variant! +## Paper +Measuring Mathematical Problem Solving With the MATH Dataset +https://arxiv.org/abs/2103.03874 + +Many intellectual endeavors require mathematical problem solving, but this skill remains beyond the capabilities of computers. To measure this ability in machine learning models, we introduce MATH, a new dataset of 12,500 challenging competition mathematics problems. Each problem in MATH has a full step-by-step solution which can be used to teach models to generate answer derivations and explanations. + +NOTE: The few-shot and the generated answer extraction is based on the [Minerva](https://arxiv.org/abs/2206.14858) and exact match equivalence is calculated using the `sympy` library. This requires additional dependencies, which can be installed via the `lm-eval[math]` extra. + +Homepage: https://github.com/hendrycks/math + + +## Citation +``` +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt}, + journal={NeurIPS}, + year={2021} +} + +@misc{2206.14858, +Author = {Aitor Lewkowycz and Anders Andreassen and David Dohan and Ethan Dyer and Henryk Michalewski and Vinay Ramasesh and Ambrose Slone and Cem Anil and Imanol Schlag and Theo Gutman-Solo and Yuhuai Wu and Behnam Neyshabur and Guy Gur-Ari and Vedant Misra}, +Title = {Solving Quantitative Reasoning Problems with Language Models}, +Year = {2022}, +Eprint = {arXiv:2206.14858}, +} +``` + +### Groups and Tasks + +#### Groups + +- `minerva_math` + +#### Tasks + +- `minerva_math_algebra` +- `minerva_math_counting_and_prob` +- `minerva_math_geometry` +- `minerva_math_intermediate_algebra` +- `minerva_math_num_theory` +- `minerva_math_prealgebra` +- `minerva_math_precalc` + +### Checklist + +The checklist is the following: + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical. + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Variant Wishlist + +- [ ] zero-shot variant + +### Changelog +version 2.0: (21-Feb-2025); added math_verify (extraction) metric. For details [see](https://huggingface.co/blog/math_verify_leaderboard) diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f15c9416c015673a037537d707923490679a176c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml @@ -0,0 +1,32 @@ +tag: + - math_word_problems +task: minerva_math_algebra +dataset_path: EleutherAI/hendrycks_math +process_docs: !function utils.process_docs +dataset_name: algebra +output_type: generate_until +training_split: train +test_split: test +doc_to_text: !function utils.doc_to_text +process_results: !function utils.process_results +doc_to_target: "{{answer if few_shot is undefined else solution}}" +generation_kwargs: + until: + - "Problem:" + do_sample: false + temperature: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: math_verify + aggregation: mean + higher_is_better: true +num_fewshot: 4 +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true +fewshot_config: + sampler: first_n + samples: !function utils.list_fewshot_samples diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml new file mode 100644 index 0000000000000000000000000000000000000000..688cd711c50d005d5d78ca55116ad333d96161ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: counting_and_probability +task: minerva_math_counting_and_prob diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..079ee70e9ed8997f351d1732c0c88dad1e4896de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: geometry +task: minerva_math_geometry diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b3f063c36e10063dd06be93c290820a787ddd1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: intermediate_algebra +task: minerva_math_intermediate_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44f587bce4cce5e4ab80d24b938b88488553d6da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: number_theory +task: minerva_math_num_theory diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..865e2f2c6e5397a07fb473a89f4d8eaf47d3eb52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: prealgebra +task: minerva_math_prealgebra diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06e63abc7c206b43759217b38cd5db2395e554a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: precalculus +task: minerva_math_precalc diff --git a/lm-evaluation-harness/lm_eval/tasks/minerva_math/utils.py b/lm-evaluation-harness/lm_eval/tasks/minerva_math/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..984ba33f229d624c9fc6036fa8f05e4da9d5cca4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/minerva_math/utils.py @@ -0,0 +1,306 @@ +import logging +import re +import signal +from importlib.metadata import version +from typing import Dict, List, Optional + +import datasets + + +eval_logger = logging.getLogger(__name__) + + +try: + import antlr4 + import sympy + from math_verify import parse, verify + from sympy.parsing.latex import parse_latex + + assert version("antlr4-python3-runtime").startswith("4.11") +except (ModuleNotFoundError, AssertionError) as e: + raise type(e)( + "`sympy`, `math_verify` and `antlr4-python3-runtime==4.11` are required for generating translation task prompt templates. " + "Please install the required packages via pip install lm-eval[math] or pip install -e .[math]" + ) from e + + +# taken from +# https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py +def doc_to_text(doc: dict) -> str: + return "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:" + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc: dict) -> dict: + out_doc = { + "problem": doc["problem"], + "solution": doc["solution"], + "answer": normalize_final_answer( + remove_boxed(last_boxed_only_string(doc["solution"])) + ), + } + if getattr(doc, "few_shot", None) is not None: + out_doc["few_shot"] = True + return out_doc + + return dataset.map(_process_doc) + + +def list_fewshot_samples() -> list[dict]: + return [ + { + "problem": "Find the domain of the expression $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}", + "solution": "The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$", + "solution": "We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?", + "solution": "If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.", + "solution": "If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.", + "few_shot": "1", + }, + ] + + +def process_results(doc: dict, results: List[str]) -> Dict[str, int]: + candidates = results[0] + + unnormalized_answer = get_unnormalized_answer(candidates) + answer = normalize_final_answer(unnormalized_answer) + + if is_equiv(answer, doc["answer"]): + retval = 1 + else: + retval = 0 + + # math_verify + res = verify(parse(doc["answer"]), parse(candidates)) + mathval = 1 if res else 0 + + results = { + "exact_match": retval, + "math_verify": mathval, + } + return results + + +def last_boxed_only_string(string: str) -> Optional[str]: + idx = string.rfind("\\boxed") + if "\\boxed " in string: + return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + retval = None + else: + retval = string[idx : right_brace_idx + 1] + + return retval + + +def remove_boxed(s: str) -> str: + if "\\boxed " in s: + left = "\\boxed " + assert s[: len(left)] == left + return s[len(left) :] + + left = "\\boxed{" + + assert s[: len(left)] == left + assert s[-1] == "}" + + return s[len(left) : -1] + + +class timeout: + def __init__(self, seconds=1, error_message="Timeout"): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + +def is_equiv(x1: str, x2: str) -> bool: + """ + x1 and x2 are normalized latex string + """ + try: + with timeout(seconds=5): + try: + parsed_x1 = parse_latex(x1) + parsed_x2 = parse_latex(x2) + except ( + sympy.parsing.latex.errors.LaTeXParsingError, + sympy.SympifyError, + TypeError, + ): + eval_logger.debug(f"couldn't parse one of {x1} or {x2}") + return False + + try: + diff = parsed_x1 - parsed_x2 + except TypeError: + eval_logger.debug(f"couldn't subtract {x1} and {x2}") + return False + + try: + if sympy.simplify(diff) == 0: + return True + else: + return False + except ValueError: + eval_logger.debug( + f"Had some trouble simplifying when comparing {x1} and {x2}" + ) + except TimeoutError: + eval_logger.debug(f"Timed out comparing {x1} and {x2}") + return False + except ImportError as e: + eval_logger.error(e) + raise + except Exception as e: + eval_logger.debug(f"Failed comparing {x1} and {x2} with {e}") + return False + + +def get_unnormalized_answer(text: str) -> str: + INVALID_ANSWER = "[invalidanswer]" + end_seq = "I hope it is correct." + text += end_seq + match = re.search( + r"Final Answer: The final answer is(.*?). I hope it is correct.", + text, + ) + if match: + return match.group(1).strip() + else: + return INVALID_ANSWER + + +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "ft", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """ + Normalize a final answer to a quantitative reasoning question. + + Copied character for character from appendix D of Lewkowycz et al. (2022) + """ + final_answer = final_answer.split("=")[-1] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalize 100,000 -> 100000 + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/README.md b/lm-evaluation-harness/lm_eval/tasks/mlqa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3d82f95ff05e8ce7dbd71ba2e36f997dad92def0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/README.md @@ -0,0 +1,101 @@ +# MLQA + +### Paper + +Title: `MLQA: Evaluating Cross-lingual Extractive Question Answering` + +Abstract: `https://arxiv.org/abs/1910.07475` + +MLQA (MultiLingual Question Answering) is a benchmark dataset for evaluating cross-lingual question answering performance. +MLQA consists of over 5K extractive QA instances (12K in English) in SQuAD format in seven languages - English, Arabic, +German, Spanish, Hindi, Vietnamese and Simplified Chinese. MLQA is highly parallel, with QA instances parallel between +4 different languages on average + +Homepage: `https://github.com/facebookresearch/MLQA` + + +### Citation + +``` +@misc{lewis2020mlqaevaluatingcrosslingualextractive, + title={MLQA: Evaluating Cross-lingual Extractive Question Answering}, + author={Patrick Lewis and Barlas Oğuz and Ruty Rinott and Sebastian Riedel and Holger Schwenk}, + year={2020}, + eprint={1910.07475}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/1910.07475}, +} +``` + +### Groups, Tags, and Tasks + +#### Groups + +* Not part of a group yet + +#### Tasks + +Tasks of the form `mlqa_context-lang_question-lang.yaml` +* `mlqa_ar_ar.yaml` +* `mlqa_ar_de.yaml` +* `mlqa_ar_vi.yaml` +* `mlqa_ar_zh.yaml` +* `mlqa_ar_en.yaml` +* `mlqa_ar_es.yaml` +* `mlqa_ar_hi.yaml` +* `mlqa_de_ar.yaml` +* `mlqa_de_de.yaml` +* `mlqa_de_vi.yaml` +* `mlqa_de_zh.yaml` +* `mlqa_de_en.yaml` +* `mlqa_de_es.yaml` +* `mlqa_de_hi.yaml` +* `mlqa_vi_ar.yaml` +* `mlqa_vi_de.yaml` +* `mlqa_vi_vi.yaml` +* `mlqa_vi_zh.yaml` +* `mlqa_vi_en.yaml` +* `mlqa_vi_es.yaml` +* `mlqa_vi_hi.yaml` +* `mlqa_zh_ar.yaml` +* `mlqa_zh_de.yaml` +* `mlqa_zh_vi.yaml` +* `mlqa_zh_zh.yaml` +* `mlqa_zh_en.yaml` +* `mlqa_zh_es.yaml` +* `mlqa_zh_hi.yaml` +* `mlqa_en_ar.yaml` +* `mlqa_en_de.yaml` +* `mlqa_en_vi.yaml` +* `mlqa_en_zh.yaml` +* `mlqa_en_en.yaml` +* `mlqa_en_es.yaml` +* `mlqa_en_hi.yaml` +* `mlqa_es_ar.yaml` +* `mlqa_es_de.yaml` +* `mlqa_es_vi.yaml` +* `mlqa_es_zh.yaml` +* `mlqa_es_en.yaml` +* `mlqa_es_es.yaml` +* `mlqa_es_hi.yaml` +* `mlqa_hi_ar.yaml` +* `mlqa_hi_de.yaml` +* `mlqa_hi_vi.yaml` +* `mlqa_hi_zh.yaml` +* `mlqa_hi_en.yaml` +* `mlqa_hi_es.yaml` +* `mlqa_hi_hi.yaml` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/generate_tasks.py b/lm-evaluation-harness/lm_eval/tasks/mlqa/generate_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..19bd3533af6c97132ec8fea3ea94997530378e66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/generate_tasks.py @@ -0,0 +1,48 @@ +# ruff: noqa: E731, E741 +""" +Script to generate task YAMLs for the mlqa dataset. +Based on `tasks/bigbench/generate_tasks.py`. +""" + +from datasets import get_dataset_config_names + + +chosen_subtasks = [] + +language_dict = { + "en": "english", + "es": "spanish", + "hi": "hindi", + "vi": "vietnamese", + "de": "german", + "ar": "arabic", + "zh": "chinese", +} + + +def main() -> None: + configs = get_dataset_config_names("facebook/mlqa", trust_remote_code=True) + for config in configs: + if len(config.split(".")) == 2: + continue + else: + chosen_subtasks.append(config) + assert len(chosen_subtasks) == 49 + for task in chosen_subtasks: + file_name = f"{task.replace('.', '_')}.yaml" + context_lang = file_name.split("_")[1] + # Not using yaml to avoid tagging issues with !function + with open(file_name, "w", encoding="utf-8") as f: + f.write("# Generated by generate_tasks.py\n") + + # Manually writing the YAML-like content inside files to avoid tagging issues + f.write("include: mlqa_common_yaml\n") + f.write(f"task: {task.replace('.', '_')}\n") + f.write(f"dataset_name: {task}\n") + f.write( + f"process_results: !function utils.process_results_{context_lang}\n" + ) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8db625acce6d92b58dc601725da7bedb3e5e76ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_ar +dataset_name: mlqa.ar.ar +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d1468a7bd82ae9d682766042e99069ed6ed92a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_de +dataset_name: mlqa.ar.de +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18e763e8ac464ad922fc228d2449be4ad20568d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_en.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_en +dataset_name: mlqa.ar.en +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c93ef03ec0a7fff3090b5e9f269b97c7de8a35cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_es.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_es +dataset_name: mlqa.ar.es +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5abb023ccdaf5912451f1093d4a1c9295902d6e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_hi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_hi +dataset_name: mlqa.ar.hi +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54869c657d682971518fc12bb83f24f8389e46c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_vi +dataset_name: mlqa.ar.vi +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5236d6cb873fa95582ac4bcc3fd95f940323a188 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_ar_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_ar_zh +dataset_name: mlqa.ar.zh +process_results: !function utils.process_results_ar diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_common_yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c52ecb8914a7ddb24a838ff0570599ff43f98836 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_common_yaml @@ -0,0 +1,22 @@ +dataset_path: facebook/mlqa +dataset_kwargs: + trust_remote_code: true +test_split: test +validation_split: validation +output_type: generate_until +doc_to_text: "Context: {{context}}\n\nQuestion: {{question}}\n\nAnswer:" +doc_to_target: "{{answers}}" +process_docs: !function utils.process_docs +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: mean + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1090a58925ed033bc18f5546e2b8d93619992b3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_ar +dataset_name: mlqa.de.ar +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be465ab57a4073d29e092178403e452df122eb2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_de +dataset_name: mlqa.de.de +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55f2652ce48613d17534020ec5e0e452812ec5dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_en.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_en +dataset_name: mlqa.de.en +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4f085e6241482f35c647a3b1398f2913b4a5a53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_es.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_es +dataset_name: mlqa.de.es +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff3bbc428634ad0afaac7dc44f1911acf1258fe7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_hi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_hi +dataset_name: mlqa.de.hi +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe61983b70ce347d774cff2d20dee1c8afb1a019 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_vi +dataset_name: mlqa.de.vi +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee1855626fdf5ab157894dfd5636fe7b2fc58739 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_de_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_de_zh +dataset_name: mlqa.de.zh +process_results: !function utils.process_results_de diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8c72d2694351760a9c9f9b6332615238af2e125 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_en_ar +dataset_name: mlqa.en.ar +process_results: !function utils.process_results_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b27e02ae6c33115ccd6f69c0487332468614d9c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_en_de +dataset_name: mlqa.en.de +process_results: !function utils.process_results_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d15e222f7bf8ec0f09526e12ecac6788c1174568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_en.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_en_en +dataset_name: mlqa.en.en +process_results: !function utils.process_results_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eddb728f02529dcaaf05ef1c57920ff36e254150 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_es.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_en_es +dataset_name: mlqa.en.es +process_results: !function utils.process_results_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a2f635ea385f0dbfff79e470f066b8f260aa52f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_en_vi +dataset_name: mlqa.en.vi +process_results: !function utils.process_results_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91336eba9a7fd8e33fe7845c27b2b21f88d9177e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_en_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_en_zh +dataset_name: mlqa.en.zh +process_results: !function utils.process_results_en diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a24508cbd630e5f1fd25ed070821051214b0a72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_ar +dataset_name: mlqa.es.ar +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a40b2b6956b91b39aa8a7d16d98ef91098f3665 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_de +dataset_name: mlqa.es.de +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..660968c7fd9131cb54ba23be4eaf51e0ed68ff35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_en.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_en +dataset_name: mlqa.es.en +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1232947b92715bf3b714ee3a9aa1f525a532bc68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_es.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_es +dataset_name: mlqa.es.es +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5502288925c25e2fda50ed630abd292989464e93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_hi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_hi +dataset_name: mlqa.es.hi +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ea9027dec2a1cee00d5e145f8ce68c02ccb9f4d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_vi +dataset_name: mlqa.es.vi +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caecd1b2d0d8c1600596cfb9ed844e044f0eefdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_es_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_es_zh +dataset_name: mlqa.es.zh +process_results: !function utils.process_results_es diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4c4263a1d4c0d326a6fabfb5d4036037c152d75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_ar +dataset_name: mlqa.hi.ar +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8069b5a07b2c63f713bcea262f9d5209545507b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_de +dataset_name: mlqa.hi.de +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7a18067bc0a568003f54317d2b9a44cc1770b2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_en.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_en +dataset_name: mlqa.hi.en +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d152ad66dc3e497c5bc5aa1094a1c0f97cdf2bae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_es.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_es +dataset_name: mlqa.hi.es +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ce79e6bbe5be2d476fdbbdf914107be31e2efea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_hi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_hi +dataset_name: mlqa.hi.hi +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..534d90f70dd08313f6f9a0cb68f5571f689e8569 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_vi +dataset_name: mlqa.hi.vi +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8432db492dfc3c593adbf7255ead41c0a36be8b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_hi_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_hi_zh +dataset_name: mlqa.hi.zh +process_results: !function utils.process_results_hi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c22c11cd0613c88743f296fce025ae774be31a5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_ar +dataset_name: mlqa.vi.ar +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..948ac3ac36637dfd300e0b0c74ca8a9c1a7bb1e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_de +dataset_name: mlqa.vi.de +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0106867703a5bcb7a447d942f8f8cf71269b4820 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_en.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_en +dataset_name: mlqa.vi.en +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ac62c1056237afcb2d9e921d6b319903ae2af25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_es.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_es +dataset_name: mlqa.vi.es +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26b232a879778b6d7eee8bedfdbe7e758acba9aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_hi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_hi +dataset_name: mlqa.vi.hi +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8277d78ebc4ddd57ab2fe955adc80259e85902c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_vi +dataset_name: mlqa.vi.vi +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ecc6b919281cd13b103a1564199b2ddf1db622b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_vi_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_vi_zh +dataset_name: mlqa.vi.zh +process_results: !function utils.process_results_vi diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42c3713d5a156ed1c44afd3af08624b8d4bc75aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_ar.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_zh_ar +dataset_name: mlqa.zh.ar +process_results: !function utils.process_results_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb5e4cb884a4bfcc25f1f5323514cfb49807654f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_de.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_zh_de +dataset_name: mlqa.zh.de +process_results: !function utils.process_results_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed58f47f4d52f8a3ba1c9240bca15ac97f113874 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_hi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_zh_hi +dataset_name: mlqa.zh.hi +process_results: !function utils.process_results_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7043676235f913e7826751503bb7146ae5a4b5fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_vi.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_zh_vi +dataset_name: mlqa.zh.vi +process_results: !function utils.process_results_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..792b5ee0c9ba13583de4914ceba5c76a941361a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/mlqa_zh_zh.yaml @@ -0,0 +1,5 @@ +# Generated by generate_tasks.py +include: mlqa_common_yaml +task: mlqa_zh_zh +dataset_name: mlqa.zh.zh +process_results: !function utils.process_results_zh diff --git a/lm-evaluation-harness/lm_eval/tasks/mlqa/utils.py b/lm-evaluation-harness/lm_eval/tasks/mlqa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..61e593716a968af4240e43024dfa90c8f0e0a53c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mlqa/utils.py @@ -0,0 +1,165 @@ +""" +Code based on Official evaluation script for the MLQA dataset. +Repo: https://github.com/facebookresearch/MLQA/blob/main/mlqa_evaluation_v1.py +""" + +import re +import string +import sys +import unicodedata +from collections import Counter + +import datasets + + +PUNCT = { + chr(i) + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") +}.union(string.punctuation) +WHITESPACE_LANGS = ["en", "es", "hi", "vi", "de", "ar"] +MIXED_SEGMENTATION_LANGS = ["zh"] + + +def whitespace_tokenize(text): + return text.split() + + +def mixed_segmentation(text): + segs_out = [] + temp_str = "" + for char in text: + if re.search(r"[\u4e00-\u9fa5]", char) or char in PUNCT: + if temp_str != "": + ss = whitespace_tokenize(temp_str) + segs_out.extend(ss) + temp_str = "" + segs_out.append(char) + else: + temp_str += char + + if temp_str != "": + ss = whitespace_tokenize(temp_str) + segs_out.extend(ss) + + return segs_out + + +def normalize_answer(s, lang): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text, lang): + if lang == "en": + return re.sub(r"\b(a|an|the)\b", " ", text) + elif lang == "es": + return re.sub(r"\b(un|una|unos|unas|el|la|los|las)\b", " ", text) + elif lang == "hi": + return text # Hindi does not have formal articles + elif lang == "vi": + return re.sub(r"\b(của|là|cái|chiếc|những)\b", " ", text) + elif lang == "de": + return re.sub( + r"\b(ein|eine|einen|einem|eines|einer|der|die|das|den|dem|des)\b", + " ", + text, + ) + elif lang == "ar": + return re.sub(r"\sال^|ال", " ", text) + elif lang == "zh": + return text # Chinese does not have formal articles + else: + raise Exception("Unknown Language {}".format(lang)) + + def white_space_fix(text, lang): + if lang in WHITESPACE_LANGS: + tokens = whitespace_tokenize(text) + elif lang in MIXED_SEGMENTATION_LANGS: + tokens = mixed_segmentation(text) + else: + raise Exception("Unknown Language {}".format(lang)) + return " ".join([t for t in tokens if t.strip() != ""]) + + def remove_punc(text): + return "".join(ch for ch in text if ch not in PUNCT) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)), lang), lang) + + +def f1_score(prediction, ground_truth, lang): + prediction_tokens = normalize_answer(prediction, lang).split() + ground_truth_tokens = normalize_answer(ground_truth, lang).split() + common = Counter(prediction_tokens) & Counter(ground_truth_tokens) + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def exact_match_score(prediction, ground_truth, lang): + return normalize_answer(prediction, lang) == normalize_answer(ground_truth, lang) + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths, lang): + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth, lang) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + out_doc = { + "context": doc["context"], + "question": doc["question"], + "answers": doc["answers"]["text"], + } + return out_doc + + return dataset.map(_process_doc) + + +# Base function +def process_results_lang(doc, results, lang): + ground_truths = doc["answers"] + prediction = results[0].strip() + exact_match = metric_max_over_ground_truths( + exact_match_score, prediction, ground_truths, lang + ) + f1 = metric_max_over_ground_truths(f1_score, prediction, ground_truths, lang) + return {"exact_match": exact_match, "f1": f1} + + +# Language Wrapper functions +def process_results_en(doc, results): + return process_results_lang(doc, results, "en") + + +def process_results_es(doc, results): + return process_results_lang(doc, results, "es") + + +def process_results_hi(doc, results): + return process_results_lang(doc, results, "hi") + + +def process_results_vi(doc, results): + return process_results_lang(doc, results, "vi") + + +def process_results_de(doc, results): + return process_results_lang(doc, results, "de") + + +def process_results_ar(doc, results): + return process_results_lang(doc, results, "ar") + + +def process_results_zh(doc, results): + return process_results_lang(doc, results, "zh") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a2372ae36cc97e6f5eca5e1e1fdde0556f68e07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_other.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1768cab3628d65dd5dff855c87d694cc70b269b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_philosophy.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a4b25c013a9bfed0b1836a84378baa88a305960 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_physics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_plus_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a3425d517654a6b93e03ee1bb681e07de18c4016 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/README.md @@ -0,0 +1,73 @@ +# Task-name + +### Paper + +Title: `Measuring Massive Multitask Language Understanding` + +Abstract: `https://arxiv.org/abs/2009.03300` + +`The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.` + +Homepage: `https://github.com/hendrycks/test` + +Note: The `Flan` variants are derived from [here](https://github.com/jasonwei20/flan-2), and as described in Appendix D.1 of [Scaling Instruction-Finetuned Language Models](https://arxiv.org/abs/2210.11416). + +### Citation + +``` +@article{hendryckstest2021, + title={Measuring Massive Multitask Language Understanding}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} + +@article{hendrycks2021ethics, + title={Aligning AI With Shared Human Values}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmlu`: `Original multiple-choice MMLU benchmark` +* `mmlu_continuation`: `MMLU but with continuation prompts` +* `mmlu_generation`: `MMLU generation` + +MMLU is the original benchmark as implemented by Hendrycks et al. with the choices in context and the answer letters (e.g `A`, `B`, `C`, `D`) in the continuation. +`mmlu_continuation` is a cloze-style variant without the choices in context and the full answer choice in the continuation. +`mmlu_generation` is a generation variant, similar to the original but the LLM is asked to generate the correct answer letter. + + +#### Subgroups + +* `mmlu_stem' +* `mmlu_humanities' +* `mmlu_social_sciences' +* `mmlu_other' + +Subgroup variants are prefixed with the subgroup name, e.g. `mmlu_stem_continuation`. + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +# changelog +ver 1: PR #497 +switch to original implementation + +ver 2: PR #2116 +add missing newline in description. diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/mmlu/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..88a7a2c2e63a5066b7f60a0bee8e8839173969e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/_generate_configs.py @@ -0,0 +1,159 @@ +# noqa +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger(__name__) + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--base_yaml_path", required=True) + parser.add_argument("--save_prefix_path", default="mmlu") + parser.add_argument("--cot_prompt_path", default=None) + parser.add_argument("--task_prefix", default="") + parser.add_argument("--group_prefix", default="") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + with open(args.base_yaml_path, encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path, encoding="utf-8") as f: + cot_file = json.load(f) + + ALL_CATEGORIES = [] + for subject, category in tqdm(SUBJECTS.items()): + if category not in ALL_CATEGORIES: + ALL_CATEGORIES.append(category) + + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"mmlu_{args.task_prefix}_{category}" + if args.task_prefix != "" + else f"mmlu_{category}", + "task": f"mmlu_{args.task_prefix}_{subject}" + if args.task_prefix != "" + else f"mmlu_{subject}", + "task_alias": subject.replace("_", " "), + "dataset_name": subject, + "description": description, + } + + file_save_path = args.save_prefix_path + f"_{subject}.yaml" + eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + allow_unicode=True, + default_style='"', + ) + + if args.task_prefix != "": + mmlu_subcategories = [ + f"mmlu_{args.task_prefix}_{category}" for category in ALL_CATEGORIES + ] + else: + mmlu_subcategories = [f"mmlu_{category}" for category in ALL_CATEGORIES] + + if args.group_prefix != "": + file_save_path = args.group_prefix + ".yaml" + else: + file_save_path = args.save_prefix_path + ".yaml" + + eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": f"mmlu_{args.task_prefix}" + if args.task_prefix != "" + else "mmlu", + "task": mmlu_subcategories, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0cabf04b8ac1e1f9c809600214c589cfefbba79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_continuation +group_alias: mmlu (continuation) +task: + - group: stem + task: + - mmlu_continuation_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_continuation_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_continuation_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_continuation_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f4e29c0fb5147d883ee993d95822dde10b69d4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc3de9c4e6679ba4c9f66494c908d99781adf5bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76aabcbfcf13a12e66e1af1daae2811b9b388fc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e64d0920b9d1ac151712aac84a9e9c3f522c3c9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79805df6f73782f25be4a302c738b73ecd2f2a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_clinical_knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..936f6ffe49245d558c0ef8fdf04b600dc177c375 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..289364ee44351c3d1bcee1193563babe6abe2a63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7d3c5696067f09f9a68fdd9c3f7a1002d264128 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dbc0932f63c0782e106db5fc27e96da9d816dec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38abd2426f844916087795c4cc04355d8d6c2776 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee6b42584c834a5e92506650ee3aba58ed1cfd66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ebb487dfbf634d390d2b2f9aa0e31e5a2f68fc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c554caf07da77e4a9bb0bea9672dfcee4777b91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_conceptual_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..848ce4e1f0dbff32d304c28f3d60d453e591a30f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d71dd16481a2bb5289ef5b713218dae0292bb11a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe8aa09718cb8aef0dad48c21926f7dacc7b8ee9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb5dbd2e505e3fb4604dd75f2d5fe1a35fce3391 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..280a50d2ee229b5f047a02024298474225203e54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_global_facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e518a5239a6da013ad31bfca284a3b7096bce840 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c38d60a7706306b215e156d4c27f05585945f7b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe34f7af35456657c1acf40e05b3aaabc7893e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..666c2742d1b762c103bbd02ff121676a047fb3e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41f6caf3e7f3b762af7c0350ca9a73d39bede2b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7fa9d5e3caa8dd3ec8e25172afda5f997b6c0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96c414d3c411c6380cf83dca3b7aedc325598220 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45ab0a539a02ae322f66db689d8eddf13c8b856a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a00f16ceba2cfd3f313c8fe0d2df4a43e4bbe23d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..575604e0acf52132d9e489a070d28fd761e739eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are questions (with answers) about management.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..165de6c90ba1d4756c39e2f5605226dbeb86e314 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02c4ee7f8af1856f498b7a55c83e085782e36666 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..700c407c2377d8d4d83bbf88d8f7a003a2e2900d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f5ef99e0f8fe8c98bc9994757d9cc6617e3550e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e78621aaa547b419f4133b94ce8dcba00c407f5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..989bb29aa095e83c2744011775864ef27258ca28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c938190bdd755f411914905d5309daa6938f313 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fab1ec2c1416bc644c8723bdb18905dff9c00040 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml @@ -0,0 +1,9 @@ +group: mmlu_social_sciences +group_alias: social sciences +task: + - mmlu_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..660f07476dfdd115fc0b8d5f04c685b23857cc33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f1b1c2dcd802effdf589d4f85b412593dfb622 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_business_ethics" +"task_alias": "business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3f692423abfbf036fc0347fdfbb2642a6d16c39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_computer_science" +"task_alias": "college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08a9628af175edb897c7f6d88b96d4969fccad29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_mathematics" +"task_alias": "college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35197a2a1885f7daf30209d4309dd059243260a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_medicine" +"task_alias": "college_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b5017afac65e0acf080a9df84098a1f21681833 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_physics" +"task_alias": "college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f9b42820f7f7196c6d02922337eaedb7ede5388 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_computer_security" +"task_alias": "computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af61a7e1579ac8613b5535e15a57adc629e2d571 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_conceptual_physics" +"task_alias": "conceptual_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..609c20af2acbdd7ef36104dc97db97a40bfca6a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8fa2137ad05a14e32d5d7e8973d6bc9c18d1a555 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_electrical_engineering" +"task_alias": "electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d15f6d5ae88b6edf0bba2298ffaacbd4d103aedd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_elementary_mathematics" +"task_alias": "elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee2fc2f61073dc11f6f745eaf8927ab70aadad3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_formal_logic" +"task_alias": "formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b27ddefd25be9c6695900ce6d290a811b68356df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_global_facts" +"task_alias": "global_facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22bc47943f0f66614f79cd0de5e7614afa1f08d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_biology" +"task_alias": "high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a25617cbd821411e6f0ca9fac853c76b7adb319 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_chemistry" +"task_alias": "high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad4c7d312c7e8f6517d308e6ffeb635a354b843e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_computer_science" +"task_alias": "high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c51bbdd7aa87b39da8145f8ea45f6fe13d17623 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_european_history" +"task_alias": "high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad87f1ad57a48102d7807a7a3fd75af86755912 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_geography" +"task_alias": "high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b93b363d658357619eaf907f8d04af339c22a12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_government_and_politics" +"task_alias": "high_school_government_and_politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a08c579d1480ab592917d5a6673e63cf09198417 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_macroeconomics" +"task_alias": "high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f1b6d70e022414b7d370635daa49e3a9a8649c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_mathematics" +"task_alias": "high_school_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac4f65dad5783bf23c50d9a39e912fe797a047e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_microeconomics" +"task_alias": "high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8c449aa1b5bb48d6899c328c82c44ee3ae3ef24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_physics" +"task_alias": "high_school_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ba836c71b2be9759bd9fe48dd0cb687ef08636 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_psychology" +"task_alias": "high_school_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef5bdd7cf1577a7ba9f3365643c5e56b21c8a77e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_statistics" +"task_alias": "high_school_statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ececdb0ab921bdc24b8aac41979a93d35670d0c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_us_history" +"task_alias": "high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af34c8ddbe51abc0f44baff2bf8087b4c749825f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_world_history" +"task_alias": "high_school_world_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ca720be7c7d757c579e4563cb805dc36a6dcc6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_human_aging" +"task_alias": "human_aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2acddd1e4ec1d85a7475202d43f5917abb085684 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_human_sexuality" +"task_alias": "human_sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fb2a162aab92931f8b560ce0e76155fbc9bb675 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_international_law" +"task_alias": "international_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c62a911ff5d849651d8c9e09feb34847846d147 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adf8821e9a8ac9d80f1cfb5c6af5b74a63efda27 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_logical_fallacies" +"task_alias": "logical_fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d846f96084a8cba059348a90d800a86b92ba09c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_machine_learning" +"task_alias": "machine_learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dff834ef804039858b6955155a8338dd11b30b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ef004988965c41ff075f2f976b98dca4657ca04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..989fb2c1aea91035421e49c7a11293c48ffec0bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_medical_genetics" +"task_alias": "medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7bb68bc2eb0f55b784943bd18296aabe3b86a31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..348d21403f06669e198146286b83e227fbde5a16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_moral_disputes" +"task_alias": "moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3762ee1200848439f08a3c69703af4cffb3a9d74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_moral_scenarios" +"task_alias": "moral_scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55f8ca01ff42a296c07d8fd2e2ccda373d91775b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5331c812ef70cb0123d754835fabde16ce330245 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b4ff970a10b7be9ab08527124ea236227b60428 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27b2ec9b9b70e00616d2560c3a8b1259781e8cfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_accounting" +"task_alias": "professional_accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07c36f1c38d46a513359d80284ead794dd72b7bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_law" +"task_alias": "professional_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c5754bf379cfd884ad837243105a49e3e28d386 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_medicine" +"task_alias": "professional_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0c0608ef6860edb4b8492402c674a7efda2070f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology" +"task_alias": "professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b675bdfd088bb7e651eece031198b5c0fb8ab3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_public_relations" +"task_alias": "public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b02342d95ede5148ee8b0aeb9e4ad4fb7dd05938 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_security_studies" +"task_alias": "security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49fa11620fb7147752328a484d56f8ead64c4387 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc4335e9eace7816ba112e4f55912223444d4c1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy" +"task_alias": "us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f8bc114c3ce7437ad0fb413a69a859f69bcbf99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b90972c7031c30d89beea835f70aab7cf45cce81 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_world_religions" +"task_alias": "world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json new file mode 100644 index 0000000000000000000000000000000000000000..c374b19d03391e61021af6640558a6de8853d7b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json @@ -0,0 +1 @@ +{"abstract_algebra": "The following are multiple choice questions (with answers) about abstract algebra.\n\nQ: Statement 1 | Every element of a group generates a cyclic subgroup of the group. Statement 2 | The symmetric group S_10 has 10 elements.\n(A) True, True (B) False, False (C) True, False (D) False, True\nA: Let's think step by step. A cyclic group is a group that is generated by a single element. Hence a subgroup generated by a single element of a group is cyclic and Statement 1 is True. The answer is (C).\n\nQ: The symmetric group $S_n$ has $\nactorial{n}$ elements, hence it is not true that $S_{10}$ has 10 elements.\nFind the characteristic of the ring 2Z.\n(A) 0 (B) 3 (C) 12 (D) 30\nA: Let's think step by step. A characteristic of a ring is R is $n$ if the statement $ka = 0$ for all $a\\in 2Z$ implies that $k$ is a multiple of $n$. Assume that $ka = 0$ for all $a\\in 2Z$ for some $k$. In particular $2k = 0$. Hence $k=0$ and $n=0$. The answer is (A).\n\nQ: Statement 1| Every function from a finite set onto itself must be one to one. Statement 2 | Every subgroup of an abelian group is abelian.\n(A) True, True (B) False, False (C) True, False (D) False, True\nA: Let's think step by step. Statement 1 is true. Let $S$ be a finite set. If $f:S \nightarrow S$ is a onto function, then $|S| = |f(S)|$. If $f$ was not one to one, then for finite domain $S$ the image would have less than $S$ elements, a contradiction.\nStatement 2 is true. Let $G$ be an abelian group and $H$ be a subgroup of $G$. We need to show that $H$ is abelian. Let $a,b \\in H$. Then $a,b \\in G$ and $ab=ba$. Since $G$ is abelian, $ab=ba$. Since $H$ is a subgroup of $G$, $ab \\in H$. Therefore, $ab=ba$ and $H$ is abelian. The answer is (A).\n\nQ: Statement 1 | If aH is an element of a factor group, then |aH| divides |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G.\n(A) True, True (B) False, False (C) True, False (D) False, True\nA: Let's think step by step. Statement 2 is false. Let $H$ be a subgroup of $S_3$ generated by the cycle $(1,2)$ and $K$ be a subgroup of $S_3$ generated by the cycle $(1,3)$. Both $H$ and $K$ have two elements, the generators and the identity. However $HK$ contains cycles (1,2), (1,3) and (2,3,1), but the inverse of (2,3,1) is (2,1,3) and it does not belong to HK, hence HK is not a subgroup. The answer is (B).\n\nQ: Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field.\n(A) 0 (B) 1 (C) 2 (D) 3\nA: Let's think step by step. Z_3[x]/(x^2 + c) is a field if and only if x^2 + c does not have roots in Z_3. That is x^2 + c != 0 for every x in Z_3. If c = 0, then x^2 + c = x^2 has root 0. If c = 1 then x^2 + c = x^2 + 1 = 0 + 1 for x = 0, 1 + 1 = 2 for x = 1 and 1 + 1 = 2 for x = 2, hence x^2 + 1 does not have any roots. For c = 2 the polynomial x^2 + 2 has two roots at x = 1 and x = 2. Hence Z_3[x]/(x^2 + c) is a field if and only if c = 1. The answer is (B).\n\n", "anatomy": "The following are multiple choice questions (with answers) about anatomy.\n\nQ: Which of the following is the body cavity that contains the pituitary gland?\n(A) Abdominal (B) Cranial (C) Pleural (D) Spinal\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. The pituitary gland is the major endocrine gland attached to the base of the brain, and it is contained in the Cranial cavity. The answer is (B).\n\nQ: Which of these branches of the trigeminal nerve contain somatic motor processes?\n(A) The supraorbital nerve (B) The infraorbital nerve (C) The mental nerve (D) None of the above\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. \nWe know the following: (A) The supraorbital nerve (also known as the frontal nerve) is the largest branch of the ophthalmic nerve and branch of ophthalmic division of the trigeminal nerve. (B) The infraorbital nerve is a branch of the maxillary division of the trigeminal nerve. (C) The mental nerve is a branch of the mandibular division of the trigeminal nerve. Because all these nerves are purely sensory nerves and do not contain any somatic motor processes. Therefore, the answer should be none of the above, which is (D). The answer is (D).\n\nQ: In Angle's Class II Div 2 occlusion there is\n(A) excess overbite of the upper lateral incisors. (B) negative overjet of the upper central incisors. (C) excess overjet of the upper lateral incisors. (D) excess overjet of the upper central incisors.\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. This is a question related to anatomy and orthodontics. Excess overjet is associated with Class II occlusions; therefore, we can safely eliminate (B) from the list, as negative overjet is often associated with Class III occlusions. Now, we need to determine the location of the excess overjet, and that would be the upper (maxillary) lateral incisors. Only (C) has the correct information. The answer is (C).\n\nQ: The pleura\n(A) have no sensory innervation. (B) are separated by a 2 mm space. (C) extend into the neck. (D) are composed of respiratory epithelium.\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. First, recall that the pleura refers to the thin layer of tissue that covers the lungs and lines the interior wall of the chest cavity. Now, let\u2019s look at each option:\nOption (A): \u201cThe pleura have no sensory innervation.\u201d This information is not correct. The pleura do have a sensory innervation.\nOption (B): \u201cThe pleura are separated by a 2 mm space.\u201d This information is not correct. There is a very thin \u201cpotential\u201d space between the layers of the pleura; however, it is typically filled with serous pleural fluid. \nOption (C): \u201cThe pleura extend into the neck.\u201d This information is actuakky true. The cervical pleura, also known as the dome of the pleuradome of the pleura, lines the extendsiton of the pleural cavity into the neck.\nOption (D): \u201cThe pleura are composed of respiratory epithelium.\u201d This information is not correct. The pleaura are composed of connective tissue (CT).\nBecause (A), (B), and (D) are all incorrect, (D) is the only correct answer. The answer is (C).\n\nQ: What is the embryological origin of the hyoid bone?\n(A) The first pharyngeal arch (B) The first and second pharyngeal arches (C) The second pharyngeal arch (D) The second and third pharyngeal arches\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. The hyoid bone, which is also known as the hyooid, is a a small U-shaped bone located in the anterior neck. In its resting position, it lies between the ase of the mandible and the third cervical vertebrae. We know that the second and the third pharyngeal arches give rise to the horns of the hyoid bone; therefore, the embryological origin of the hyoid bone are the second and the third pharyngeal arches\u2014this information is covered in the last option (D). Therefore, we conclude that (D) must be the correct answer. The answer is (D).\n\n", "astronomy": "The following are multiple choice questions (with answers) about astronomy.\n\nQ: Where do most short-period comets come from and how do we know?\n(A) The Kuiper belt; short period comets tend to be in the plane of the solar system just like the Kuiper belt. (B) The Kuiper belt; short period comets tend to come from random directions indicating a spherical distribution of comets called the Kuiper belt. (C) The asteroid belt; short period comets have orbital periods similar to asteroids like Vesta and are found in the plane of the solar system just like the asteroid belt. (D) The Oort cloud; short period comets tend to be in the plane of the solar system just like the Oort cloud.\nA: Let's think step by step. Most short-period comets come from the Kuiper belt, and we know because short period coments tend to be in the plane of the solar system, just like the Kuiper belt is. The answer is (A).\n\nQ: You are pushing a truck along a road. Would it be easier to accelerate this truck on Mars? Why? (Assume there is no friction)\n(A) It would be harder since the truck is heavier on Mars. (B) It would be easier since the truck is lighter on Mars. (C) It would be harder since the truck is lighter on Mars. (D) It would be the same no matter where you are.\nA: Let's think step by step. If we assume that there is no friction, the force needed to accelerate the truck is by Newton\u2019s second law only dependent on the mass of the truck. Hence (A), (B) and (C) are incorrect since it doesn\u2019t matter that it\u2019s on Mars, and (D) is the correct answer. The answer is (D).\n\nQ: Say the pupil of your eye has a diameter of 5 mm and you have a telescope with an aperture of 50 cm. How much more light can the telescope gather than your eye?\n(A) 10000 times more (B) 100 times more (C) 1000 times more (D) 10 times more\nA: Let's think step by step. The amount of light is proportional to the aperture area $A = \\pi D^2/4$ for a lens with diameter $D$, so the relative amounts of light between the eye with diameter 5mm and the telescope with diameter 50mm is $(50 cm)^2/(5mm)^2 = 10000$. The answer is (A).\n\nQ: Why isn't there a planet where the asteroid belt is located?\n(A) A planet once formed here but it was broken apart by a catastrophic collision. (B) There was not enough material in this part of the solar nebula to form a planet. (C) There was too much rocky material to form a terrestrial planet but not enough gaseous material to form a jovian planet. (D) Resonance with Jupiter prevented material from collecting together to form a planet.\nA: Let's think step by step. The asteroid belt is a stellar disc consisting of a large number of asteroids between Mars and Jupiter's orbits. The asteroids in this belt are affected by the gravitational pull from both other asteroids and nearby planets. Due to the strong gravitational force of Jupiter there are resonances that give rise to low density regions of asteroids known as the Kirkwood gap. So (B) and (C) are not correct since it\u2019s not a lack of material that prevents a planet from being formed, and (A) is incorrect because the Kirkwood gap would have prevented a planet from forming in the first place, and (D) is the correct option. The answer is (D).\n\nQ: Why is Mars red?\n(A) Because the surface is covered with heavily oxidized (\"rusted\") minerals. (B) Because the atmosphere scatters more light at bluer wavelengths transmitting mostly red light. (C) Because Mars is covered with ancient lava flows which are red in color. (D) Because flowing water on Mars's surface altered the surface minerals several billion years ago.\nA: Let's think step by step. Option (B) is not correct because if the red color was caused by the scattering off the atmosphere, then the earth with a much thicker atmosphere would also look red. Options (C) and (D) are not specific enough about why the color of the surface would be red, while (A) is correct because it explains that the surface is red due to the rusted materials on the surface and the red color comes from the rust. So the correct option is (A). The answer is (A).\n\n", "business_ethics": "The following are multiple choice questions (with answers) about business ethics.\n\nQ: In contrast to _______, _______ aim to reward favourable behaviour by companies. The success of such campaigns have been heightened through the use of ___________, which allow campaigns to facilitate the company in achieving _________ .\n(A) Buycotts, Boycotts, Blockchain technology, Charitable donations (B) Buycotts, Boycotts, Digital technology, Increased Sales (C) Boycotts, Buyalls, Blockchain technology, Charitable donations (D) Boycotts, Buycotts, Digital technology, Increased Sales\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cIn contrast to *boycotts*, *buycotts* aim to reward favourable behavior by companies. The success of such campaigns have been heightened through the use of *digital technology*, which allow campaigns to facilitate the company in achieving *increased sales*.\u201d The answer is (D).\n\nQ: _______ is the direct attempt to formally or informally manage ethical issues or problems, through specific policies, practices and programmes.\n(A) Corporate social responsibility (B) Business ethics management (C) Sustainability (D) Environmental management\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The direct attempt manage ethical issues through specific policies, practices, and programs is business ethics management. The answer is (B).\n\nQ: Three contrasting tactics that CSO's can engage in to meet their aims are ________ which typically involves research and communication, ________, which may involve physically attacking a company's operations or ________, often involving some form of _______.\n(A) Non-violent direct action, Violent direct action, Indirect action, Boycott (B) Indirect action, Instrumental action, Non-violent direct action, Information campaign (C) Indirect action, Violent direct action, Non-violent direct-action Boycott (D) Non-violent direct action, Instrumental action, Indirect action, Information campaign\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cThree contrasting tactics that CSO's can engage in to meet their aims are *indirect action*, which typically involves research and communication, *violent direct action*, which may involve physically attacking a company's operations or *non-violent direct action*, often involving some form of *boycott*.\u201d The answer is (C).\n\nQ: To ensure the independence of the non-executive board members, there are a number of steps which can be taken, which include non-executives being drawn from _______ the company, being appointed for a _________ time period as well as being appointed _________.\n(A) Outside, Limited, Independently (B) Inside, Limited, Intermittently (C) Outside, Unlimited, Intermittently (D) Inside, Unlimited, Independently\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cTo ensure the independence of the non-executive board members, there are a number of steps which can be taken, which include non-executives being draw from *outside* the company, being appointed for a *limited* time period as well as being imported *independently*. The answer is (A).\n\nQ: Beyond the business case for engaging in CSR there are a number of moral arguments relating to: negative _______, the _______that corporations possess and the ________ of business and society.\n(A) Externalities, Power, Independence (B) Publicity, Insubstantial resources, Mutual dependence (C) Publicity, Power, Independence (D) Externalities, Power, Mutual dependence\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cBeyond the business case for engaging the CSR there are a number of moral arguments relating to: negative *externalities*, the *power* that corporations possess and the *mutual independence* of business and society. The answer is (D).\n\n", "clinical_knowledge": "The following are multiple choice questions (with answers) about clinical knowledge.\n\nQ: Glycolysis is the name given to the pathway involving the conversion of:\n(A) glycogen to glucose-1-phosphate. (B) glycogen or glucose to fructose. (C) glycogen or glucose to pyruvate or lactate. (D) glycogen or glucose to pyruvate or acetyl CoA.\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. Glycolysis is the name given to the pathway involving conversion of glycogen or glucose to pyruvate or lactate. The answer is (C).\n\nQ: What is the difference between a male and a female catheter?\n(A) Male and female catheters are different colours. (B) Male catheters are longer than female catheters. (C) Male catheters are bigger than female catheters. (D) Female catheters are longer than male catheters.\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. The difference between a male and female catheter is that male catheters tend to be longer than female catheters. The answer is (B).\n\nQ: How many attempts should you make to cannulate a patient before passing the job on to a senior colleague, according to the medical knowledge of 2020?\n(A) 4 (B) 3 (C) 2 (D) 1\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. According to the medical protocol as of 2020, you should make two attempts to cannulate a patient before passing the job on to a more-senior practitioner. The answer is (C).\n\nQ: In the assessment of the hand function which of the following is true?\n(A) Abduction of the thumb is supplied by spinal root T2 (B) Opposition of the thumb by opponens policis is supplied by spinal root T1 (C) Finger adduction is supplied by the median nerve (D) Finger abduction is mediated by the palmar interossei\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. Of all the options, it is only true that the opposition of the thumb by opponens pollicis is supplied by spinal root T1. The answer is (B).\n\nQ: The energy for all forms of muscle contraction is provided by:\n(A) ATP. (B) ADP. (C) phosphocreatine. (D) oxidative phosphorylation.\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. The energy for muscular contraction is provided by ATP (adenosine triphosphate), which is the powerhouse of the cell. The answer is (A).\n\n", "college_biology": "The following are multiple choice questions (with answers) about college biology.\n\nQ: Which of the following represents an accurate statement concerning arthropods?\n(A) They possess an exoskeleton composed primarily of peptidoglycan. (B) They possess an open circulatory system with a dorsal heart. (C) They are members of a biologically unsuccessful phylum incapable of exploiting diverse habitats and nutrition sources. (D) They lack paired, jointed appendages.\nA: Let's think step by step. Peptidoglycan is known to comprise the plasma membrane of most bacteria, rather than the exoskeleton of arthropods, which is made of chitin, which rules out (A). The answer (C) is false because arthropods are a highly successful phylum. Likewise, arthropods have paired, jointed appendages, which rules out (D). The only remaining option is (B), as arthropods have an open circulatory system with a dorsal tubular heart. The answer is (B).\n\nQ: In a given population, 1 out of every 400 people has a cancer caused by a completely recessive allele, b. Assuming the population is in Hardy-Weinberg equilibrium, which of the following is the expected proportion of individuals who carry the b allele but are not expected to develop the cancer?\n(A) 1/400 (B) 19/400 (C) 20/400 (D) 38/400\nA: Let's think step by step. According to the Hardy Weinberg Law, $p^2 + 2 p q + q^2 = 1$, and $p + q = 1$ where $p$ is the frequency of the dominant allele, $q$ is the frequency of the recessive allele, and $p^2$, $q^2$, and $2pq$ are the frequencies of dominant homozygous, recessive homozygous, and heterozygous individuals, respectively. \u200bThe frequency of the recessive allele (q) is $\\sqrt{\frac{1}{400}} = 0.05$. We have $p = 1 - q = 0.95$. The frequency of heterozygous individuals is $2pq = 2 \\cdot 0.05 \\cdot 0.95 = 0.095$. The number of heterozygous individuals is equal to the frequency of heterozygous individuals times the size of the population, or $0.095 * 400 = 38$. So we end up with 38/400. The answer is (D).\n\nQ: According to the pressure-flow model of movement of phloem contents, photosynthate movement from source to sink is driven by\n(A) an ATP-dependent pressure-flow pump (B) a water-pressure potential gradient (C) transpiration (D) apoplastic diffusion\nA: Let's think step by step. It is a gradient in water pressure that induces the movement of phloem content, which refers to answer (B). The mechanism of movement does not rely on metabolism, which rules out (A). Transpiration refers to the exhalation of water vapor through plant stomata, and is also not related, which rules out (C). While the apoplastic pathway is one of two main pathways for water transport in plants, it is not central to the pressure flow model, which rules out (D). The answer is (B).\n\nQ: Which of the following contain DNA sequences required for the segregation of chromosomes in mitosis and meiosis?\n(A) Telomeres (B) Centromeres (C) Nucleosomes (D) Spliceosomes\nA: Let's think step by step. The genetic material in Telomeres is not used, which rules out (A). Nucleosomes are the repeating subunit that comprises chromatin packed in a cell nucleus, and do not specifically refer to DNA sequences necessary for segregating chromosomes in cell division, which rules out (C). A spliceosome is a large ribonucleoprotein that removes introns from transcribed pre-mRNA rather than governing chromosome segregation. Centromeres are directly responsible for segregating chromosomes in cell division. The answer is (B).\n\nQ: The presence of homologous structures in two different organisms, such as the humerus in the front limb of a human and a bird, indicates that\n(A) the human and bird are polyphyletic species (B) a human's and bird's evolution is convergent (C) the human and bird belong to a clade (D) the human and bird developed by analogy\nA: Let's think step by step. Polyphyletic species are organisms that are grouped due to having similar characteristics but which do not have a common ancestor. This is not the case for humans and birds, which rules out (A). Convergent evolution refers to the indepdendent development of similar features in different species at different periods, which is also not the case for humans and birds, which rules out (B). Analogy refers to the superficial resemblance of structures that have different origins, which is not the case for the human and bird forearms, which rules out (D). Humans and birds do belong to the same clade - a group of organisms composed of a common ancestor. The answer is (C).\n\n", "college_chemistry": "The following are multiple choice questions (with answers) about college chemistry.\n\nQ: 3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq) + 4 Cr3+(aq) + 10 H2O(l). In the reaction shown above, Cl\u2212(aq) behaves as\n(A) an acid (B) a base (C) a catalyst (D) a reducing agent\nA: Let's think step by step. A molecule that behaves as a base accepts an H+ ion (or proton) from another molecule, whereas a molecule that behaves as an acid donates an H+ ion (or proton) to another molecule. Neither of these is the case for Cl in this reaction, which rules out (A) and (B). A catalyst is a substance that only accelerates a reaction without itself undergoing chemical change, which is not the case here. This rules out (C). Instead, the $Cl^{-} molecules carry a negative charge, which they donate in the reaction to form 3 HClO2. This is the behavior of a reducing agent, or (D). The answer is (D).\n\nQ: Which of the following statements about the lanthanide elements is NOT true?\n(A) The most common oxidation state for the lanthanide elements is +3. (B) Lanthanide complexes often have high coordination numbers (> 6). (C) All of the lanthanide elements react with aqueous acid to liberate hydrogen. (D) The atomic radii of the lanthanide elements increase across the period from La to Lu.\nA: Let's think step by step. The atomic radii of the lanthanide elements in fact decrease across the period from La to Lu. Options (A), (B), and (C) are all true. This means that only (D) is NOT true. The answer is (D).\n\nQ: Which of the following lists the hydrides of group-14 elements in order of thermal stability, from lowest to highest?\n(A) PbH4 < SnH4 < GeH4 < SiH4 < CH4 (B) PbH4 < SnH4 < CH4 < GeH4 < SiH4 (C) CH4 < SiH4 < GeH4 < SnH4 < PbH4 (D) CH4 < PbH4 < GeH4 < SnH4 < SiH4\nA: Let's think step by step. The thermal stability of group-14 hydrides decreases as we move from the top of group 14 to the bottom. The order of elements in the group from top to bottom is C, Si, Ge, Sn, Pb. Therefore in order of increasing thermal stability we have PbH4, SnH4, GeH4, SiH4, and CH4, or answer (A). The answer is (A).\n\nQ: Predict the number of lines in the EPR spectrum of a solution of 13C-labelled methyl radical (13CH3\u2022), assuming the lines do not overlap.\n(A) 4 (B) 3 (C) 6 (D) 24 (E) 8\nA: Let's think step by step. The electron paramagnetic resonance spectrum will be split by two forms of interactions. The first is the hyperfine interaction with the 13C (nuclear spin $I = \nrac{1}{2}$) which will split the spectrum into 2 lines. This will be further split into 4 lines by the interaction with three equivalent 1H nuclei. The total number of lines is therefore $2 \\cdot 4 = 8$. The answer is (E).\n\n", "college_computer_science": "The following are multiple choice questions (with answers) about college computer science.\n\nQ: Which of the following regular expressions is equivalent to (describes the same set of strings as) (a* + b)*(c + d)?\n(A) a*(c + d)+ b(c + d)\n(B) a*(c + d)* + b(c + d)*\n(C) a*(c + d)+ b*(c + d)\n(D) (a + b)*c +(a + b)*d\nA: Let's think step by step. We know that:\n1. (X* + Y)* = (X + Y)*\n2. X(Y + Z)? = XY + XZ\nUsing equation 1 we can rewrite (a* + b)*(c + d)? as:\n3. (a + b)*(c + d)?\nUsing equation 2 we can rewrite equation 3 as:\n(a + b)*c + (a + b)*d The answer is (D).\n\nQ: The Singleton design pattern is used to guarantee that only a single instance of a class may be instantiated. Which of the following is (are) true of this design pattern?\nI. The Singleton class has a static factory method to provide its instance.\nII. The Singleton class can be a subclass of another class.\nIII. The Singleton class has a private constructor.\n(A) I only\n(B) II only\n(C) III only\n(D) I, II, and III\nA: Let's think step by step. Statement I is a correct statement about a Singleton, because a Singleton restricts instantiation to a single, static method. Statement II is also correct, because there is no inherent restriction regarding the inheritance of a Singleton. Statement III is also correct, because a Singletons must be instantiated only once, so its constructor is made private to prevent any construction except via its static factory method.\nGiven these facts, statements I, II, and III are all correct. The answer is (D).\n\nQ: A certain pipelined RISC machine has 8 general-purpose registers R0, R1, . . . , R7 and supports the following operations:\nADD Rs1, Rs2, Rd (Add Rs1 to Rs2 and put the sum in Rd)\nMUL Rs1, Rs2, Rd (Multiply Rs1 by Rs2 and put the product in Rd)\nAn operation normally takes one cycle; however, an operation takes two cycles if it produces a result required by the immediately following operation in an operation sequence.\nConsider the expression AB + ABC + BC, where variables A, B, C are located in registers R0, R1, R2. If the contents of these three registers must not be modified, what is the minimum number of clock cycles required for an operation sequence that computes the value of AB + ABC + BC?\n(A) 5 (B) 6 (C) 7 (D) 8\nA: Let's think step by step. First, we are given that A is in R0, B is in R1, and C is in R2.\nNext, we can see that we must compute three multiplies (AB, BC, and ABC) and two adds (AB + ABC, (AB + ABC) + BC) to compute our final answer, resulting in a minimum of five clock cycles.\nNext, we can see that there is no way to avoid at least one pipeline stall when computing our final answer, because to compute our final sum we must wait at least one cycle for the results from the previous stage to be ready. Thus, our minimum number of cycles must be 6.\nWe can verify that we can create a solution that requires only six cycles as follows:\ncompute AB: MUL R0, R1, R3\ncompute BC: MUL R1, R2, R4\ncompute ABC: MUL R3, R4, R5\ncompute AB + BC: ADD R3, R4, R6\nSTALL\ncompute AB + ABC + BC: ADD R5, R6, R7\nSo there are 6 cycles. The answer is (B).\n\nQ: A compiler generates code for the following assignment statement.\nG := (A + B) * C - (D + E) * F\nThe target machine has a single accumulator and a single-address instruction set consisting of instructions load, store, add, subtract, and multiply. For the arithmetic operations, the left operand is taken from the accumulator and the result appears in the accumulator. The smallest possible number of instructions in the resulting code is\n(A) 5 (B) 6 (C) 7 (D) 9\nA: Let's think step by step. We can compute the final answer with the following sequence of operations:\n1. LOAD D (accumulator = D)\n2. ADD E (accumulator = D+E)\n3. MUL F (accumulator = (D+E)*F)\n4. STORE X (X = (D+E)*F)\n5. LOAD A (accumulator = A)\n6. ADD B (accumulator = A+B)\n7. MUL C (accumulator = (A+B)*C)\n8. SUB X (accumulator = (A+B)*C - (D+E)*F)\n9. STORE G (G = (A+B)*C - (D+E)*F)\nThis sequence takes 9 instructions. The answer is (D).\n\nQ: Consider a computer design in which multiple processors, each with a private cache memory, share global memory using a single bus. This bus is the critical system resource. Each processor can execute one instruction every 500 nanoseconds as long as memory references are satisfied by its local cache. When a cache miss occurs, the processor is delayed for an additional 2,000 nanoseconds. During half of this additional delay, the bus is dedicated to serving the cache miss. During the other half, the processor cannot continue, but the bus is free to service requests from other processors. On average, each instruction requires 2 memory references. On average, cache misses occur on 1 percent of references. What proportion of the capacity of the bus would a single processor consume, ignoring delays due to competition from other processors?\n(A) 1/50 (B) 1/27 (C) 1/25 (D) 2/27\nA: Let's think step by step. We know that each instruction requires two memory references per instruction, and that there is an average cache miss rate of one percent.\nThus a given processor has:\n(1 cache miss / 100 references) * (2 references / instruction) =\n(2 cache misses / 100 instructions), so:\nmisses_per_instruction = 1 cache miss / 50 instructions.\nNext, we know that each instruction requires 500 nanoseconds when there is no cache miss, and 500 + 2000 = 2500 nanoseconds when there is a cache miss. Thus:\n50 instructions / (49 * 500) + (1 * 2500) nanoseconds, so:\ninstructions_per_ns = 50 instructions / 27000 nanoseconds.\nNow, we know that each cache miss locks the bus for half of the 2000 nanosecond cache miss delay, or 1000 nanoseconds, so:\nlock_ns_per_miss = 1000 nanoseconds / cache miss.\nThus we can see that on average a single processor will lock the bus for:\nlock_ns_per_miss * misses_per_instruction * instructions_per_ns =\n(1000 nanoseconds / cache miss) * (1 cache miss / 50 instructions) * (50 instructions / 27000 nanoseconds) = 1000 * (1/50) * (50/27000) = 1000/27000 = 1/27. The answer is (B).\n\n", "college_mathematics": "The following are multiple choice questions (with answers) about college mathematics.\n\nQ: Let V be the set of all real polynomials p(x). Let transformations T, S be defined on V by T:p(x) -> xp(x) and S:p(x) -> p'(x) = d/dx p(x), and interpret (ST)(p(x)) as S(T(p(x))). Which of the following is true?\n(A) ST = 0 (B) ST = T (C) ST = TS (D) ST - TS is the identity map of V onto itself.\nA: Let's think step by step. For a given polynomial $p$ we have\n\\[ST(p) = (xp(x))\u2019 = p(x) + xp\u2019(x)\\]\nand\n\\[TS(p) = xp\u2019(x).\\]\nHence \\[ST(p) - TS(p) = p(x) + xp\u2019(x) - xp\u2019(x).\\] The answer is (D).\n\nQ: Suppose that f(1 + x) = f(x) for all real x. If f is a polynomial and f(5) = 11, then f(15/2)\n(A) -11 (B) 0 (C) 11 (D) 33/2\nA: Let's think step by step. The only polynomial so that $f(1 + x) = f(x)$ is a constant polynomial. Hence $f(5) = 11 = f(15/2)$. The answer is (C).\n\nQ: Let A be a real 2x2 matrix. Which of the following statements must be true?\nI. All of the entries of A^2 are nonnegative.\nII. The determinant of A^2 is nonnegative.\nIII. If A has two distinct eigenvalues, then A^2 has two distinct eigenvalues.\n(A) I only (B) II only (C) III only (D) II and III only\nA: Let's think step by step. We have \\[ det(A^2) = (det(A))^2 \\geq 0,\\] hence II holds.\nIII is false: as a counterexample take a diagonal matrix with -1 and 1 on the diagonal. Then $A^2$ is the identity matrix. The answer is (B).\n\nQ: Let A be the set of all ordered pairs of integers (m, n) such that 7m + 12n = 22. What is the greatest negative number in the set B = {m + n : (m, n) \\in A}?\n(A) -5 (B) -4 (C) -3 (D) -2\nA: Let's think step by step. We have 12n = 22 - 7m and one of the solutions is $m = -2$, $n = 3$. Then $m + n = 1$, hence we need to look for smaller $m$ in order to make $m + n$ negative. The next solution is $m = -14$ and $n = 10$. For smaller $m$ we have $m + n$ smaller than $-4$. The answer is (B).\n\nQ: A tank initially contains a salt solution of 3 grams of salt dissolved in 100 liters of water. A salt solution containing 0.02 grams of salt per liter of water is sprayed into the tank at a rate of 4 liters per minute. The sprayed solution is continually mixed with the salt solution in the tank, and the mixture flows out of the tank at a rate of 4 liters per minute. If the mixing is instantaneous, how many grams of salt are in the tank after 100 minutes have elapsed?\n(A) 2 (B) 2 - e^-2 (C) 2 + e^-2 (D) 2 + e^-4\nA: Let's think step by step. For all $t \\in \\mathbb{R}$, let $s(t)$ denote the number grams of salt in the tank at the $t$ minute mark. Then $s(0) = 3$.\nWe use $s$ and $s(t)$ interchangeably. We also use $s^{\\prime}$ and $s^{\\prime}(t)$ interchangeably. The solution sprayed into the tank adds $(0.02) 4=2 / 25$ grams of salt per minute. There are always 100 liters of liquid in the tank, containing $s$ grams of salt. So the density of salt in the tank is $s / 100$ grams per liter. The flow of water out of the tank therefore subtracts $4(s / 100)=s / 25$ grams of salt per minute. Then, for all $t \\in \\mathbb{R}$, we have $s^{\\prime}(t)=(2 / 25)-(s / 25)=(2-s) / 25$, and so $[s(t)=2] \\Rightarrow\\left[s^{\\prime}(t)=0\right]$. For all $t \\in \\mathbb{R}$,\n$$\n\frac{d}{d t}[\\ln (s-2)]=\frac{s^{\\prime}}{s-2}=\frac{-1}{25}=\frac{d}{d t}\\left[-\frac{t}{25}\right] .\n$$\nChoose $C \\in \\mathbb{R}$ such that, for all $t \\in \\mathbb{R}, \\ln ((s(t)-2))=-[t / 25]+C$. Let $K:=e^{C}$. Then, for all $t \\in \\mathbb{R}$, we have $(s(t))-2=K e^{-t / 25}$, and so $s(t)=2+K e^{-t / 25}$. Then $3=s(0)=2+K e^{0}=2+K$, so $K=1$. Then $s(100)=2+K e^{-100 / 25}=2+1 \\cdot e^{-4}=2+e^{-4}$. The answer is (D).\n\n", "college_medicine": "The following are multiple choice questions (with answers) about college medicine.\n\nQ: An expected side effect of creatine supplementation is:\n(A) muscle weakness. (B) gain in body mass. (C) muscle cramps. (D) loss of electrolytes.\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Creatine supplementation is a dietary supplement that results in body mass gain. The answer is (B).\n\nQ: Which of the following is not a true statement?\n(A) Muscle glycogen is broken down enzymatically to glucose-1-phosphate (B) Elite endurance runners have a high proportion of Type I fibres in their leg muscles (C) Liver glycogen is important in the maintenance of the blood glucose concentration (D) Insulin promotes glucose uptake by all tissues in the body\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Let\u2019s solve this step by step and go over each choice: \n(A) \u201cMuscle glycogen is broken down enzymatically to glucose-1-phosphate\u201d: This is a correct statement.\n(B) \u201cElite endurance runners have a high proportion of Type I fibres in their leg muscles\u201d: This is a correct statement.\n(C) \u201cLiver glycogen is important in the maintenance of the blood glucose concentration\u201d: This is a correct statement. \n(D) \u201cInsulin promotes glucose uptake by all tissues in the body\u201d: This is not a correct statement, because insulin promotes glucose uptake by the liver, adipose tissue, and muscle, but not all tissues. For instance, the tissues in the brain and red blood cells are not affected by insulin. The answer is (D).\n\nQ: A high school science teacher fills a 1 liter bottle with pure nitrogen and seals the lid. The pressure is 1.70 atm, and the room temperature is 25\u00b0C. Which two variables will both increase the pressure of the system, if all other variables are held constant?\n(A) Increasing temperature, increasing moles of gas (B) Increasing temperature, increasing volume (C) Decreasing volume, decreasing temperature (D) Decreasing moles of gas, increasing volume\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The relevant equation for this is the ideal gas law: PV=nRT. To increase the pressure of the system (P), then either n (number of moles of the gas) or T (temperature) have to increase. The answer is (A).\n\nQ: In a genetic test of a newborn, a rare genetic disorder is found that has X-linked recessive transmission. Which of the following statements is likely true regarding the pedigree of this disorder?\n(A) All descendants on the maternal side will have the disorder. (B) Females will be approximately twice as affected as males in this family. (C) All daughters of an affected male will be affected. (D) There will be equal distribution of males and females affected.\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Let\u2019s solve this step by step. Let's recall first that females have two X chromosomes, while males have one X and one Y chromosome. This is an important fact we need to know before answering this question. \nBecause a male can only pass his only one X chromosome to a daughter, if he is affected by this rare genetic disorder, then we know for sure that he will pass this rare genetic disorder to all his future-born daughters. Therefore, \u201c(C): All daughters of an affected male will be affected\u201d is a correct statement. The answer is (C).\n\nQ: Glucose is transported into the muscle cell:\n(A) via protein transporters called GLUT4. (B) only in the presence of insulin. (C) via hexokinase. (D) via monocarbylic acid transporters.\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Glucose (also known as the blood sugar) is the main sugar found in the human body. It is transported into the muscle cell via diffusion through protein transporters called GLUT4. The answer is (A).\n\n", "college_physics": "The following are multiple choice questions (with answers) about college physics.\n\nQ: A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is\n(A) 4 (B) 5 (C) 6 (D) 20\nA: Let's think step by step. In a refracting telescope, if both lenses are converging, the focus of both lenses must be between the two lenses, and thus the focal lengths of the two lenses must add up to their separation. Since the focal length of one lens is 20 cm, the focal length of the other must be 80 cm. The magnification is the ratio of these two focal lengths, or 4. The answer is (A).\n\nQ: The muon decays with a characteristic lifetime of about 10^-6 second into an electron, a muon neutrino, and an electron antineutrino. The muon is forbidden from decaying into an electron and just a single neutrino by the law of conservation of\n(A) charge (B) mass (C) energy and momentum (D) lepton number\nA: Let's think step by step. Lepton number must be conserved, meaning the total number of leptons minus the number of antileptons. If a muon decays into an electron and a single neutrino, the total lepton number would go from one to two, violating lepton number conservation. The answer is (D).\n\nQ: One end of a Nichrome wire of length 2L and cross-sectional area A is attached to an end of another Nichrome wire of length L and cross- sectional area 2A. If the free end of the longer wire is at an electric potential of 8.0 volts, and the free end of the shorter wire is at an electric potential of 1.0 volt, the potential at the junction of the two wires is most nearly equal to\n(A) 2.4 V (B) 3.3 V (C) 4.5 V (D) 5.7 V\nA: Let's think step by step. This is a simple voltage divider problem, where the longer wire has a resistance four times that of the shorter end. So the voltage divider ratio is 1 / 5, meaning that the potential in the middle is 1.0 V + (8.0 V - 1.0 V) * 1/5 = 2.4 V. The answer is (A).\n\nQ: A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is\n(A) 4 (B) 5 (C) 6 (D) 20\nA: Let's think step by step. In a refracting telescope, if both lenses are converging, the focus of both lenses must be between the two lenses, and thus the focal lengths of the two lenses must add up to their separation. Since the focal length of one lens is 20 cm, the focal length of the other must be 80 cm. The magnification is the ratio of these two focal lengths, or 4. The answer is (A).\n\nQ: For which of the following thermodynamic processes is the increase in the internal energy of an ideal gas equal to the heat added to the gas?\n(A) Constant temperature (B) Constant volume (C) Constant pressure (D) Adiabatic\nA: Let's think step by step. Heat added to the gas can go into the gases internal energy or work done against an external force. However, if the volume of the gas container is constant, no work will be done (since work is pressure times change in volume). So, at constant volume, all of the heat goes into the internal energy. The answer is (B).\n\n", "computer_security": "The following are multiple choice questions (with answers) about computer security.\n\nQ: SHA-1 has a message digest of\n(A) 160 bits (B) 512 bits (C) 628 bits (D) 820 bits\nA: Let's think step by step. Since SHA-1 is a hash function which takes an input and produces a 160-bit (20-byte) hash value, its message digest is 160 bits. The answer is (A).\n\nQ: _____________ can modify data on your system \u2013 so that your system doesn\u2019t run correctly or you can no longer access specific data, or it may even ask for ransom in order to give your access.\n(A) IM \u2013 Trojans (B) Backdoor Trojans (C) Trojan-Downloader (D) Ransom Trojan\nA: Let's think step by step. The system is asking for trojans, which are for ransom, which means ransom trojan. The answer is (D).\n\nQ: What is ethical hacking?\n(A) \"Hacking\" ethics so they justify unintended selfish behavior (B) Hacking systems (e.g., during penetration testing) to expose vulnerabilities so they can be fixed, rather than exploited (C) Hacking into systems run by those whose ethics you disagree with (D) A slang term for rapid software development, e.g., as part of hackathons\nA: Let's think step by step. Ethical hacking is a process of detecting vulnerabilities in an application, system, or organization's infrastructure that an attacker can use to exploit an individual or organization. They use this process to prevent cyberattacks and security breaches by lawfully hacking into the systems and looking for weak points. The answer is (B).\n\nQ: The ____________ is anything which your search engine cannot search.\n(A) Haunted web (B) World Wide Web (C) Surface web (D) Deep Web\nA: Let's think step by step. The search engine searches on the Surface Web, which is the portion of the world wide web which is visible so (B,C) are wrong. The Haunted Web doesn\u2019t correspond to an internet concept. The Deep Web is the part of the World Wide Web which is not indexed. The answer is (D).\n\nQ: Exploitation of the Heartbleed bug permits\n(A) overwriting cryptographic keys in memory (B) a kind of code injection (C) a read outside bounds of a buffer (D) a format string attack\nA: Let's think step by step. The Heartbleed Bug is a serious vulnerability in the popular OpenSSL cryptographic software library. Heartbleed resulted from improper input validation (due to a missing bounds check) in the implementation of the TLS heartbeat extension. The vulnerability was classified as a buffer over-read, a situation where more data can be read than should be allowed. The answer is (C).\n\n", "conceptual_physics": "\nThe following are multiple choice questions (with answers) about conceptual physics.\n\nQ: Colors in a soap bubble result from light\n(A) converted to a different frequency (B) deflection (C) interference (D) polarization\nA: Let's think step by step. In a soap bubble film, the light bounces between the two soap-air interfaces many times, interfering with itself constructively or destructively depending on the width of the film. This results in different colors being visible. The answer is (C).\n\nQ: Compared with the mass of a uranium atom undergoing fission, the combined masses of the products after fission are\n(A) less (B) more (C) the same (D) zero\nA: Let's think step by step. Fission releases energy, which comes from the rest mass of its initial nucleus. Thus the mass of the products is less than the mass of the reactant uranium nucleus. The answer is (A).\n\nQ: Things that are equivalent according to the equivalence principle are\n(A) space and time. (B) a traveling twin and a stay-at-home twin. (C) gravity and acceleration. (D) mass and energy.\nA: Let's think step by step. Einstein\u2019s famous equivalence principle states that gravity and acceleration are equivalent. The answer is (C).\n\nQ: Which of these three elements has the most mass per nucleon?\n(A) Hydrogen (B) Iron (C) Uranium (D) Same in each\nA: Let's think step by step. Due to nuclear binding energy, the mass of an atomic nucleus is less than the sum of individual masses of the free constituent protons and neutrons; this is known as the mass defect. Hydrogen has no mass defect because it has only a single nucleon, so it will have the most mass per nucleon. The answer is (A).\n\nQ: A model airplane flies slower when flying into the wind and faster with wind at its back. When launched at right angles to the wind a cross wind its groundspeed compared with flying in still air is\n(A) the same (B) greater (C) less (D) either greater or less depending on wind speed\nA: Let's think step by step. The plane\u2019s speed in the direction of the wind is greater than it would be in the absence of wind, and its direction orthogonal to the wind is the same as it would be in the absence of the wind. The total speed, which is these two components added in quadrature, is thus greater than the speed in still air. The answer is (B).\n\n", "econometrics": "The following are multiple choice questions (with answers) about econometrics.\n\nQ: Suppose now that a researcher wishes to use information criteria to determine the optimal lag length for a VAR. 500 observations are available for the bi-variate VAR, and the values of the determinant of the variance-covariance matrix of residuals are 0.0336, 0.0169, 0.0084, and 0.0062 for 1, 2, 3, and 4 lags respectively. What is the optimal model order according to Akaike's information criterion?\n(A) 1 lag (B) 2 lags (C) 3 lags (D) 4 lags\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Let\u2019s solve this problem step by step. First of all, let\u2019s recall that for a given set of data, Akaike's information criterion (AIC) allows us to measure how well a statistical model fits the data; it is an estimator of prediction error. Here in this problem we will need to use the formula ln(det(sigma_hat)) + (2 * k / T) to determine the values of Akaike\u2019s criterion, where ln denotes the natural log function, det the determinant function, k the total number of parameters in total (across both equations), and T the number of observations (which, in this case, is equal to 500). For 1 lag, the number of parameters in total is equal to 6; for 2 lags, it is 10; for 3 lags, it is 14; and for 4 lags, it is 18. Now, let\u2019s calculate the values of the criterion for each lag:\n(A) 1 lag: ln(0.0336) + (2 * 6 / 500) = ln(0.0336) + (12 / 500) = -3.369\n(B) 2 lags: ln(0.0169) + (2 * 10 / 500) = ln(0.0169) + (20 / 500) = -4.040\n(C) 3 lags: ln(0.0084) + (2 * 14 / 500) = ln(0.0084) + (28 / 500) =-4.724\n(D) 4 lags: ln(0.0062) + (2 * 18 / 500) = ln(0.0062) + (36 / 500) =-5.011\nBecause the optimal model order according to AIC minimizes the information criterion, the answer should be the one with the lowest value. In this case, (D) has the lowest value. The answer is (C).\n\nQ: Consider the following AR(1) model with the disturbances having zero mean and unit variance\nyt = 0.2 + 0.4 yt-1 + ut\nThe (unconditional) mean of y will be given by\n(A) 0.2 (B) 0.4 (C) 0.5 (D) 0.33\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Let\u2019s solve this problem step by step. If we have a an AR(1) model with the disturbances having zero mean and unit variance, then the unconditional mean of y is equal to the following:\nunconditional mean of y = (the intercept term) / (1 - autoregressive coefficient)\nWe know that the intercept term is 0.2 and the autoregressive coefficient is 0.4; thus, we have:\nunconditional mean of y = (0.2) / (1 - 0.4) = (0.2) / (0.6) = 2 / 6 = 1 / 3, which is approximately 0.33. That means that the answer should be (D) 0.33. The answer is (D).\n\nQ: What would be then consequences for the OLS estimator if heteroscedasticity is present in a regression model but ignored?\n(A) It will be biased (B) It will be inconsistent (C) It will be inefficient (D) All of (a), (b) and (c) will be true.\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Heteroscedasticity refers to the condition where the variance of the error terms is not constant across multiple observations. If heteroscedasticity is present in a regression model, then the coefficient estimates in the OLS estimator will be not only unbiased and consistent but also inefficient. Because (A) and (B) are incorrect choices and (C) is a correct choice, (D) cannot be the right answer. Ultimately, (C) is the only true choice. The answer is (C).\n\nQ: Suppose that a test statistic has associated with it a p-value of 0.08. Which one of the following statements is true?\n(i) If the size of the test were exactly 8%, we would be indifferent between rejecting and not rejecting the null hypothesis\n(ii) The null would be rejected if a 10% size of test were used\n(iii) The null would not be rejected if a 1% size of test were used\n(iv) The null would be rejected if a 5% size of test were used.\n(A) (ii) and (iv) only (B) (i) and (iii) only (C) (i), (ii), and (iii) only (D) (i), (ii), (iii), and (iv).\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Let\u2019s reason about each of the options.\n(i) is a true statement.\n(ii) is a true statement.\n(iii) is a true statement.\n(iv) is not a true statement. Thus, (i), (ii), and (iii) are true. The answer is (C).\n\nQ: For a stationary autoregressive process, shocks will\n(A) Eventually die away (B) Persist indefinitely (C) Grow exponentially (D) Never occur\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. This is a formal logic problem about stationally process. For a stationary autoregressive process, shocks will eventually die away. The answer is (A).\n\n", "electrical_engineering": "\nThe following are multiple choice questions (with answers) about electrical engineering.\n\nQ: A point pole has a strength of 4\u03c0 * 10^-4 weber. The force in newtons on a point pole of 4\u03c0 * 1.5 * 10^-4 weber placed at a distance of 10 cm from it will be\n(A) 15 N. (B) 20 N. (C) 7.5 N. (D) 3.75 N.\nA: Let's think step by step. The force between two point poles is given by m_1m_2/(mu_0 4 \\pi r^2), in analogy to Coulomb\u2019s law. Plugging in the values given in the question, we calculate that the force is approximately 15 N. The answer is (A).\n\nQ: The coil of a moving coil meter has 100 turns, is 40 mm long and 30 mm wide. The control torque is 240*10-6 N-m on full scale. If magnetic flux density is 1Wb/m2 range of meter is\n(A) 1 mA. (B) 2 mA. (C) 3 mA. (D) 4 mA.\nA: Let's think step by step. The torque on a coil in a uniform magnetic field is given by BANI, where B is the magnetic flux density, A is the area of the coil, N is the number of turns, and I is the current. So we have that I = (Torque)/(BAN), or 240e-6/(1200e-6 * 100 * 1) = 2e-3. The answer is (B).\n\nQ: In an SR latch built from NOR gates, which condition is not allowed\n(A) S=0, R=0 (B) S=0, R=1 (C) S=1, R=0 (D) S=1, R=1\nA: Let's think step by step. An SR latch is a set-reset latch; in the case where S=1 and R=1, the circuit has no stable state; instead a race condition will be produced within the circuit, so the device will be in an undefined state. So S=1, R=1 is an illegal input. The answer is (D).\n\nQ: Two long parallel conductors carry 100 A. If the conductors are separated by 20 mm, the force per meter of length of each conductor will be\n(A) 100 N. (B) 0.1 N. (C) 1 N. (D) 0.01 N.\nA: Let's think step by step. The magnetic force-per-length between two current-carrying conductors is given by \\mu_0 I_1 I_2 / (2 \\pi r), where $r$ is the separation distance and I_1 and I_2 are the currents. Plugging in 100 A for I_1 and I_2, and 20 mm for r, gives 0.1 N. The answer is (B).\n\nQ: In a 2 pole lap winding dc machine , the resistance of one conductor is 2\u03a9 and total number of conductors is 100. Find the total resistance\n(A) 200\u03a9 (B) 100\u03a9 (C) 50\u03a9 (D) 10\u03a9\nA: Let's think step by step. In lap winding, effectively two resistors are connected in parallel, so the actual resistance of each pair is 1 Ohm. Since we have 50 pairs, we get a total resistance of 50 Ohms. The answer is (C).\n\n", "elementary_mathematics": "The following are multiple choice questions (with answers) about elementary mathematics.\n\nQ: Olivia used the rule \"Add 11\" to create the number pattern shown below. 10, 21, 32, 43, 54. Which statement about the number pattern is true?\n(A) The 10th number in the pattern will be an even number.\n(B) The number pattern will never have two even numbers next to each other.\n(C) The next two numbers in the pattern will be an even number then an odd number.\n(D) If the number pattern started with an odd number then the pattern would have only odd numbers in it.\nA: Let's think step by step. Choice A is incorrect because every even-numbered term in the pattern is odd, and 10 is an even number. Choice B is correct, because adding an odd number (in this case 11) to an odd number produces an even number, and adding an odd number to an even number produces an odd number. Thus the terms in the pattern will alternate between odd and even, so there will never be two even numbers next to each other. Choice C is incorrect because the last term in the example is even (54), and we know that the terms will alternate between even and odd. Choice D is incorrect because the terms in the pattern will alternate between odd and even, regardless of the value of the first term. The answer is (B).\n\nQ: The population of the city where Michelle was born is 145,826. What is the value of the 5 in the number 145,826?\n(A) 5 thousands\n(B) 5 hundreds\n(C) 5 tens\n(D) 5 ones\nA: Let's think step by step. Choice A is correct, because there are three digits following the 5, so\nthe 5 is in the thousands place. Thus the other choices are incorrect. The answer is (A).\n\nQ: A store sells 107 different colors of paint. They have 25 cans of each color in storage. The number of cans of paint the store has in storage can be found using the expression below. 107 \u00d7 25. How many cans of paint does the store have in storage?\n(A) 749\n(B) 2,675\n(C) 2,945\n(D) 4,250\nA: Let's think step by step. We can calculate 107 x 25 = (100 x 25) + (7 x 25) = 2500 + 175 = 2675. The answer is (B).\n\nQ: A total of 30 players will play basketball at a park. There will be exactly 5 players on each team. Which statement correctly explains how to find the number of teams needed?\n(A) Add 5 to 30 to find 35 teams.\n(B) Divide 30 by 5 to find 6 teams.\n(C) Multiply 30 and 5 to find 150 teams.\n(D) Subtract 5 from 30 to find 25 teams.\nA: Let's think step by step. We want to find the number of teams. We know that there are 5 players/team, and 30 players. Thus to get the number of teams we divide players by players/team, so 30 players / 5 players/team = 6 teams. The answer is (B).\n\nQ: Which expression is equivalent to 5 x 9?\n(A) (5 x 4) x (6 x 5)\n(B) (5 x 5) + (5 x 4)\n(C) (5 x 5) + (5 x 9)\n(D) (5 x 9) x (6 x 9)\nA: Let's think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5 + 4) = (5 x 5) + (5 x 4). The answer is (B).\n\n", "formal_logic": "The following are multiple choice questions (with answers) about formal logic.\n\nQ: Which of the given formulas of PL is the best symbolization of the following sentence?\nTurtles live long lives and are happy creatures, unless they are injured.\n(A) (L \u2022 H) \u2261 I (B) (L \u2022 H) \u2228 I (C) L \u2022 (H \u2228 I) (D) L \u2022 (H \u2283 R).\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. Let \u201cL\u201d denote \u201cliving long\u201d, H \u201cbeing happy\u201d, and \u201cI\u201d \u201cbeing injured\u201d. Now, consider each choice:\n(A) means (living long AND being happy) is equivalent to (being injured). \n(B) means (living long AND being happy) OR (being injured). \n(C) means (living long) AND (being happy OR being injured). \n(D) means (living long) AND (being happy implies being R), but what R denotes is not clear.\nObviously, (B) is the best symbolization of the original sentence. The answer is (B).\n\nQ: Select the best translation into predicate logic.George borrows Hector's lawnmower. (g: George; h: Hector; l: Hector's lawnmower; Bxyx: x borrows y from z).\n(A) Blgh (B) Bhlg (C) Bglh (D) Bghl\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. We are told that \u201cBxyx\u201d means \u201cx borrows y from z\u201d. We can rewrite \u201cGeorge borrows Hector's lawnmower\u201d as \u201cGeorge borrows a lawnmower from Hector\u201d, which can then be translated into predicate logic as \u201cBglh\u201d. The answer \u201cBglh\u201d appears in (C); therefore, (C) must be the correct answer. The answer is (C).\n\nQ: \nSelect the best English interpretation of the given arguments in predicate logic.\nDm\n(\u2200x)(Wx \u2283 ~Dx). \n(\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax\n(A) Marina is a dancer. Some weaklings are not dancers. Either everything is a weakling or Georgia plays volleyball. So something plays volleyball. (B) Marina is a dancer. No weakling is a dancer. Everything is either a weakling or plays volleyball. So something plays volleyball. (C) Marina is a dancer. Some weaklings are not dancers. Everything is either a weakling or plays volleyball. So something plays volleyball. (D) Marina is a dancer. No weakling is a dancer. Either everything is a weakling or Georgia plays volleyball. So something plays volleyball.\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. Let \u201cD\u201d denote \u201cbeing a dancer\u201d, \u201cm\u201d denote \u201cMaria\u201d, \u201cg\u201d denote \u201cGeorgia\u201d, \u201cW\u201d denote \u201cweakling\u201d, \u201cA\u201d denote \u201cplaying volleyball\u201d. Then, we have the following:\n1. Dm \u2192 Maria is a dance.\n2. (\u2200x)(Wx \u2283 ~Dx). \u2192 For all x, if x is a weakling, then x is not a dancer. In other words, no weakling is a dancer.\n3. (\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax \u2192 For all x, x is a weakling or Georgia plays volleyball. So there exists an x that plays volleyball. \nOptions (A) and (C) do claim that some weaklings are not dancers, but the second argument strongly states that no weakling is a dancer. Thus, we can eliminate them. Option (B) omits the important detail about Georgia playing volleyball. Option (D) has all the details presented in the arguments and is the best English interpretation of the arguments. The answer is (D).\n\nQ: Select the best translation into predicate logic: No people drive on Mars.\n(A) ~Pd (B) (\u2200x)(Px \u2228 ~Dx) (C) (\u2200x)(Px \u2283 ~Dx) (D) ~Dp\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. Let \u201cP\u201d denote \u201cbeing on Mars\u201d and \u201cD\u201d denote \u201cdriving on Mars\u201d. Then let\u2019s consider each option:\nOption (A): ~Pd \u2192 d is not on Mars.\nOption (B): (\u2200x)(Px \u2228 ~Dx) \u2192 For all x, x is on Mars and x do not drive on Mars.\nOption (C): (\u2200x)(Px \u2283 ~Dx) \u2192 For all x, x is on Mars implies that x do not drive on Mars.\nOption (D): ~Dp: \u2192 p do not drive on Mars.\nOf all these options, Option (C) appears to be the best and most meaningful interpretation of the argument \u201cNo people drive on Mars.\u201d The answer is (C).\n\n", "global_facts": "The following are multiple choice questions (with answers) about global facts.\n\nQ: As of 2017, how many of the world\u2019s 1-year-old children today have been vaccinated against some disease? *\n(A) 80% (B) 60% (C) 40% (D) 20%\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. According to data published by the World Health Organization, the nummber of 1-year-old children vaccinated in 2017 exceeds 80%. The answer is (A).\n\nQ: As of 2019, about what percentage of Americans agree that the state is run for the benefit of all the people?\n(A) 31% (B) 46% (C) 61% (D) 76%\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. In 2019, about 46% percentage of Americans agree that the state is run for the benefit of all the people. The answer is (B).\n\nQ: As of 2019, about what percentage of Russians say it is very important to have free media in our country without government/state censorship?\n(A) 38% (B) 53% (C) 68% (D) 83%\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. As of 2019, about 38% of Russians say it is very important to have free media in our country. The answer is (A).\n\nQ: As of 2015, since 1990 forests have ____ in Europe and have ____ in Africa and the Americas.\n(A) increased, increased (B) increased, decreased (C) decreased, increased (D) decreased, decreased\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. As of 2015, since 1990 forests have increased in Europe and have decreased in Africa and the Americas. The answer is (B).\n\nQ: Which of the following pairs of statements are both true (as of 2019)?\n(A) People tend to be optimistic about their own future and the future of their nation or the world. (B) People tend to be optimistic about their own future but pessimistic about the future of their nation or the world. (C) People tend to be pessimistic about their own future but optimistic about the future of their nation or the world. (D) People tend to be pessimistic about their own future and the future of their nation or the world.\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. As of 2019, most people tend to be optimistic about their own future but pessimistic about the future of their nation or the world. The answer is (B).\n\n", "high_school_biology": "The following are multiple choice questions (with answers) about high school biology.\n\nQ: In animal cells, which of the following represents the most likely pathway that a secretory protein takes as it is synthesized in a cell?\n(A) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough ER (B) Ribosome\u2013Golgi apparatus\u2013rough ER\u2013secretory vesicle\u2013plasma membrane (C) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough ER (D) Ribosome\u2013rough ER\u2013Golgi apparatus\u2013secretory vesicle\u2013plasma membrane\nA: Let's think step by step. Protein synthesis starts at the ribosome, so we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum and moves from there to the Golgi apparatus, where it is modified and packaged into a vesicle. The vesicle then floats to the plasma membrane and is secreted. The answer is (D).\n\nQ: A mutation in a bacterial enzyme changed a previously polar amino acid into a nonpolar amino acid. This amino acid was located at a site distant from the enzyme\u2019s active site. How might this mutation alter the enzyme\u2019s substrate specificity?\n(A) By changing the enzyme\u2019s pH optimum (B) By changing the enzyme\u2019s location in the cell (C) By changing the shape of the protein (D) An amino acid change away from the active site cannot alter the enzyme\u2019s substrate specificity.\nA: Let's think step by step. A change in an amino acid leads to a change in the primary structure of the protein. A change in the primary structure may lead to a change in the secondary and the tertiary structure of the protein. A change in the tertiary structure means a change in the shape of the protein, so (C) has to be correct. Since the change does not affect the active site of the enzyme, we do not expect the activity of the enzyme to be affected. The answer is (C).\n\nQ: Which of the following is not a way to form recombinant DNA?\n(A) Translation (B) Conjugation (C) Specialized transduction (D) Transformation\nA: Let's think step by step. The introduction of foreign DNA or RNA into bacteria or eukaryotic cells is a common technique in molecular biology and scientific research. There are multiple ways foreign DNA can be introduced into cells including transformation, transduction, conjugation, and transfection. In contrast, (A) is not a way to form DNA: during translation the ribosomes synthesize proteins from RNA. The answer is (A).\n\nQ: Homologous structures are often cited as evidence for the process of natural selection. All of the following are examples of homologous structures EXCEPT\n(A) the wings of a bird and the wings of a bat (B) the flippers of a whale and the arms of a man (C) the pectoral fins of a porpoise and the flippers of a seal (D) the forelegs of an insect and the forelimbs of a dog\nA: Let's think step by step. \u200b\u200bHomologous structures are similar physical features in organisms that share a common ancestor \u200b\u200bbut different functions. Comparisons (B) and (C) are clearly homologous because they share a common ancestor and the structures serve different purposes. Bat wings and birg wings are also homologous, while they are both wings, the forelimbs serve different purposes. Insects and dogs are very far ancestors since one is vertebrate while the other is invertebrate and the forelimbs serve the same purpose, so they are not homologous. The answer is (D).\n\nQ: Which of the following is not known to be involved in the control of cell division?\n(A) Cyclins (B) Protein kinases (C) Checkpoints (D) Fibroblast cells\nA: Let's think step by step. Normal cells move through the cell cycle in a regulated way. At the checkpoint stage, they use information about their own internal state and cues from the environment around them to decide whether to proceed with cell division. Cues like these act by changing the activity of core cell cycle regulators inside the cell. The most common regulators are cyclins and cyclin-dependent kinases. Fibroblast cells do not play any role in cell division. The answer is (D).\n\n", "high_school_chemistry": "The following are multiple choice questions (with answers) about high school chemistry.\n\nQ: Which of the following is considered an acid anhydride?\n(A) HCl (B) H2SO3 (C) SO2 (D) Al(NO3)3\nA: Let's think step by step. An acid anhydride is a compound that is derived by removing water from an acid. The chemical formula for water is H2O, which means that we need to determine which of these options, when combined with H2O, forms an acid. SO2, or Sulfur dioxide, when combined with H2O, makes H2SO4, or sulfuric acid. The answer is (C).\n\nQ: Which of the following is expected to be a polar molecule?\n(A) PCl4F (B) BF3 (C) CO2 (D) Si(CH3)4\nA: Let's think step by step. A polar molecule is one that has a slightly positive charge on one end of the molecule and a slightly negative charge on the other end. Boron trifluoride (BF3) has Boron as the center atom and three fluorine atoms attached to it; it is trigonal planar and symmetric, so it is nonpolar. Carbon Dioxide (CO2) has Carbon as the central atom with double bonds to two Oxygen atoms - this is also symmetrical and therefore nonpolar. The same is the case for tetramethyl silane (SI(CH3)4), which is a Silicon atom surrounded by four methyl groups. The structure of PCL4F is that Phosphorus is the central atom, attached to four chlorines and one fluorine atom. This is asymmetrical, and therefore has a net dipole and is expected to be a polar molecule. The answer is (A).\n\nQ: From the solubility rules, which of the following is true?\n(A) All chlorides, bromides, and iodides are soluble (B) All sulfates are soluble (C) All hydroxides are soluble (D) All ammonium-containing compounds are soluble\nA: Let's think step by step. The chlorides, bromides, and iodides of lead, silver, and mercury are not soluble in water. This rules out (A). The sulfates of lead, barium, and calcium are not soluble in water, which rules out (B). The hydroxides of any metal besides sodium, potassium, ammonium, calcium, and barium are insoluble. This rules out (C). Typically ammonium ions indicate a soluble ionic substance. The answer is (D).\n\nQ: A new compound is synthesized and found to be a monoprotic acid with a molar mass of 248 g/mol. When 0.0050 mol of this acid are dissolved in 0.500 L of water, the pH is measured as 3.89. What is the pKa of this acid?\n(A) 3.89 (B) 7.78 (C) 5.78 (D) 2.33\nA: Let's think step by step. Recall that $[A] = [H^{+}]$. Here, this is equal to $$10^{-3.89}$. Then we have $K_{a} = $\nrac{[H^{+}][A^{-}]}{[HA]} = \nrac{10^{-3.89} \\cdot 10^{-3.89}}{10^{-2}}. The resulting exponent is $-3.89 + (-3.89) - (-2) = 5.78$, therefore $K_a = 10^{-5.78}$. The $pK_a$ is the negative log of $K_a$, which is equal to $5.78$. The answer is (C).\n\nQ: A solution contains 2.00 mole of acetic acid, CH3COOH, and 1.00 mole of calcium acetate, Ca(CH3COO)2. The solution is able to resist the addition of a small amount of strong acid or strong base with only minor changes in the pH of the solution. Larger quantities of strong acid or strong base can cause a significant change in pH. How many moles of nitric acid, HNO3, may be added before the pH begins to change significantly?\n(A) 0.500 mole (B) 1.00 mole (C) 2.00 mole (D) 3.00 mole\nA: Let's think step by step. We would like to compute the buffer capacity of this solution. First we write the equation for the ionization of the weak acid, in this case of acetic acid. $CH_{3}COOH (aq) + H_{2}O \nightarrow H_{3}O^{+} + CH3COO^{-}$. The conjugate base is therefore the acetate ion. The added strong acid, Nitric acid, will react with the conjugate base. Therefore the maximum amount of acid that can be added will be equal to the amount of acetate ion, or 2 moles. The answer is (C).\n\n", "high_school_computer_science": "The following are multiple choice questions (with answers) about high school computer science.\n\nQ: Which of the following is an example of the use of a device on the Internet of Things (IoT) ?\n(A) A car alerts a driver that it is about to hit an object. (B) A hiker uses a G P S watch to keep track of her position. (C) A refrigerator orders milk from an online delivery service when the milk in the refrigerator is almost gone. (D) A runner uses a watch with optical sensors to monitor his heart rate.\nA: Let's think step by step. The term Internet of Things (IoT) refers to common devices which are connected to the internet, enabling new functionality. Choice A is incorrect because it does not describe an internet connected device. In choice B, the watch is only described as having GPS functionality but no internet connectivity. Choice C describes a common device (a refrigerator) which has internet connectivity enabling new functionality (online ordering). Choice D does not mention internet connectivity for the watch, only optical sensors. The answer is (C).\n\nQ: Many Web browsers allow users to open anonymous windows. During a browsing session in an anonymous window, the browser does not record a browsing history or a list of downloaded files. When the anonymous window is exited, cookies created during the session are deleted. Which of the following statements about browsing sessions in an anonymous window is true?\n(A) The activities of a user browsing in an anonymous window will not be visible to people who monitor the user's network, such as the system administrator. (B) Items placed in a Web store's shopping cart for future purchase during the anonymous browsing session will not be saved on the user's computer. (C) A user will not be able to log in to e-mail or social media accounts during the anonymous browsing session. (D) A user browsing in an anonymous window will be protected from viruses launched from any web sites visited or files downloaded.\nA: Let's think step by step. Choice A is incorrect as it only describes network traffic, which an anonymous browser does not change. Choice B is correct as it correctly describes how an anonymous browser will prevent saving data on the user\u2019s computer after the session is ended. Choice C is incorrect because an anonymous browser will not prevent logging in to email or social media accounts. Choice D is incorrect because an anonymous browser in itself performs no virus protection. The answer is (B).\n\nQ: In the program below, the initial value of X is 5 and the initial value of Y is 10.\nIF (X < 0){\n DISPLAY (\"Foxtrot\")\n} ELSE {\n IF (X > Y){\n DISPLAY (\"Hotel\")\n } ELSE {\n IF (Y > 0){\n DISPLAY (\"November\")\n } ELSE {\n DISPLAY (\"Yankee\")\n }\n }\n}\nWhat is displayed as a result of running the program?\n(A) Foxtrot (B) Hotel (C) November (D) Yankee\nA: Let's think step by step. Because X has the value 5, the first conditional IF (X < 0) is false, so we move to the first ELSE clause. Because X is 5 and Y is 10, the second conditional IF (X > Y) is false, so we move to the following ELSE clause. Since Y is 10, the conditional IF (Y > 0) is true, so the command DISPLAY (\"November\") is executed. The answer is (C).\n\nQ: What is the output of \"abc\"[::-1] in Python 3?\n(A) Error (B) abc (C) cba (D) c\nA: Let's think step by step. We know that the slicing operator [::-1] takes all of the elements in the string in reverse order, so we reverse the order of the string \"abc\", resulting in \"cba\". The answer is (C).\n\nQ: A list of numbers has n elements, indexed from 1 to n. The following algorithm is intended to display the number of elements in the list that have a value greater than 100. The algorithm uses the variables count and position. Steps 3 and 4 are missing.\n Step 1: Set count to 0 and position to 1.\n Step 2: If the value of the element at index position is greater than 100, increase the value of count by 1.\n Step 3: (missing step)\n Step 4: (missing step)\n Step 5: Display the value of count.\nWhich of the following could be used to replace steps 3 and 4 so that the algorithm works as intended?\n(A) Step 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until the value of count is greater than 100.\n(B) Step 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until the value of position is greater than n.\n(C) Step 3: Repeat step 2 until the value of count is greater than 100.\n Step 4: Increase the value of position by 1.\n(D) Step 3: Repeat step 2 until the value of position is greater than n.\n Step 4: Increase the value of count by 1.\nA: Let's think step by step. Choice A is incorrect, because its Step 4 has an incorrect termination condition, stopping when count is greater than 100. We need to stop after inspecting all elements in the list. Choice B is correct because it correctly increments both count and position, and correctly repeats these steps and terminates when all elements in the list have been inspected. Choice C is incorrect because it incorrectly increments the variable count until its value is greater than 100, regardless of the elements in the list. Choice D is incorrect because its step 3 does not increment the value of position, so it will repeat forever. The answer is (B).\n\n", "high_school_european_history": "The following are multiple choice questions (with answers) about high school european history.\n\nQ: This question refers to the following information.\nAlbeit the king's Majesty justly and rightfully is and ought to be the supreme head of the Church of England, and so is recognized by the clergy of this realm in their convocations, yet nevertheless, for corroboration and confirmation thereof, and for increase of virtue in Christ's religion within this realm of England, and to repress and extirpate all errors, heresies, and other enormities and abuses heretofore used in the same, be it enacted, by authority of this present Parliament, that the king, our sovereign lord, his heirs and successors, kings of this realm, shall be taken, accepted, and reputed the only supreme head in earth of the Church of England, called Anglicans Ecclesia; and shall have and enjoy, annexed and united to the imperial crown of this realm, as well the title and style thereof, as all honors, dignities, preeminences, jurisdictions, privileges, authorities, immunities, profits, and commodities to the said dignity of the supreme head of the same Church belonging and appertaining; and that our said sovereign lord, his heirs and successors, kings of this realm, shall have full power and authority from time to time to visit, repress, redress, record, order, correct, restrain, and amend all such errors, heresies, abuses, offenses, contempts, and enormities, whatsoever they be, which by any manner of spiritual authority or jurisdiction ought or may lawfully be reformed, repressed, ordered, redressed, corrected, restrained, or amended, most to the pleasure of Almighty God, the increase of virtue in Christ's religion, and for the conservation of the peace, unity, and tranquility of this realm; any usage, foreign land, foreign authority, prescription, or any other thing or things to the contrary hereof notwithstanding.\nEnglish Parliament, Act of Supremacy, 1534\nFrom the passage, one may infer that the English Parliament wished to argue that the Act of Supremacy would\n(A) give the English king a new position of authority (B) give the position of head of the Church of England to Henry VIII alone and exclude his heirs (C) establish Calvinism as the one true theology in England (D) end various forms of corruption plaguing the Church in England\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The Act of Supremacy states that it grants authority to the king \"to repress and extirpate all errors, heresies, and other enormities and abuses\", referring to the corruption in the Church of England. The answer is (D).\n\nQ: This question refers to the following information.\nRead the following excerpt.\nThe revolutionary seed had penetrated into every country and spread more or less. It was greatly developed under the r\u00e9gime of the military despotism of Bonaparte. His conquests displaced a number of laws, institutions, and customs; broke through bonds sacred among all nations, strong enough to resist time itself; which is more than can be said of certain benefits conferred by these innovators.\nThe monarchs will fulfil the duties imposed upon them by Him who, by entrusting them with power, has charged them to watch over the maintenance of justice, and the rights of all, to avoid the paths of error, and tread firmly in the way of truth. Placed beyond the passions which agitate society, it is in days of trial chiefly that they are called upon to despoil realities of their false appearances, and to show themselves as they are, fathers invested with the authority belonging by right to the heads of families, to prove that, in days of mourning, they know how to be just, wise, and therefore strong, and that they will not abandon the people whom they ought to govern to be the sport of factions, to error and its consequences, which must involve the loss of society.\nUnion between the monarchs is the basis of the policy which must now be followed to save society from total ruin. . . .\nLet them not confound concessions made to parties with the good they ought to do for their people, in modifying, according to their recognized needs, such branches of the administration as require it.\nLet them be just, but strong; beneficent, but strict.\nLet them maintain religious principles in all their purity, and not allow the faith to be attacked and morality interpreted according to the social contract or the visions of foolish sectarians.\nLet them suppress Secret Societies; that gangrene of society.\n\u2014Klemens von Metternich, Political Confession of Faith, 1820\nWhich of the following was the greatest cause of the fears expressed by Metternich in the document above?\n(A) The ideas of personal liberty and nationalism conceived during the Enlightenment resulted in radical revolutions that could spread throughout Europe. (B) The conquest of Europe by Napoleon led to the creation of new factions and shifted the European balance of power. (C) The power of monarchs had grown to the point where it needed to be checked by other powers within each nation or domination of civilians would occur. (D) The rising and falling economic cycle of the newly emerging capitalist economy could lead to civilian unrest that must be suppressed.\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The fears of revolution in early 19th century Europe expressed by Klemens von Metternich, a conservative Austrian statesman, were a direct result of the age of Enlightenment, a period of European history where the absolute power of the monarchy was challenged with ideas of individual liberty and nationalism, leading to the French revolution and its effects all over Europe. The answer is (A).\n\nQ: This question refers to the following information.\nThe excerpts below are from the Navigation Acts of 1651.\n[A]fter the first day of December, one thousand six hundred fifty and one, and from thence forwards, no goods or commodities whatsoever of the growth, production or manufacture of Asia, Africa or America, or of any part thereof; or of any islands belonging to them, or which are described or laid down in the usual maps or cards of those places, as well of the English plantations as others, shall be imported or brought into this Commonwealth of England, or into Ireland, or any other lands, islands, plantations, or territories to this Commonwealth belonging, or in their possession, in any other ship or ships, vessel or vessels whatsoever, but only in such as do truly and without fraud belong only to the people of this Commonwealth, or the plantations thereof, as the proprietors or right owners thereof; and whereof the master and mariners are also of the people of this Commonwealth, under the penalty of the forfeiture and loss of all the goods that shall be imported contrary to this act, , , ,\n[N]o goods or commodities of the growth, production, or manufacture of Europe, or of any part thereof, shall after the first day of December, one thousand six hundred fifty and one, be imported or brought into this Commonwealth of England, or any other lands or territories to this Commonwealth belonging, or in their possession, in any ship or ships, vessel or vessels whatsoever, but in such as do truly and without fraud belong only to the people of this Commonwealth, and in no other, except only such foreign ships and vessels as do truly and properly belong to the people of that country or place, of which the said goods are the growth, production or manufacture.\nWhich of the following best describes the outcome of the Navigation Acts of 1651?\n(A) They served as a catalyst for the growth of English shipping and overseas trade, but did little to limit the prospects of the Dutch in the seventeenth century. (B) They brought about almost immediate hardships for the Dutch economy as their dominance of overseas trade quickly ended. (C) They were rescinded during the restoration of the Stuarts as they sought normal diplomatic relations with the Dutch so not as to need Parliament's financial support for war. (D) They led to nearly a century of recurrent war between England and the Netherlands, which would not end until after American independence.\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The Navigation Acts of 1651 helped English shipping by restricting the ability of ships from other European countries, especially the Dutch, to transport goods from colonies in Asia and Africa into England. The answer is (A).\n\nQ: This question refers to the following information.\nIn Russia there was nothing going on well, and [Souvarine] was in despair over the news he had received. His old companions were all turning to the politicians; the famous Nihilists who made Europe tremble-sons of village priests, of the lower middle class, of tradesmen-could not rise above the idea of national liberation, and seemed to believe that the world would be delivered-when they had killed their despot&\u2026\n\"Foolery! They'll never get out of it with their foolery.\"\nThen, lowering his voice still more, in a few bitter words he described his old dream of fraternity. He had renounced his rank and his fortune; he had gone among workmen, only in the hope of seeing at last the foundation of a new society of labour in common. All the sous in his pockets had long gone to the urchins of the settlement; he had been as tender as a brother with the colliers, smiling at their suspicion, winning them over by his quiet workmanlike ways and his dislike of chattering. But decidedly the fusion had not taken place.\nHis voice changed, his eyes grew bright, he fixed them on \u00e9tienne, directly addressing him:\n\"Now, do you understand that? These hatworkers at Marseilles who have won the great lottery prize of a hundred thousand francs have gone off at once and invested it, declaring that they are going to live without doing anything! Yes, that is your idea, all of you French workmen; you want to unearth a treasure in order to devour it alone afterwards in some lazy, selfish corner. You may cry out as much as you like against the rich, you haven't got courage enough to give back to the poor the money that luck brings you. You will never be worthy of happiness as long as you own anything, and your hatred of the bourgeois proceeds solely from an angry desire to be bourgeois yourselves in their place.\"\n\u00e9mile Zola, French writer, Germinal, 1885\nThe passage displays the direct concern for the welfare of the working classes that was typically a part of which movement?\n(A) Capitalist (B) Scientific (C) Communist (D) Existentialist\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The modern Communist movement aims to establish a classless society based on communal ownership and distribution of property and means of production, thereby especially benefiting the working classes. The answer is (C).\n\nQ: This question refers to the following information.\nThe following excerpt is from a pamphlet.\nYou will do me the justice to remember, that I have always strenuously supported the Right of every man to his own opinion, however different that opinion might be to mine. He who denies to another this right, makes a slave of himself to his present opinion, because he precludes himself the right of changing it.\nThe most formidable weapon against errors of every kind is Reason. I have never used any other, and I trust I never shall.\nThe circumstance that has now taken place in France of the total abolition of the whole national order of priesthood, and of everything appertaining to compulsive systems of religion, and compulsive articles of faith, has not only precipitated my intention, but rendered a work of this kind exceedingly necessary, lest in the general wreck of superstition, of false systems of government, and false theology, we lose sight of morality, of humanity, and of the theology that is true.\nI believe in one God, and no more; and I hope for happiness beyond this life.\nI believe in the equality of man; and I believe that religious duties consist in doing justice, loving mercy, and endeavoring to make our fellow-creatures happy.\nI do not believe in the creed professed by the Jewish church, by the Roman church, by the Greek church, by the Turkish church, by the Protestant church, nor by any church that I know of. My own mind is my own church.\nAll national institutions of churches, whether Jewish, Christian or Turkish, appear to me no other than human inventions, set up to terrify and enslave mankind, and monopolize power and profit.\nI do not mean by this declaration to condemn those who believe otherwise; they have the same right to their belief as I have to mine.\n\u2014Thomas Paine, The Age of Reason, 1794\u20131795\nWhich of the following Enlightenment philosophes designed a system of checks and balances for government to avoid abuses of power?\n(A) Jean Jacques Rousseau (B) Baron Montesquieu (C) Mary Wollstonecraft (D) Adam Smith\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. Baron Montesquieu was a 18th centrury French philsopher who wrote extensively against the monoplization of power and advocated for a system of checks and balances in government to prevent the rise of despotism. The answer is (B).\n\n", "high_school_geography": "The following are multiple choice questions (with answers) about high school geography.\n\nQ: Which one of the following items is an example of nonmaterial culture?\n(A) Dove soap (B) Dove candy bar (C) Dove symbol (D) A dove (bird).\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. Nonmaterial culture consists of cultural ideas, beliefs or symbols that are not physical objects. The answer is (C).\n\nQ: During the third stage of the demographic transition model, which of the following is true?\n(A) Birth rates increase and population growth rate is less rapid. (B) Birth rates decline and population growth rate is less rapid. (C) Birth rates increase and population growth rate increases. (D) Birth rates decrease and population growth rate increases.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. The demographic transition model models the five different stages of population growth as a country goes through economic development, where the third stage refers to a period of declining birth rates and lower population growth. The answer is (B).\n\nQ: The practice of hiring a foreign third-party service provider to run an operation is called\n(A) outsourcing. (B) offshoring. (C) maquiladoras. (D) locational interdependence.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. \"Offshoring\" literally means to move or base some of the activities or processes of a company to a foreign country. The answer is (B).\n\nQ: Which of the following statements is NOT accurate regarding the services provided by local governments in the United States?\n(A) Duplication of efforts occurs often. (B) Social problems of the central city spill over into the surrounding residential suburbs. (C) Inefficiency in providing services occurs often. (D) One neighborhood's efforts to reduce pollution are always supported by neighboring communities.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. There may be economic, social or political reasons for two neighboring communities and their local governments not agreeing to pollution reduction efforts initiated by one of them. The answer is (D).\n\nQ: The rate of natural increase of a population is found by subtracting the\n(A) crude death rate from the crude birth date. (B) crude birth rate from the crude death rate. (C) doubling time from the crude birth rate. (D) fertility rate from the crude death rate.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. The difference between number of births and deaths gives the population increase at any given time. The answer is (A).\n\n", "high_school_government_and_politics": "The following are multiple choice questions (with answers) about high school government and politics.\n\nQ: Which of the following best states an argument made by James Madison in The Federalist number 10?\n(A) Honest politicians can prevent factions from developing. (B) Factions are more likely to occur in large republics than in small ones. (C) The negative effects of factionalism can be reduced by a republican government. (D) Free elections are the people's best defense against factionalism.\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. In the Federalist number 10, James Madison advocated for a representative republican form of government to guard against factionalism. The answer is (C).\n\nQ: The term \"budget deficit\" refers to the\n(A) annual increase in federal spending on the military (B) amount of interest on the national debt (C) difference between the initial budget proposals made by the president and Congress (D) amount the government spends in excess of its revenues\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. When the goverment spends more than it earns, their difference is the budget deficit. The answer is (D).\n\nQ: Which of the following statements about cabinet departments is FALSE?\n(A) They are established by the legislative branch. (B) Their members often don't have much influence over presidential decisions. (C) They cannot all be run by leaders who belong to the same political party the president does. (D) Not every federal agency is a cabinet department.\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. There is no law stipulating that some cabinet department leaders have to belong to a political party different from that of the president. The answer is (C).\n\nQ: Which of the following cases established the precedent that a defendant must be informed of the right to remain silent, the right to a lawyer, and protection from self-incrimination?\n(A) Weeks v. United States (B) Betts v. Brady (C) Mapp v. Ohio (D) Miranda v. Arizona\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. In the landmark Miranda v. Arizona in 1966, the US Supreme Court, based on the Fifth and Sixth Amendment of the US Constitution, guaranteed a defendant's right to an attorney and protection from self-incrimination. The answer is (D).\n\nQ: Uncertainty over the limits to presidential power is caused primarily by the fact that\n(A) the constitutional definition of those powers is broad and unspecific (B) most people agree that the Constitution places too many limits on presidential power (C) the Supreme Court consistently refuses to rule on cases concerning presidential powers (D) constitutional amendments have greatly increased presidential powers\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. The US Constitution is not very specific about the powers of the president, leading to uncertainty over its limits. The answer is (A).\n\n", "high_school_macroeconomics": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\nQ: Which of the following policies best describes supply-side fiscal policy?\n(A) An increase in the money supply (B) Increased government spending (C) Lower taxes on research and development of new technology (D) Higher taxes on household income\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. Supply-side fiscal policy stimulates the economy by encouraging more production of goods and services through reduction in taxes and deregulation. The answer is (C).\n\nQ: The short-run Phillips curve indicates a\n(A) direct relation between unemployment and inflation (B) direct relation between price and quantity demanded (C) inverse relation between price and quantity demanded (D) inverse relation between unemployment and inflation\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. The short-run Phillips curve shows that whenever unemployment decreases below a natural level, the inflation starts increasing, and vice-versa. The answer is (D).\n\nQ: Holding all else equal which of the following monetary policies would be used to boost U.S. exports?\n(A) Increasing the discount rate (B) Increasing the reserve ratio (C) Buying government securities (D) Lowering tariffs\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. Buying government securities leads to reduction in demand for US dollars from foreign buyers, thereby making it cheaper and hence making US exports more attractive. The answer is (C).\n\nQ: A federal deficit occurs when\n(A) exports exceed imports. (B) imports exceed exports. (C) federal tax collections exceed spending. (D) federal spending exceeds federal tax revenues.\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. A federal deficit occurs when federal spending exceeds federal income which is primarily from tax revenues. The answer is (D).\n\nQ: Which of the following is not included in the U.S. GDP?\n(A) The U.S. military opens a new base in a foreign country with 1000 U.S. personnel. (B) Japanese consumers buy thousands of CDs produced in the United States. (C) An American pop singer performs a sold-out concert in Paris. (D) A French theatrical production tours dozens of American cities.\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. The economic transactions related to the performance of the American pop-singer in Paris happens entirely outside the U.S. and hence is not included in the GDP numbers. The answer is (C).\n\n", "high_school_mathematics": "The following are multiple choice questions (with answers) about high school mathematics.\n\nQ: Simplify and write the result with a rational denominator: $$\\sqrt{\\sqrt[3]{\\sqrt{\\frac{1}{729}}}}$$\n(A) \\frac{3\\sqrt{3}}{3} (B) \\frac{1}{3} (C) \\sqrt{3} (D) \\frac{\\sqrt{3}}{3}\nA: Let's think step by step. Factoring $729=3^6$ and combining the roots $\\frac{1}{2}\\frac{1}{3}\\frac{1}{2}=\\frac{1}{12}$, we get that $\\sqrt{\\sqrt[3]{\\sqrt{\\frac{1}{729}}}}=\\left(\\frac{1}{3^6}\\right)^{\\frac{1}{12}}=\\frac{1}{3^{\\frac{1}{2}}}=\\frac{3}{\\sqrt{3}}$ The answer is (D).\n\nQ: Five thousand dollars compounded annually at an $x\\%$ interest rate takes six years to double. At the same interest rate, how many years will it take $\\$300$ to grow to $\\$9600$?\n(A) 12 (B) 1 (C) 30 (D) 5\nA: Let's think step by step. To go from $\\$300$ to $\\$9600$, the value must go up by a factor of $9600/300=32=2^5$. Since at this interest rate it takes six years for it to double, it will take $5*6=30$ years to grow to $\\$9600$. The answer is (C).\n\nQ: Ten students take a biology test and receive the following scores: 45, 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students\u2019 test scores?\n(A) 55 (B) 60 (C) 62 (D) 65\nA: Let's think step by step. There are 10 students and the sum of their scores is $45 + 55 + 50 + 70 + 65 + 80 + 40 + 90 + 70 + 85 = 650$, the mean is $650/10=65$. The answer is (D).\n\nQ: The variable $x$ varies directly as the square of $y$, and $y$ varies directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is the value of $x$ when $z$ equals $\\frac{1}{2}$?\n(A) -1 (B) 16 (C) -\\frac{1}{256} (D) \\frac{1}{16}\nA: Let's think step by step. We know that $x \\propto y^2$ and $y \\propto z^3$, so $x = k z^6$ for some constant $k$. Plugging in for $x=-16$ and $z=2$, the constant value is $k=\\frac{x}{z^6}=\\frac{-16}{64}=-\\frac{1}{4}$. So, when $z=\\frac{1}{2}$, the value of $x$ is $x=kz^6=-\\frac{1}{4}\\frac{1}{2^6}=-\\frac{1}{256}$. The answer is (C).\n\nQ: Joe was in charge of lights for a dance. The red light blinks every two seconds, the yellow light every three seconds, and the blue light every five seconds. If we include the very beginning and very end of the dance, how many times during a seven minute dance will all the lights come on at the same time? (Assume that all three lights blink simultaneously at the very beginning of the dance.)\n(A) 3 (B) 15 (C) 6 (D) 5\nA: Let's think step by step. The least common multiple of 2, 3 and 5 is 30, so during a 7 minute dance, all the three lights will come on at the same time $2*7+1=15$ times. The answer is (B).\n\n", "high_school_microeconomics": "The following are multiple choice questions (with answers) about high school microeconomics.\n\nQ: Which of the following is necessarily a characteristic of oligopoly?\n(A) Free entry into and exit from the market (B) A few large producers (C) One producer of a good with no close substitutes (D) A homogenous product\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. An oligopoly is when a market is dominated by just one or a few number of sellers or producers. To get oligopoly, the market should have high barriers to new entry, and the product has differentiation. The answer is (B).\n\nQ: If the government subsidizes producers in a perfectly competitive market, then\n(A) the demand for the product will increase (B) the demand for the product will decrease (C) the consumer surplus will increase (D) the consumer surplus will decrease\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. (A) and (B) are wrong because the demand curve does not change at all. If the government subsidizes producers, the supply will increase, and thus the consumer surplus also increases. The answer is (C).\n\nQ: Which of the following is true of a price floor?\n(A) The price floor shifts the demand curve to the left. (B) An effective floor creates a shortage of the good. (C) The price floor shifts the supply curve of the good to the right. (D) To be an effective floor, it must be set above the equilibrium price.\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. Price floor does not shift the demand or shift curve. An effective price floor should be set above the equilibrium price, otherwise the market bears and the floor does not have effective effect. The answer is (D).\n\nQ: The concentration ratio for a monopoly is\n(A) 0 (B) 5 (C) 10 (D) 100\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. The concentration ratio is calculated as the sum of market share of a specific number of largest companies. Monopoly means one company or entity controls the entire market, therefore, the concentration ratio is 100 percent. The answer is (D).\n\nQ: In a competitive labor market for housepainters, which of the following would increase the demand for housepainters?\n(A) An effective minimum wage imposed on this labor market. (B) An increase in the price of gallons of paint. (C) An increase in the construction of new houses. (D) An increase in the price of mechanical painters so long as the output effect exceeds the substitution effect.\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. An increase in the construction of new houses means an increase demand of in-house painting, thus increases the demand for housepainters. The answer is (C).\n\n", "high_school_physics": "The following are multiple choice questions (with answers) about high school physics.\n\nQ: A microwave oven is connected to an outlet, 120 V, and draws a current of 2 amps. At what rate is energy being used by the microwave oven?\n(A) 10 W (B) 30 W (C) 60 W (D) 240 W\nA: Let's think step by step. Rate of energy usage is known as power; in an dissipative electrical circuit, power is given by voltage times current. So in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D).\n\nQ: A point charge, Q = +1 mC, is fixed at the origin. How much work is required to move a charge, Q = +8 \u00b5C, from the point (0, 4 meters) to the point (3 meters, 0)?\n(A) 3.5 J (B) 6.0 J (C) 22.5 J (D) 40 J\nA: Let's think step by step. To calculate the work required to move a charge from one location to another in a fixed electric field, it is enough to calculate the potential difference between the two locations. Here, the potential only depends on the distance between the charges; it\u2019s $k q_1 q_2 / r$, where $k$ is Coulomb\u2019s constant. Plugging in values $q_1 = $ 1 mC, $q_2 = 8 \\mu$ C, gives the answer as 5.992 J, which rounds to 6 J. The answer is (B).\n\nQ: Which of the following conditions will ensure that angular momentum is conserved? I. Conservation of linear momentum II. Zero net external force III. Zero net external torque\n(A) I and II only (B) I and III only (C) II and III only (D) III only\nA: Let's think step by step. Torque is defined as the change in angular momentum; if there is zero external torque, angular momentum is conserved. The answer is (D).\n\nQ: A photocell of work function \u03d5 = 2eV is connected to a resistor in series. Light of frequency f = 1 \u00d7 10^15 Hz hits a metal plate of the photocell. If the power of the light is P = 100 W, what is the current through the resistor?\n(A) 2:00 AM (B) 6:00 AM (C) 12:00 AM (D) 24 A\nA: Let's think step by step. The only answer above which has units of current is D, 24 A. The answer is (D).\n\nQ: A pipe full of air is closed at one end. A standing wave is produced in the pipe, causing the pipe to sound a note. Which of the following is a correct statement about the wave\u2019s properties at the closed end of the pipe?\n(A) The pressure is at a node, but the particle displacement is at an antinode. (B) The pressure is at an antinode, but the particle displacement is at a node. (C) The pressure and the particle displacement are both at nodes. (D) The pressure and the particle displacement are both at antinodes.\nA: Let's think step by step. At the closed end of the pipe, the particles cannot have any net displacement because the pipe closure stops them. So the particle displacement is at a node. This closure also causes the pressure to be maximal, i.e. an antinode. The answer is (B).\n\n", "high_school_psychology": "The following are multiple choice questions (with answers) about high school psychology.\n\nQ: Pascale is interested in the processing strategies children use to learn new information. Pascale would best be classified as what type of psychologist?\n(A) sociocultural (B) clinical (C) cognitive (D) behaviorist\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Sociocultural psychologist focuses on the effect of societal factors on people. Clinical psychologist focuses on people with mental issues. Cognitive psychologist focuses on how people think and learn, including the processing strategies. Behaviorist focuses more on the environment and experience effect on people. The answer is (C).\n\nQ: According to Caplan's model of consultee-centered case consultation, the consultant is primarily interested in\n(A) identifying the causes and solutions of the client's presenting problems (B) identifying and eliminating the causes of the consultee's difficulties in handling a problem (C) establishing a hierarchy of authority to enable effective decision making (D) presenting a single, well-defined and unambiguous course of action for the consultant to overcome skills deficits\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Caplan defines two type of consultation. Client-centered case consultation aims to handle client's problems, while consultee-centered case consultation aims to identify the reason of client's difficulty to solve problems. The answer is (B).\n\nQ: According to the Individuals with Disabilities Education Improvement Act, which of the following must an educational agency do before it changes the educational placement of a student with a disability?\n(A) Give the child a trial period in the new environment (B) Notify the parents in writing (C) Obtain school board approval (D) Obtain parental consent\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. When the decision to change the educational placement of a student with a disability is made, the educational agency must notify the parents in writing on that date. The answer is (B).\n\nQ: While swimming in the ocean, Ivan is frightened by a dark shadow in the water even before he has the chance to identify what the shadow is. The synaptic connections taking place during this incident of fright are best described by which of the following?\n(A) Messages are sent from the thalamus directly to the amygdala. (B) Messages are sent from the thalamus to the \"what\" and \"where\" pathways. (C) Messages are sent from the parasympathetic nervous system to the cerebral cortex. (D) Messages are sent from the frontal lobes to the pituitary gland.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Our neural system has a mechanism that can respond immediate emotional signal before going to the thought center. In the Ivan's case, messages travel directly from thalamus to amygdala. The answer is (A).\n\nQ: Ani believes that her attitudes and behavior play a central role in what happens to her. Such a belief is likely to be associated with\n(A) a strong superego. (B) low self-esteem. (C) low self-efficacy. (D) an internal locus of control.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. People with an external locus of control believes fate and luck play an important role in their lives, while people with an internal locus of control believes they control their lives. The answer is (D).\n\n", "high_school_statistics": "The following are multiple choice questions (with answers) about high school statistics.\n\nQ: A new smartwatch is manufactured in one part of a factory, then secured for shipping in another, independent part of the factory. The weight of the smartwatch has a mean of 62 grams and a standard deviation of 1.0 grams. The weight of the packaging (box, user's guide, bubble wrap, etc.) has a mean of 456 grams and a standard deviation of 6 grams. Together, the distribution of the weight of the smartwatch and its packaging would have the following mean and standard deviation:\n(A) Mean 518 grams; standard deviation 7.0 grams (B) Mean 518 grams; standard deviation 3.5 grams (C) Mean 518 grams; standard deviation 6.1 grams (D) Mean 394 grams; standard deviation 6.1 grams\nA: Let's think step by step. Since the weight of the watch and the weight of the packaging are independent random variables, the mean and variance of their sum is equal to the sum of their individual means and variances. So the mean is 62 + 456 = 518 grams, and the variances is 1.0^2 + 6.0^2 = 37, leading to a standard deviation of 6.1 grams. The answer is (C).\n\nQ: After a frost warning was issued, the owner of a large orange grove asked his workers to spray all his trees with water. The water was supposed to freeze and form a protective covering of ice around the orange blossom. Nevertheless, the owner suspected that some trees suffered considerable damage due to the frost. To estimate the proportion of trees that suffered more than 50 percent damage due to the frost, he took a random sample of 100 trees from his grove. What is the response variable in this experiment?\n(A) The proportion of trees that suffered more than 50 percent damage due to frost. (B) The number of trees affected by the frost. (C) The number of trees sampled from the grove. (D) For each sampled tree, whether it suffered more than 50 percent damage or at most 50 percent damage.\nA: Let's think step by step. In this experiment, the response variable is what is measured. For each tree, what is measured is whether or not it suffered more than 50 percent damage due to the frost. The answer is (D).\n\nQ: Suppose X and Y are random variables with E(X) = 37, var(X) = 5, E(Y) = 62, and var(Y) = 12. What are the expected value and variance of the random variable X + Y?\n(A) E(X + Y) = 99, var(X + Y) = 8.5 (B) E(X + Y) = 99, var(X + Y) = 13 (C) E(X + Y) = 99, var(X + Y) = 17 (D) There is insufficient information to answer this question.\nA: Let's think step by step. While means of sums of random variables add (regardless of whether the variables are independent) in order to determine the variance of a sum of random variables, we need to know not just their individual variances but the covariance of the two variables, which is not given in this problem. The answer is (D).\n\nQ: Which of the following sets has the smallest standard deviation? Which has the largest?\nI: {1,2,3}\nII: {-10,10}\nIII: {100}\n(A) I, II (B) II, III (C) III, I (D) III, II\nA: Let's think step by step. The variance of distribution I is the expected squared deviation from its mean (which is 2), so the variance is 2/3 . The variance of distribution II is 10^2 (because both elements are 10 away from the mean of zero). The variance of distribution III is 0, since it has a single entry. So distribution III has the smallest standard deviation and distribution II has the largest. The answer is (D).\n\nQ: Which of the following is a correct statement about correlation?\n(A) If the slope of the regression line is exactly 1, then the correlation is exactly 1. (B) If the correlation is 0, then the slope of the regression line is undefined. (C) Switching which variable is called x and which is called y changes the sign of the correlation. (D) The correlation r is equal to the slope of the regression line when z-scores for the y-variable are plotted against z-scores for the x-variable.\nA: Let's think step by step. Statement A is false because the slope of the regression line being exactly 1 can occur even when the two variables are not perfectly correlated. Statement B is false because uncorrelated variables regression lines can have slope zero. Statement C is false because correlation is symmetric in the two random variables. The answer is (D).\n\n", "high_school_us_history": "The following are multiple choice questions (with answers) about high school us history.\n\nQ: This question refers to the following information.\nI come not to urge personal claims, nor to seek individual benefits; I appear as the advocate of those who cannot plead their own cause; I come as the friend of those who are deserted, oppressed, and desolate. In the Providence of God, I am the voice of the maniac whose piercing cries from the dreary dungeons of your jails penetrate not your Halls of Legislation. I am the Hope of the poor crazed beings who pine in the cells, and stalls, and cages, and waste rooms of your poor-houses. I am the Revelation of hundreds of wailing, suffering creatures, hidden in your private dwellings, and in pens and cabins\u2014shut out, cut off from all healing influences, from all mind-restoring cares.\u2026 Could their melancholy histories be spread before you as revealed to my grieved spirit during the last three months, how promptly, how earnestly would you search out the most approved means of relief; how trifling, how insignificant, by comparison, would appear the sacrifices you are asked to make; how would a few dimes and dollars, gathered from each citizen, diminish in value as a possession, compared with the certain benefits and vast good to be secured for the suffering insane...by the consecration and application of a sufficient fund to the construction of a suitable hospital.\u2026\n\u2014Dorothea Dix, Memorial Soliciting a State Hospital for the Protection and Cure of the Insane,\nSubmitted to the General Assembly of North Carolina, November 1848\nDorothea Dix can best be compared to whom?\n(A) Abigail Adams (B) Clara Barton (C) Shirley Temple (D) Hillary Clinton\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. Both Dorothea Dix and Clara barton are American nurses. The answer is (B).\n\nQ: This question refers to the following information.\n\"As our late Conduct at the Conestoga Manor and Lancaster have occasioned much Speculation & a great diversity of Sentiments in this and neighboring Governments; some vindicating & others condemning it; some charitably alleviating the Crime, & others maliciously painting it in the most odious & detestable Colours, we think it our duty to lay before the Publick, the whole Matter as it appeared, & still appears, to us. . . .\n\"If these things are not sufficient to prove an unjustifiable Attachment in the Quakers to the Indians Savages, a fixed Resolution to befriend them & an utter insensibility to human Distresses, let us consider a few more recent Facts. When we found the last Summer that we were likely to get no Assistance from the Government, some Volunteers went out at our own Expense, determined to drive our Enemies from our Borders; & when we came near to the great Island, we understood that a Number of their Warriors had gone out against our Frontiers. Upon this we returned and came up with them and fought with them at the Munfey Hill where we lost some of our Men & killed some of their Warriors & thereby saved our Frontiers from this Story in another Expedition. But no sooner had we destroyed their Provisions on the great Island, & ruined their trade with the good People at Bethlehem, but these very Indians, who were justly suspected of having murdered our Friends in Northampton County, were by the Influence of some Quakers taken under the Protection of the Government to screen them from the Resentments of the Friends and Relations of the Murdered, & to support them thro the Winter.\"\n\u2014\"Apology of the Paxton Boys\" (pamphlet), 1764 (Note: \"apology\" in this context should be read as an explanation, not an admission of guilt or regret.\nThe sentiments expressed in the explanation above reflect which of the ongoing tensions during the colonial period of American history?\n(A) Tensions between British policies and the aspirations of North American colonists. (B) Tensions between American Indians allied with the French and those allied with the British. (C) Tensions between freed African Americans and white planters. (D) Tensions between backcountry settlers and elites within colonial America.\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. After the French and Indian War, the Scotch-Irish settlers attacked American Indians. After the attacks on the Conestoga, about 250 Paxton Boys present their grievances to the Pennsylvania legislature. As mentioned in the information, the Paxton Boys cited resentiment at local elites. The answer is (D).\n\nQ: This question refers to the following information.\nOur leaders talk about stopping aggression from the north, but this was a struggle among groups of Vietnamese until we intervened. We seem bent upon saving the Vietnamese from Ho Chi Minh even if we have to kill them and demolish their country to do it. As the native people survey bombed-out villages, women and children burned by napalm, rice crops destroyed and cities overrun with our military personnel, they are doubtless saying secretly of the Vietcong guerillas and of the American forces, \"A plague on both your houses.\" \u2026 Stop the bombing, north and south, end search and destroy offensive sweeps, and confine our military action to holding operations on the ground. Bombing the north has failed to halt or seriously check the flow of troops to the south and may, in fact, have prompted a much greater war effort by Hanoi.\n\u2014Senator George McGovern, \"The Lessons of Vietnam,\" April 25, 1967\nWhich of the following opinions from the 1960s most directly reflects the perspective of George McGovern's speech?\n(A) Americans must maximize their technological edge in Vietnam. (B) American bombing in Vietnam is step by step leading to progress in the war. (C) American bombing in Vietnam is a failure. (D) America must not give in to defeatism about the war in Vietnam.\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. \"Stop the bombing\" and \"Bombing the north has failed to halt or seriously check the flow of troops to the south\" indicate that the perspective of George McGovern's speech is that Amerian bombing in Vietnam is a failure. The answer is (C).\n\nQ: This question refers to the following information.\n\"In the new Code of Laws which I suppose it will be necessary for you to make I desire you would Remember the Ladies, and be more generous and favorable to them than your ancestors. Do not put such unlimited power into the hands of the Husbands. Remember all Men would be tyrants if they could. If particular care and attention is not paid to the Ladies we are determined to foment a Rebellion, and will not hold ourselves bound by any Laws in which we have no voice, or Representation.\"\nAbigail Adams, in a letter to John Adams, 1776\n\"Special legislation for woman has placed us in a most anomalous position. Women invested with the rights of citizens in one section\u2014voters, jurors, office-holders\u2014crossing an imaginary line, are subjects in the next. In some States, a married woman may hold property and transact business in her own name; in others, her earnings belong to her husband. In some States, a woman may testify against her husband, sue and be sued in the courts; in others, she has no redress in case of damage to person, property, or character. In case of divorce on account of adultery in the husband, the innocent wife is held to possess no right to children or property, unless by special decree of the court. But in no State of the Union has the wife the right to her own person, or to any part of the joint earnings of the co-partnership during the life of her husband. In some States women may enter the law schools and practice in the courts; in others they are forbidden. In some universities girls enjoy equal educational advantages with boys, while many of the proudest institutions in the land deny them admittance, though the sons of China, Japan and Africa are welcomed there. But the privileges already granted in the several States are by no means secure.\"\nSusan B. Anthony, \"Declaration of Rights for Women,\" July 4, 1876\nThe sentiments expressed in the second excerpt by Susan B. Anthony are most likely in support of\n(A) the Equal Rights Amendment (B) universal suffrage (C) states' rights (D) prohibition\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. The above information mentioned that women are in an anomalous position in terms of legislation. Women's earnings do not belong to themselves, or they cannot testify against her husbands. Susan believes women should have equal legal rights as men. The answer is (B).\n\nQ: This question refers to the following information.\n\"Society in every state is a blessing, but government even in its best state is but a necessary evil; in its worst state an intolerable one; for when we suffer, or are exposed to the same miseries by a government, which we might expect in a country without government, our calamity is heightened by reflecting that we furnish the means by which we suffer. Government, like dress, is the badge of lost innocence; the palaces of kings are built on the ruins of the bowers of paradise. For were the impulses of conscience clear, uniform, and irresistibly obeyed, man would need no other lawgiver; but that not being the case, he finds it necessary to surrender up a part of his property to furnish means for the protection of the rest; and this he is induced to do by the same prudence which in every other case advises him out of two evils to choose the least. Wherefore, security being the true design and end of government, it unanswerably follows that whatever form thereof appears most likely to ensure it to us, with the least expense and greatest benefit, is preferable to all others.\"\nThomas Paine, Common Sense, 1776\nWhich of the following \"miseries\" alluded to above were most condemned by Anti-Federalists of the post-Revolutionary era?\n(A) Organized response to Bacon's Rebellion (B) Federal response to Shays's Rebellion (C) Federal response to the Whiskey Rebellion (D) Federal response to Pontiac's Rebellion\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. Anti-Federalists do not believe centralized government power, and suspect Washington's military response to Whiskey Rebellion. Bacon's Rebellion and Pontiac's Rebellion happen before the Revolution and they can be ruled out. The answer is (C).\n\n", "high_school_world_history": "The following are multiple choice questions (with answers) about high school world history.\n\nQ: This question refers to the following information.\n\"At least one of the [world's] societies would have to somehow enormously increase its productivity [in order to achieve global hegemony]. That quantum jump would have to be made before the various scientific, technological, agricultural, and industrial revolutions on which our post-quantum-leap world rests. It could only be accomplished by exploiting the ecosystems, mineral resources, and human assets of whole continents outside the lands of the society making the jump. Western Europe did just that by means of its brutality and guns and, more important, by geographical and ecological luck.\"\nCopyright \u00a9 2015 Cambridge University Press.\nAlfred Crosby, historian, Ecological Imperialism, 2004\nThe \"quantum jump\" mentioned in the passage most directly contributed to which of the following developments in the period 1450\u20131750 C.E.?\n(A) A breakdown in trade routes through the collapse of the established state structure (B) An increase in the population of the world through more plentiful supplies of food (C) The spread of Chinese and Indian belief systems across the world (D) An increase in social unrest\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The \"quantum jump\" mentioned in the passage refers to the conquest of the New World and the Columbian Exchange. Choice (A) and (C) did not happen in history. Choice (C) refers to the human assets. The answer is (B).\n\nQ: This question refers to the following information.\n\"The struggle against neo-colonialism is not aimed at excluding the capital of the developed world from operating in less developed countries. It is aimed at preventing the financial power of the developed countries being used in such a way as to impoverish the less developed.\nNon-alignment, as practiced by Ghana and many other countries, is based on co-operation with all States whether they be capitalist, socialist or have a mixed economy. Such a policy, therefore, involves foreign investment from capitalist countries, but it must be invested in accordance with a national plan drawn up by the government of the non-aligned State with its own interests in mind. The issue is not what return the foreign investor receives on his investments\u2026The question is one of power. A State in the grip of neo-colonialism is not master of its own destiny.\"\nKwame Nkrumah, Neo-Colonialism, 1965\nWhich of the following provides the best context for Nkrumah's writings?\n(A) The Industrial Revolution (B) Decolonization (C) Regional Free Trade Associations (D) Autarky\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The passage expresses a point that the successful fight against neo-colonialism were in danger and the newly independent nations like Ghana may be re-colonized via financial power of the developed countries. The answer is (B).\n\nQ: This question refers to the following information.\n\"Indeed, as both the fatwas of distinguished [scholars] who base their opinion on reason and tradition alike and the consensus of the Sunni community agree that the ancient obligation of extirpation, extermination, and expulsion of evil innovation must be the aim of our exalted aspiration, for \"Religious zeal is a victory for the Faith of God the Beneficent\"; then, in accordance with the words of the Prophet (Peace upon him!) \"Whosoever introduces evil innovation into our order must be expelled\" and \"Whosoever does aught against our order must be expelled,\" action has become necessary and exigent\u2026\"\nLetter from Ottoman Sultan Selim I to Safavid Shah Ismail I, 1514\nThe letter from Selim I is most clearly an example of which of the following?\n(A) The maintenance of military supremacy at all costs (B) Expanding tensions between religious sects (C) Factors that brought about the collapse of the Ottoman Empire (D) Peacemaking efforts among the Islamic empires\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The passage is an example of expanding tensions between Selim and Ismail. In the passage the Selim references the fatwa and the consensus of the Sunni community to against whosoever introduces evil. The answer is (B).\n\nQ: This question refers to the following information.\n\"The real grievance of the worker is the insecurity of his existence; he is not sure that he will always have work, he is not sure that he will always be healthy, and he foresees that he will one day be old and unfit to work. If he falls into poverty, even if only through a prolonged illness, he is then completely helpless, exam_ins to his own devices, and society does not currently recognize any real obligation towards him beyond the usual help for the poor, even if he has been working all the time ever so faithfully and diligently. The usual help for the poor, however, leaves a lot to be desired, especially in large cities, where it is very much worse than in the country.\"\nOtto von Bismarck, 1884\nOtto von Bismarck likely made this speech in reaction to which of the following issues?\n(A) Social acceptance of child labor (B) Declining life expectancy in Germany (C) Criticisms of German trade tariffs (D) Negative effects attributed to industrial capitalism\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The passage talks about the grievance of the work under the industrial capitalism. The answer is (D).\n\nQ: This question refers to the following information.\nHe contains all works and desires and all perfumes and all tastes. He enfolds the whole universe and in silence is loving to all. This is the Spirit that is in my heart, this is Brahman. To him I shall come when I go beyond this life, and to him will come he who has faith and doubts not.\n\u2014The Upanishads, India, c. 1000 BCE\nTo which religion does the speaker most likely belong?\n(A) Hinduism (B) Buddhism (C) Shintoism (D) Zoroastrianism\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. Brahman refers to the ultimate reality of all things in the Hindu religion. In contrast, Buddhism does not have a concept of supreme God. The answer is (A).\n\n", "human_aging": "The following are multiple choice questions (with answers) about human aging.\n\nQ: All other things being equal, which of the following persons is more likely to show osteoporosis?\n(A) An older Hispanic American woman (B) An older African American woman (C) An older Asian American woman (D) An older Native American woman\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. Although osteoporosis can occur at any age, the risk is higher for older people. It is most common in Asian and non-Hispanic white women. The answer is (C).\n\nQ: The finding that adults tend to remember events from their adolescence better than from other periods in their lives is referred to as the\n(A) Adolescence advantage (B) Reminiscence bump (C) Memorial memorial (D) Quadratic retrieval spike\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. Reminiscence bump is a phenomenon that older adults tend to recollect events during their young ages. People usually have a period of childhood amnesia from birth to around age 5, and a reminiscence bump between 10 and 30. The answer is (B).\n\nQ: Which element in tobacco smoke is responsible for cancers?\n(A) Nicotine (B) Tar (C) Carbon monoxide (D) Smoke particles\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. The benzene, acrylamide and acrylonitrile in tar interact with the lungs and cause DNA mutations in cells of the lungs, and lead to cancer. The answer is (B).\n\nQ: When older adults move to a new state after retirement, which of the following is the more likely destination?\n(A) Texas (B) California (C) Hawaii (D) Vermont\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. Texas does not have state tax, and has low cost of living compared with the other three options. The answer is (A).\n\n", "human_sexuality": "The following are multiple choice questions (with answers) about human sexuality.\n\nQ: The most common disorder among men who seek sexual therapy is:\n(A) premature ejaculation (B) inhibited ejaculation (C) erectile disorder (D) ejaculatory disorder\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. The most common type of arousal disorder for men is erectile dysfunction, meaning that a person is not able to get any physical satisfaction from sexual activity although he may be interested in it. The answer is (C).\n\nQ: A woman who knows she has active herpes and untreated syphilis but continues to have sex without informing her partners of her condition has, in psychoanalytic terms:\n(A) a strong ego (B) a weak superego (C) a weak id (D) a strong superego\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. A person with weak superego tends to be delinquent, criminal or have antisocial personality. The action of the woman who knows she has active venereal disease but still have sex with her partners indicate she may has antisocial personality. The answer is (B).\n\nQ: The nature of homosexual activities that occur during preadolescence include all but which of the following?\n(A) sexual intercourse (B) circle jerks (C) exhibitionism (D) touching each other's genitals\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. Sexual intercourse is prohibited during preadolescence. The answer is (A).\n\nQ: Women's ability to have multiple orgasms is primarily due to:\n(A) the fact that they do not have a refractory period. (B) the response of the inner layers of the vagina. (C) having alternating orgasms in different locations. (D) the G-Spot.\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. The refractory period is the time when a person is not able to be erect or is not interested in sex. The answer is (A).\n\nQ: Morning sickness is typically a problem:\n(A) during the first trimester (B) during the second trimester (C) during the third trimester (D) all through the pregnancy\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. Morning sickness usually begins by nine weeks after conception, corresponding to the first trimester. The answer is (A).\n\n", "international_law": "The following are multiple choice questions (with answers) about international law.\n\nQ: How the consent to be bound of a State may be expressed?\n(A) The consent of a State to be bound is expressed only by ratification (B) The consent of a state to be bound by a treaty may be expressed by signature, ratification, acceptance, approval or accession (C) The consent of a State to be bound is expressed by signature (D) The consent of a State to be bound is expressed by whatever means they choose\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. Article 11 of Vienna Convention on the Law of Treaties signed in 1969 states that \"the consent of a State to be bound by a treaty may be expressed by signature, exchange of instruments constituting a treaty, ratification, acceptance, approval or accession, or by any other means if so agreed.\" (B) is the most precise and accurate answer. The answer is (B).\n\nQ: What is the judge ad hoc?\n(A) If a party to a contentious case before the ICJ does not have a national sitting as judge, it is entitled to nominate someone as a judge solely for that case, with the title of judge ad hoc (B) Judge ad hoc is the member of the bench of the ICJ with a casting vote (C) Judge ad hoc is a surrogate judge, in case a judge is disqualified or passes away (D) Judge ad hoc is the judge that each party will always nominate in every contentious case\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. As \"ad hoc\" implies, a judge ad hoc is appointed only for a specific case or period, when a party to a contentious case before the International Court of Justice does not have a regular national sitting as judge. The answer is (A).\n\nQ: When 'consent' can serve as a circumstance precluding the wrongfulness of a State conduct?\n(A) Consent can serve as a circumstance precluding the wrongfulness whenever it is given (B) Consent can never serve as a circumstance precluding wrongfulness (C) Consent can serve as a circumstance precluding wrongfulness, provided the consent is valid and to the extent that the conduct remains within the limits of the consent given (D) Consent can always serve as a circumstance precluding wrongfulness, no matter which organ of the State gives it\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. Valid consent can serve as a circumstance precluding the wrongfulness of a State conduct if the conduct remains within the limits of that consent, according to Chapter V of the Responsibility of States for Internationally Wrongful Acts, 2001, United Nations. The answer is (C).\n\nQ: Would a reservation to the definition of torture in the ICCPR be acceptable in contemporary practice?\n(A) This is an acceptable reservation if the reserving country's legislation employs a different definition (B) This is an unacceptable reservation because it contravenes the object and purpose of the ICCPR (C) This is an unacceptable reservation because the definition of torture in the ICCPR is consistent with customary international law (D) This is an acceptable reservation because under general international law States have the right to enter reservations to treaties\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. For it contravenes the object and purpose of the ICCPR, this is an unacceptable reservation in contemporary practice. The answer is (B).\n\nQ: What types of force does Article 2(4) of the UN Charter prohibit?\n(A) Article 2(4) encompasses only armed force (B) Article 2(4) encompasses all types of force, including sanctions (C) Article 2(4) encompasses all interference in the domestic affairs of States (D) Article 2(4) encompasses force directed only against a State's territorial integrity\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. Article 2(4) of the UN Charter prohibits states from using armed forces in their international relations. The answer is (A).\n\n", "jurisprudence": "The following are multiple choice questions (with answers) about jurisprudence.\n\nQ: Iverson Jewelers wrote a letter to Miller, 'We have received an exceptionally fine self winding Rolox watch which we will sell to you at a very favorable price.'\n(A) The letter is an offer to sell (B) A valid offer cannot be made by letter. (C) The letter contains a valid offer which will terminate within a reasonable time. (D) The letter lacks one of the essential elements of an offer.\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. An offer shows the intent to enter into a mutually-beneficial contract with specific terms. An offer can be made by a letter. While this letter indicates the willingness to sell, the lack of specific terms, such as transaction price and offer expiration date, makes it an incomplete offer. The answer is (D).\n\nQ: Functions of the law include all but which of the following?\n(A) maximizing individual freedom (B) providing a basis for compromise (C) keeping the peace (D) promoting the principles of the free enterprise system\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. Laws are fundamentally about helping resolve disputes between individuals, and therefore essential for maximizing individual freedom, providing a basis for compromise, and keeping the peace. The answer is (D).\n\nQ: The ________ School of jurisprudence postulates that the law is based on what is \"correct.\"\n(A) Natural Law (B) Analytical (C) Historical (D) Sociological\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. Natural Law School of jurisprudence focuses on the laws of nature, and states that the law should be based on ethics, morals, and what is \"correct\". Analytical deals with the law as it already exists, Historical postulates that the law was found and not made, and Sociological studies how the law and society impact each other. The answer is (A).\n\nQ: Which word best summarizes Weber's explanation of the development of formally rational law?\n(A) Authority. (B) Charisma. (C) Co-operation. (D) Capitalism.\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. Weber explained the development of formal rationality in laws as how the modern society moved from tradition to rationality, where people decide actions based less on how they were culturally done and more on expected utilities. How rational individuals optimize efficiency of accomplishing tasks for higher rewards is a core principle of Capitalism. The answer is (D).\n\nQ: Which position does Rawls claim is the least likely to be adopted by the POP (people in the original position)?\n(A) The POP would choose equality above liberty. (B) The POP would opt for the 'maximin' strategy. (C) The POP would opt for the 'difference principle'. (D) The POP would reject the 'system of natural liberty.'\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. The POP would opt for the 'maximin' strategy, opt for the 'difference principle', and reject the 'system of natural liberty', but the POP would not choose equality above liberty, since the POP assume both equal and free citizens. The answer is (A).\n\n", "logical_fallacies": "The following are multiple choice questions (with answers) about logical fallacies.\n\nQ: When an arguer causes confusion during refutation because of real or feigned lack of an ability to engage in refutation, that arguer may have committed the fallacy of\n(A) poor sportsmanship (B) appeal to compassion (C) argument against the person (D) ignorance of refutation\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. Ignorance of refutation, one of Aristotle's original list of logical fallacies in his Organon, is when someone causes confusion in an argument through real or feigned inability to engage in refutation, in order to win the argument. The answer is (D).\n\nQ: The complex question fallacy consists of\n(A) arguing something is inferior just because it doesn't do something it was never intended to do. (B) including more than one claim in the proposition and treating proof for one claim as proof for all the claims. (C) drawing a conclusion before examining the evidence, and only considering evidence that supports that conclusion. (D) asking a question that includes either an unproven assumption or more than one question, thus making a straightforward yes or no answer meaningless.\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. The complex question fallacy is when someone makes a single yes or no answer to a question meaningless, by including either an unproven assumption or many questions. The latter is also known as the many questions fallacy. The answer is (D).\n\nQ: Arguing that what is true of the parts must be true of the whole is the fallacy of...\n(A) Division (B) Composition (C) Appeal to the person (D) Appeal to ignorance\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. Fallacy of composition occurs when someone argues what is true of the parts must be true of the whole. The answer is (B).\n\nQ: Which of the following is true of a valid categorical syllogism?\n(A) The minor premise must deny the antecedent (B) The major premise must affirm the consequent (C) The middle term must be used in at least one premise in a universal or unqualified sense (D) All of the above\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. A valid categorical syllogism must satisfy several conditions: (1) the syllogism must have exactly three terms (2) every term of the syllogism must be used twice exactly, (3) a term may be used only once in any premise, and (4) the middle term must be used in at least one premise in a universal or unqualified sense, etc. Only (C) is true. The answer is (C).\n\nQ: If someone attacks the character of an opposing arguer, instead of responding to that opponent's arguments, the first person has probably committed which of the following fallacies?\n(A) tu quoque (B) horse laugh (C) argument against the person (D) ignoratio elenchi\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. The argument against the person fallacy occurs when someone irrelevantly attacks the character of an opposing arguer, instead of addressing that opponent's arguments. The answer is (C).\n\n", "machine_learning": "The following are multiple choice questions (with answers) about machine learning.\n\nQ: Which image data augmentation is most common for natural images?\n(A) random crop and horizontal flip (B) random crop and vertical flip (C) posterization (D) dithering\nA: Let's think step by step. Data augmentation is used to increase the diversity of images in the training dataset. It is important that natural images are kept natural after being augmented. Vertical flips of images are not natural, so (B) is false. Posterization makes the image look like a poster and and dithering increases color depth. None of these two preserve the natural property. The only natural data augmentation technique is (A). The answer is (A).\n\nQ: Traditionally, when we have a real-valued input attribute during decision-tree learning we consider a binary split according to whether the attribute is above or below some threshold. Pat suggests that instead we should just have a multiway split with one branch for each of the distinct values of the attribute. From the list below choose the single biggest problem with Pat\u2019s suggestion:\n(A) It is too computationally expensive. (B) It would probably result in a decision tree that scores badly on the training set and a testset. (C) It would probably result in a decision tree that scores well on the training set but badly on a testset. (D) It would probably result in a decision tree that scores well on a testset but badly on a training set.\nA: Let's think step by step. Because the input is real valued, it is unlikely that the same values appear both at training and test time. This means that while such a decision tree could yield good performance on the training data, when evaluated on the test data it will perform badly because the decision tree won\u2019t know what to do with numbers that did not appear in the training data. The answer is (C).\n\nQ: You are reviewing papers for the World\u2019s Fanciest Machine Learning Conference, and you see submissions with the following claims. Which ones would you consider accepting?\n(A) My method achieves a training error lower than all previous methods! (B) My method achieves a test error lower than all previous methods! (Footnote: When regularisation parameter \u03bb is chosen so as to minimise test error.) (C) My method achieves a test error lower than all previous methods! (Footnote: When regularisation parameter \u03bb is chosen so as to minimise cross-validaton error.) (D) My method achieves a cross-validation error lower than all previous methods! (Footnote: When regularisation parameter \u03bb is chosen so as to minimise cross-validaton error.)\nA: Let's think step by step. In machine learning, we train with some data and fixed hyperparameters and the training error can be arbitrarily low, so (A) can\u2019t be right. Then, one compares different hyperparameters by selecting the model with the lowest cross-validation error, this means that (B) and (D) are not the right procedure. The only relevant number after these is the test error and thus (C) is the right answer. The answer is (C).\n\nQ: A 6-sided die is rolled 15 times and the results are: side 1 comes up 0 times; side 2: 1 time; side 3: 2 times; side 4: 3 times; side 5: 4 times; side 6: 5 times. Based on these results, what is the probability of side 3 coming up when using Add-1 Smoothing?\n(A) 2.0/15 (B) 1.0/7 (C) 3.0/16 (D) 1.0/5\nA: Let's think step by step. Add-1 smoothing adds the value of one to the different counts and then normalizes the probabilities accordingly. The counts after adding one will be: side 1 comes up 1 time; side 2: 2 times; side 3: 3 times; side 4: 4 times; side 5: 5 times; side 6: 6 times. The number of sum one die rolls will be 21, so the probability of drawing a three is 3/21 = 1/7. The answer is (B).\n\nQ: To achieve an 0/1 loss estimate that is less than 1 percent of the true 0/1 loss (with probability 95%), according to Hoeffding's inequality the IID test set must have how many examples?\n(A) around 10 examples (B) around 100 examples (C) between 100 and 500 examples (D) more than 1000 examples\nA: Let's think step by step. By the Hoeffding\u2019s inequality, we expect that with 95% probability the in-sample and out-of-sample errors differ by epsilon when we have N samples if 2 exp(-2 epsilon^2 N)<0.05, this implies that N > -1/(2*epsilon**2) log ( 0.05/2 )= log (40)*5000. Since log(40)>1, we have that one needs more than 1000 examples. The answer is (D).\n\n", "management": "The following are multiple choice questions (with answers) about management.\n\nQ: How can organisational structures that are characterised by democratic and inclusive styles of management be described?\n(A) Hierarchical (B) Bureaucratic (C) Flat (D) Functional\nA: Let's think step by step. We refer to Wikipedia articles on management for help. Flat organizational structures are characterized by democratic and inclusive styles of management, and have few (if any) levels of management between the workers and managers. The answer is (C).\n\nQ: Hygiene factors are associated with which writer?\n(A) Frederick Hertzberg (B) D.C. McClelland (C) Abraham Maslow (D) Douglas McGregor\nA: Let's think step by step. We refer to Wikipedia articles on management for help. Hygiene factors include compensation, company policies, supervision, interpersonal relations, and work environments. Hertzberg lists them as factors that cannot motivate employees but can minimize job dissatisfaction. The answer is (A).\n\nQ: What characteristic is not a key feature of the 'open systems' model of management?\n(A) Morale (B) Innovation (C) Growth resource (D) Adaptation\nA: Let's think step by step. We refer to Wikipedia articles on management for help. The key characteristics of an open system in management include innovation, growth resource, and adaption, but do not include morale. The answer is (A).\n\nQ: Which element of the cultural web forms regalia?\n(A) Symbols (B) Rituals and routines (C) Power structures (D) Control systems\nA: Let's think step by step. We refer to Wikipedia articles on management for help. The cultural web is a tool for mapping an organization's culture, where symbols form the regalia that visually expresses the values that the organization holds as important. The answer is (A).\n\nQ: What are the two main dimensions of the Ohio Studies into leadership?\n(A) Starting position and end position (B) Initial environment and changed environment (C) Organisational structure and conditioning (D) Initiating structure and considerations\nA: Let's think step by step. We refer to Wikipedia articles on management for help. The Ohio State Leadership Studies conducted in the 1940s identified initiating structure and consideration as the two main dimensions of leader behavior. The answer is (D).\n\n", "marketing": "The following are multiple choice questions (with answers) about marketing.\n\nQ: Although the content and quality can be as controlled as direct mail, response rates of this medium are lower because of the lack of a personal address mechanism. This media format is known as:\n(A) Care lines. (B) Direct mail. (C) Inserts. (D) Door to door.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Door to door marketing delivers non-addressed items within all buildings within a geographic area. While it can control the content and quality as well as direct mail marketing, its response rate is lower because of the lack of a personal address mechanism. The answer is (D).\n\nQ: In an organization, the group of people tasked with buying decisions is referred to as the _______________.\n(A) Outsourcing unit. (B) Procurement centre. (C) Chief executive unit. (D) Decision-making unit.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. In an organization, the group of the people tasked with buying decision is referred to as the decision-making unit. The answer is (D).\n\nQ: The single group within society that is most vulnerable to reference group influence is:\n(A) The older consumer who feels somewhat left out of things. (B) The married women, many of whom feel a need for stability in their lives. (C) New immigrants who really want to assimilate into their new culture. (D) Children, who base most of their buying decisions on outside influences.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Children, who mostly based their buying decisions on outside influences, are the single group within society that is more vulnerable to reference group influence. The answer is (D).\n\nQ: Which of the following is an assumption in Maslow's hierarchy of needs?\n(A) Needs are dependent on culture and also on social class. (B) Lower-level needs must be at least partially satisfied before higher needs can affect behaviour. (C) Needs are not prioritized or arranged in any particular order. (D) Satisfied needs are motivators, and new needs emerge when current needs remain unmet.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Maslow's hierarchy of needs, from the bottom upwards, are physiological (food and clothing), safety, love and belonging needs, esteem, and self-actualization. Lower-level needs must be at least partially satisfied before higher ones can affect behavior. The answer is (B).\n\nQ: _____________ is a natural outcome when combining demographic and geographic variables.\n(A) Geodemographics (B) Product differentiation. (C) ANSOFF matrix. (D) Brand management.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Geodemographics is a natural outcome when combining demographic and geographic variables. The answer is (A).\n\n", "medical_genetics": "The following are multiple choice questions (with answers) about medical genetics.\n\nQ: The stage of meiosis in which chromosomes pair and cross over is:\n(A) prophase I (B) metaphase I (C) prophase II (D) metaphase II\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. Prophase I is the stage of meiosis where homologous chromosomes pair with each other and exchange genetic material. The answer is (A).\n\nQ: DNA ligase is\n(A) an enzyme that joins fragments in normal DNA replication (B) an enzyme of bacterial origin which cuts DNA at defined base sequences (C) an enzyme that facilitates transcription of specific genes (D) an enzyme which limits the level to which a particular nutrient reaches\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. DNA ligase is a type of enzyme (EC 6.5.1.1) responsible for joining DNA strands together by catalyzing a phosphodiester bond. The answer is (A).\n\nQ: Which of the following conditions does not show multifactorial inheritance?\n(A) Pyloric stenosis (B) Schizophrenia (C) Spina bifida (neural tube defects) (D) Marfan syndrome\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. Multifactorial inheritance is when more than a single factor is responsible for causing a given trait or health problem. Genes cannot be the only factor. Marfan syndrome, on the other hand, requires only one abnormal copy of the of the Marfan gene, from one parent, to inherit the trait. The answer is (D).\n\nQ: A gene showing codominance\n(A) has both alleles independently expressed in the heterozygote (B) has one allele dominant to the other (C) has alleles tightly linked on the same chromosome (D) has alleles expressed at the same time in development\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. Codominance, as it relates to genetics, refers to a type of genetic inheritance where the phenotype of both the parents is easily observed in the offspring. A heterozygote is an individual having two different alleles of a gene. The answer is (A).\n\nQ: Large triplet repeat expansions can be detected by:\n(A) polymerase chain reaction. (B) single strand conformational polymorphism analysis. (C) Southern blotting. (D) Western blotting.\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. A Southern blot is a method in molecular biology for detecting specific DNA sequences in a sample. Large triplet repeat expansions are usually detected with this method. The answer is (C).\n\n", "miscellaneous": "The following are multiple choice questions (with answers) about miscellaneous.\n\nQ: Which of these songs was a Top 10 hit for the rock band The Police?\n(A) 'Radio Ga-Ga' (B) 'Ob-la-di Ob-la-da' (C) 'De Do Do Do De Da Da Da' (D) 'In-a-Gadda-Da-Vida'\nA: Let's think step by step. We refer to Wikipedia for help. Radio Ga-Ga is by Queen. Ob-la-di Ob-la-da is by The Beatles. And In-a-Gadda-Da-Vida is by Iron Butterfly. Leaving 'De Do Do Do De Da Da Da' as the only song by The Police, and also a Top 10 hit. The answer is (C).\n\nQ: What place is named in the title of the 1979 live album by rock legends Cheap Trick?\n(A) Budapest (B) Budokan (C) Bhutan (D) Britain\nA: Let's think step by step. We refer to Wikipedia for help. Nippon Budokan is an indoor arena in Tokyo, Japan renowned for hosting rock music concerts including Cheap Trick in 1978. 'Cheap Trick at Budokan' became the name of their album. The answer is (B).\n\nQ: What is produced during photosynthesis?\n(A) hydrogen (B) nylon (C) oxygen (D) light\nA: Let's think step by step. We refer to Wikipedia for help. Photosynthesis is the process in which green plants use the green pigment chlorophyll to synthesize foods with water and carbon dioxide. Oxygen is the byproduct of this process. The answer is (C).\n\nQ: Who is the shortest man to ever win an NBA slam dunk competition?\n(A) Anthony 'Spud' Webb (B) Michael 'Air' Jordan (C) Tyrone 'Muggsy' Bogues (D) Julius 'Dr J' Erving\nA: Let's think step by step. We refer to Wikipedia for help. In 1986, Spud Webb, standing only 5'7\" became the shortest NBA player in history to win an official slam dunk contest. The answer is (A).\n\nQ: How many axles does a standard automobile have?\n(A) one (B) two (C) four (D) eight\nA: Let's think step by step. We refer to Wikipedia for help. Most cars have two axles to rotate the wheels.. The answer is (B).\n\n", "moral_disputes": "The following are multiple choice questions (with answers) about moral disputes.\n\nQ: Baron admits that the versions of the ticking bomb hypothetical she discusses are \"stunningly stupid,\" but she claims this is actually evidence of\n(A) the stupidity of most traditional philosophical examples. (B) a general lack of intelligence among people with advanced degrees. (C) the wrongness of torture. (D) the readiness on the part of many intelligent people to see torture as the best solution to deal with terrorism.\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. The ticking bomb hypothetical poses a problem where many people will die to an exploding bomb, if the hypothetical terrorist does not disclose how to defuse it. Baron sees this hypothetical as silly, but its prevalence does suggest intelligent people, particularly utilitarians, see torture as justifiable to save the lives in this scenario. The answer is (D).\n\nQ: A fertilized ovum is also known as\n(A) a zygote. (B) an embryo. (C) a viability. (D) a blastocyst.\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. Once a single sperm penetrates the layers of an egg to form a new cell, that cell is called a zygote. The answer is (A).\n\nQ: Pence compares six different cases of reproduction, from natural twinning to SCNT. What conclusion does he draw from this comparison?\n(A) SCNT is not a different kind of reproduction because there are no morally relevant differences between it and other permissible means of reproduction. (B) Because there is a low risk of harm for natural twinning, there will be a low risk of harm for SCNT. (C) both A and B (D) neither A nor B\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. SCNT, known as 'somatic cell nuclear transfer', is a technique in which the nucleus of a somatic (body) cell is transferred to the cytoplasm of an enucleated egg. This is used in farm animal cloning. Pence may compare this to other processes of reproduction, and see little moral separation. The answer is (A).\n\nQ: Which of the following is an intrinsic good, according to Singer?\n(A) being in the upper class (B) the ability to learn and solve problems (C) being more beautiful than most of the population (D) being shorter than average\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. Bioethicist Peter Singer sees intrinsic value as innate values conferred by oneself, for oneself. Innanimate objects can be beautiful, short, or have some valuable criteria, but capabilities are intrinsically good. The answer is (B).\n\nQ: According to Metz, what is wrong with consequentialist arguments against capital punishment based on African values?\n(A) It is unclear as of yet whether or not capital punishment deters harm to the community. (B) It is unclear as of yet whether or not capital punishment deters harm to any individuals. (C) Consequentialism is not supported by African values. (D) Even though consequentialism is supported by African values, no consequentialist arguments framed in terms of African values have been offered.\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. Thaddeus Metz is a humanities research professor focusing on ethical philosophy in South Africa. Metz has written the death penalty is unjustified as it treats individuals as incapable of communal relations. It is unclear that capital punishment is to the benefit of, or a deterrent of harm to the community. The answer is (A).\n\n", "moral_scenarios": "The following are multiple choice questions (with answers) about moral scenarios.\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I took his wallet after he harassed me at the bar. Scenario 2 | I made love to my newborn baby boy.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Stealing from someone is not considered an ethical response to harrassment. And having sex with a minor is egregiously unethical. The answer is (A).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I asked for a cigarette from the guy outside the store. Scenario 2 | When the security guard left his post, I pocketed a rare coin.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Asking for something is not morally wrong. Stealing is morally wrong. The answer is (C).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | During the house's demolition, I pushed him into a wall. Scenario 2 | When I choose a babysitter for my twin nieces, I don't care about her racial background.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Physical violence is morally wrong. Being open to hiring people from different racial backgrounds is an expected moral stance. The answer is (B).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I bought an old pick-up truck for work. Scenario 2 | I looked up at the gorgeous waterfall and threw a rock at it.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Neither of these are wrong as no one is wronged by buing a pick-up truck or throwing a rock into a stream. The answer is (D).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I really loved my newborn baby boy. Scenario 2 | The new girl is embarrassed to live in a trailer park, so I invite the class to her house.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Loving someone is not wrong. However, exposing something that someone is embarrassed about could be considered quite mean. The answer is (C).\n\n", "nutrition": "The following are multiple choice questions (with answers) about nutrition.\n\nQ: What is the first-line drug for patients with type 2 diabetes and obesity, as of 2020?\n(A) Acarbose (B) Metformin (C) Sulphonylureas (D) Insulin\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. Metformin (Fortamet, Glumetza, or others) is usually the first medication prescribed for type 2 diabetes, as well as obesity. It works by lowering glucose production in the liver and improving the body's sensitivity to insulin. The answer is (B).\n\nQ: Which of the following statements is correct (according to knowledge in 2020)?\n(A) Consumers with phenylketonuria must avoid the consumption of the sweetener aspartame (B) Consumers with phenylketonuria must avoid the consumption of the sweetener saccharin (C) Consumers with phenylketonuria must avoid the consumption of the sweetener sucralose (D) Consumers with phenylketonuria must avoid the consumption of the sweetener acesulfame K\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. People with phenylketonuria (PKU) cannot break down the amino acid phenylalanine. As it builds up in the blood and brain it can lead to brain damage. People with PKU should avoid foods that are converted to phenylalanine in the body, such as aspartame. The answer is (A).\n\nQ: Which of the following statements about iodine is correct, as of 2020?\n(A) 50% of adults consume iodine at levels below the RNI (B) Dairy products are a poor source of iodine (C) The iodine content of organic milk is generally lower that the level in non-organic milk (D) UK dietary reference values recommend an increase in iodine intake in pregnancy\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. Organic milk usually has less iodine content than non-organic milk. The answer is (C).\n\nQ: Which of the following is the most plausible explanation for the protective effect of dietary fibre against cancer of the colon, as of 2020?\n(A) Propionic acid, formed during colonic fibre fermentation inhibits liver fatty acid synthesis (B) Butyric acid, formed during colonic fibre fermentation stimulates \"silencing\" of the SLC5A8 tumour suppressor gene (C) None of these options are correct (D) Butyric acid, formed during colonic fibre fermentation stimulates anti-oxidant defences in the colon\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. Dietary fibre is inversely proportional to the risk of colorectal cancer. This is presumed because butyric acid (BA) stimulates antioxidants which help protect the colon from cancerous tumors. The answer is (D).\n\nQ: In a cohort study, the risk ratio of developing diabetes was 0.86 when comparing consumers of tea (the exposed) to those who did not drink tea (the unexposed). Which one statement is correct (according to knowledge in 2020)?\n(A) The tea drinkers have lower risk of developing diabetes. (B) The tea drinkers have higher risk of developing diabetes. (C) Based on the information given we cannot tell if the observed difference in disease risk is the result of chance. (D) The risk ratio is close to the value one, so there is no difference in disease risk between the two groups.\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. The risk ratio is not sufficiently reduced that it could not be explained by random chance given the studies sample size. The answer is (C).\n\n", "philosophy": "The following are multiple choice questions (with answers) about philosophy.\n\nQ: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\n(A) metaphysics (B) epistemology (C) quantum physics (D) axiology\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Among the options, only metaphysics studies the nature of reality and existence. The answer is (A).\n\nQ: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\n(A) pleasure. (B) happiness. (C) good. (D) virtue.\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Moore's \"ideal utilitarianism\" states that one's actions should maximize intrinsic goods. The answer is (C).\n\nQ: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\n(A) optimist (B) satisfied (C) nominally religious (D) pessimist\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Before his conversion, Tolstoy feels that life was uncertain, which is a pessimist's point of view. The answer is (D).\n\nQ: According to d'Holbach, people always act according to _____.\n(A) free choices (B) dictates of the soul (C) necessary natural laws (D) undetermined will\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. d'Holbach believes that people act according to necessary laws, and it proves nothing about people's free will. The answer is (C).\n\nQ: Psychological egoism is:\n(A) an ethical theory about how we ought to behave. (B) a generalization concerning the way people tend to behave. (C) a claim about human nature and the ways people are capable of behaving. (D) none of the above.\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Psychological egoism suggests that one behaves based on what makes one feels good, hence it is a claim about human nature and how humans are capable of behaving. The answer is (C).\n\n", "prehistory": "The following are multiple choice questions (with answers) about prehistory.\n\nQ: What is the approximate mean cranial capacity of Homo erectus?\n(A) under 650 cc (B) about 800 cc (C) just under 1000 cc (D) 1200 cc\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. The average cranium capacity of Homo erectus is less than 1000 cubic cm. The answer is (C).\n\nQ: According to Timothy Pauketat, the evidence for social stratification and political power at Cahokia suggests:\n(A) a center of Mississippian civilization with conditions similar to the rise of early states. (B) the limitations of authority in a Native American society of egalitarian foragers. (C) a simple chiefdom or perhaps a complex chiefdom had evolved by A.D. 1500. (D) a center of Mississippian civilization with conditions similar to societies on the Northwest Coast of North America.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Timothy Pauketat is known for his research on Cahokia, the center of the Mississippian culture, where he found similar conditions to the rise of early states. The answer is (A).\n\nQ: Recent research on hominid species dating from the Middle Pliocene indicates there was (as of 2020):\n(A) a great amount of species diversity, or a single species that exhibited a lot of diversity. (B) very little species diversity during this period and very few hominids. (C) decreased species diversity due to a prolonged ice age followed by a severe drought. (D) decreased species diversity but increased numbers of hammerstones and flakes, indicating stone tool manufacture.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Recent research has recognized multiple hominid species from the Middle Pliocene, meaning that there is a great amount of species diversity or diversity in a single species. The answer is (A).\n\nQ: Researchers now believe that the decline of the Maya was caused chiefly by:\n(A) a cataclysm of some kind, such as an earthquake, volcano, or tsunami. (B) ecological degradation resulting from slash-and-burn farming techniques. (C) endless wars between neighboring Mayan city-states. (D) practices of interbreeding that led to a steep rise in congenital disorders.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Researchers believe that the Maya collapse was mainly caused by over-exploitation of natural resources like the slash-and-burn farming techniques. The answer is (B).\n\nQ: The great Mayan king Pacal built temples in the city of Palenque in order to:\n(A) satisfy the powerful Mayan astronomer priests. (B) display his generosity to the common people, since they were allowed to live in the temples. (C) frighten away enemies, in particular the Spaniards. (D) legitimize his kingship, since his father was not royal.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Pacal built the temples as the funerary monument to legitimize his kingship. The answer is (D).\n\n", "professional_accounting": "The following are multiple choice questions (with answers) about professional accounting.\n\nQ: An auditor traces the serial numbers on equipment to a nonissuer\u2019s subledger. Which of the following management assertions is supported by this test?\n(A) Valuation and allocation (B) Completeness (C) Rights and obligations (D) Presentation and disclosure\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. The completeness assertion is tested by tracing supporting documents to the record entries. The answer is (B).\n\nQ: One hundred years ago, your great-great-grandmother invested $100 at 5% yearly interest. What is the investment worth today?\n(A) $13,000 (B) $600 (C) $15,000 (D) $28,000\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. A $100 investment at 5% yearly interest is worth 100*(1.05)^100=13150 after 100 years, which is around $13,000. The answer is (A).\n\nQ: On January 1, year 1, Alpha Co. signed an annual maintenance agreement with a software provider for $15,000 and the maintenance period begins on March 1, year 1. Alpha also incurred $5,000 of costs on January 1, year 1, related to software modification requests that will increase the functionality of the software. Alpha depreciates and amortizes its computer and software assets over five years using the straight-line method. What amount is the total expense that Alpha should recognize related to the maintenance agreement and the software modifications for the year ended December 31, year 1?\n(A) $5,000 (B) $13,500 (C) $16,000 (D) $20,000\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. The maintenance period begins on March 1, so only 10 months of expenses should be recognized, which is $15,000/12*10=$12,500. The software modification cost is amortized over 5 years, so each year is $5,000/5=$1,000. So the total expense is $12,500+$1,000=$13,500. The answer is (B).\n\nQ: Krete is an unmarried taxpayer with income exclusively from wages. By December 31, year 1, Krete's employer has withheld $16,000 in federal income taxes and Krete has made no estimated tax payments. On April 15, year 2, Krete timely filed for an extension request to file her individual tax return, and paid $300 of additional taxes. Krete's year 1 tax liability was $16,500 when she timely filed her return on April 30, year 2, and paid the remaining tax liability balance. What amount would be subject to the penalty for underpayment of estimated taxes?\n(A) $0 (B) $500 (C) $1,650 (D) $16,500\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. The tax due after withholding is $16,500-$16,000=$500, which is less than $1000, hence there is no underpayment penalty of estimated taxes. The answer is (A).\n\nQ: Box a nongovernmental not-for-profit organization had the following transactions during the year: Proceeds from sale of investments $80000 Purchase of property plant and equipment $10000 Proceeds from long-term debt $100000 Loss on sale of investment $5000 What amount should be reported as net cash provided by financing activities in Box's statement of cash flows?\n(A) $70,000 (B) $75,000 (C) $80,000 (D) 100000\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. Among the four transactions, only Proceeds from long-term debt belongs to the financing activities section of cashflow, hence the amount reported should be $100000. The answer is (D).\n\n", "professional_law": "The following are multiple choice questions (with answers) about professional law.\n\nQ: A son owed a creditor $5,000. The son's father contacted the creditor and told him that he wanted to pay the son's debt. The father signed a document that stated the father would pay the son's debt at a rate of $500 a month for 10 months. The creditor made no written or oral commitment to forbear to sue the son to collect the $5,000 debt, and the father made no oral or written request for any such forbearance. For the next five months, the father made and the creditor accepted the $500 monthly payments as agreed. During that period, the creditor, in fact, did forbear to take any legal action against the son. However, the father then informed the creditor that he would make no further payments on the debt. Which of the following is the most persuasive argument that the father is liable to the creditor under the terms of their agreement?\n(A) The father's promise and the creditor's reliance thereon, if proved, gave rise to a valid claim by the creditor against the father based on the doctrine of promissory estoppel. (B) Because it was foreseeable that the father's promise would induce the creditor to forbear taking any action against the son, such forbearance was, as a matter of law, a bargained-for consideration for the father's promise. (C) The father's five payments to the creditor totaling $2,500 manifested a serious intent on the father's part to be contractually bound, and such manifestation is generally recognized as an effective substitute for consideration. (D) By assuming the antecedent debt obligation that the son owed to the creditor, the father became a surety whose promise to the creditor was enforceable, since it was in writing and supported by adequate consideration. \nA: Let's think step by step. We refer to Wikipedia articles on law for help. The doctrine of promissory estoppel stops a person from going back on a promise in contract law, hence option (A) should be the most persuasive argument. The answer is (A).\n\nQ: A state has recently enacted a statute prohibiting the disposal of any nuclear wastes within the state. This law does not contravene or conflict with any federal statutes. A man operates a company in the state that is engaged in the disposal of nuclear wastes. Subsequent to the passage of the state statute, the man, not yet aware of the new law, entered into contracts with many out-of-state firms to dispose of their nuclear wastes in the state. On account of this new law, however, the man will be unable to perform these contracts. Assume that the man has standing to challenge this state law. Which of the following presents his strongest constitutional grounds to challenge the state law prohibiting the disposal of nuclear wastes within the state?\n(A) The commerce clause. (B) The equal protection clause of the Fourteenth Amendment. (C) The privileges and immunities clause of Article IV, Section 2. (D) The contract clause.\nA: Let's think step by step. We refer to Wikipedia articles on law for help. The commerce clause states that Congress shall have the power to regulate commerce with foreign Nations, and among the several States, and with the Indian Tribes. The statute affects inter-state commerce which puts it into question. Hence the man's strongest argument should be the commerce clause. The answer is (A).\n\nQ: On October 1, 1980, a developer, owner of several hundred acres in a rural county, drafted a general development plan for the area. The duly recorded plan imposed elaborate limitations and restrictions upon the land in the plan, which was to be developed as a residential district. The restrictions were to extend to all persons acquiring any of the lots and to their heirs, assigns, and lessees. It was further provided that all subsequent owners would be charged with due notice of the restrictions. Among those restrictions in the general plan were the following:(22) A franchise right is created in a strip of land 10 feet in width along the rear of each lot for the use of public utility companies with right of ingress and egress. (23) No house or structure of any kind shall be built on the aforementioned strip of land running through the said blocks. In 2000, a retiree purchased one of the lots, built a house, and erected a fence in the rear of his property within the restricted area. In 2004, a teacher purchased a lot adjacent to the retiree's property and built a new house. Two years later, a librarian purchased the lot that adjoined the teacher's property. The three deeds to those properties each contained references to the deed book where the general plan was recorded. In 2008, the librarian began the construction of a seven-foot post-and-rail fence along the line dividing his lot with the teacher's, and along the center of the area subject to the franchise right. Although the teacher objected to its construction, the fence was completed. If the teacher seeks a mandatory injunction to compel removal of the librarian's fence, the court will most likely\n(A) grant relief, because the fence was in violation of the easement restriction. (B) grant relief, because the encroachment of the fence violated the restriction in the original plan. (C) deny relief, because the teacher failed to enforce the restriction against the retiree. (D) deny relief, because the fence would not be construed as \"a structure\" within the terms of the restriction. \nA: Let's think step by step. We refer to Wikipedia articles on law for help. The restrictions in the original plan say no house or structure of any kind shall be built on the aforementioned strip of land running through the said blocks. Hence the court will most likely grant relief because the fence violated the restriction in the original plan. The answer is (B).\n\nQ: Judge took judicial notice of some facts at the beginning of the trial. Which of the following is not an appropriate kind of fact for judicial notice?\n(A) Indisputable facts. (B) Facts that have been asserted by individual political organizations. (C) Facts recognized to be true by common knowledge. (D) Facts capable of scientific verification.\nA: Let's think step by step. We refer to Wikipedia articles on law for help. Among the options, facts that have been asserted by individual political organizations is not an appropriate kind of fact for judicial notice. The answer is (B).\n\nQ: A state legislature has recently enacted a statute making it a misdemeanor to curse or revile or use obscene or opprobrious language toward or in reference to a police officer perfonning his duties. A student at a state university organized a demonstration on campus to protest the war. The rally was attended by a group of 50 students who shouted anti-war messages at cars passing by. To show his contempt for the United States, the student sewed the American flag to the rear of his jeans. When a police officer saw the flag sown on the student's jeans, he approached and told him to remove the flag or he would be placed under arrest. The student became angered and shouted at the police officer, \"Listen, you bastard, I'll wear this rag anywhere I please. \" The student was subsequently placed under arrest and charged with violating the state statute. The student subsequently brings suit in state court challenging the constitutionality of the statute. The strongest constitutional argument for the student is that\n(A) the statute is void for vagueness under the Fourteenth Amendment's due process clause. (B) the statute is invalid because it violates the petitioner's freedom of speech under the First Amendment. (C) the statute is an abridgment of freedom of speech under the First Amendment because less restrictive means are available for achieving the same purpose. (D) the statute is overbroad and consequently invalid under the First and Fourteenth Amendments.\nA: Let's think step by step. We refer to Wikipedia articles on law for help. The Fourteenth Amendment further supports the First Amendment by establishing a due process clause. Hence the strongest argument should be the statute is overbroad and consequently invalid under the First and Fourteenth Amendments. The answer is (D).\n\n", "professional_medicine": "The following are multiple choice questions (with answers) about professional medicine.\n\nQ: A 22-year-old male marathon runner presents to the office with the complaint of right-sided rib pain when he runs long distances. Physical examination reveals normal heart and lung findings and an exhalation dysfunction at ribs\u00a04-5 on the right. Which of the following muscles or muscle groups will be most useful in correcting this dysfunction utilizing a direct method?\n(A) anterior scalene (B) latissimus dorsi (C) pectoralis minor (D) quadratus lumborum\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Among the options, only pectoralis minor muscle origins from the outer surfaces of the 3rd to 5th ribs. The answer is (C).\n\nQ: A 36-year-old male presents to the office with a\u00a03-week\u00a0history of low back pain. He denies any recent trauma but says that he climbs in and out of his truck numerous times a day for his job. Examination of the patient in the prone position reveals a deep sacral sulcus on the left, a posterior inferior lateral angle on the right, and a lumbosacral junction that springs freely on compression. The most likely diagnosis is\n(A) left-on-left sacral torsion (B) left-on-right sacral torsion (C) right unilateral sacral flexion (D) right-on-right sacral torsion\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The deep sulcus on the left, a posterior ILA on the right, with a negative spring test suggests a right-on-right sacral torsion. All other options have a deep sulcus on the right. The answer is (D).\n\nQ: A 44-year-old man comes to the office because of a 3-day history of sore throat, nonproductive cough, runny nose, and frontal headache. He says the headache is worse in the morning and ibuprofen does provide some relief. He has not had shortness of breath. Medical history is unremarkable. He takes no medications other than the ibuprofen for pain. Vital signs are temperature 37.4\u00b0C (99.4\u00b0F), pulse 88/min, respirations 18/min, and blood pressure 120/84 mm Hg. Examination of the nares shows erythematous mucous membranes. Examination of the throat shows erythema and follicular lymphoid hyperplasia on the posterior oropharynx. There is no palpable cervical adenopathy. Lungs are clear to auscultation. Which of the following is the most likely cause of this patient's symptoms?\n(A) Allergic rhinitis (B) Epstein-Barr virus (C) Mycoplasma pneumonia (D) Rhinovirus\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The symptoms, especially the headache, suggest that the most likely cause is Rhinovirus. Epstein-Barr virus will cause swollen lymph nodes but there is no palpable cervical adenopathy. Lungs are clear to auscultation suggests it's not Mycoplasma pneumonia. The answer is (D).\n\nQ: A previously healthy 32-year-old woman comes to the physician 8 months after her husband was killed in a car crash. Since that time, she has had a decreased appetite and difficulty falling asleep. She states that she is often sad and cries frequently. She has been rechecking the door lock five times before leaving her house and has to count exactly five pieces of toilet paper before she uses it. She says that she has always been a perfectionist but these urges and rituals are new. Pharmacotherapy should be targeted to which of the following neurotransmitters?\n(A) Dopamine (B) Glutamate (C) Norepinephrine (D) Serotonin\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The patient feels sad and among the options, only Dopamine and Serotonin can help increase positive emotions. Serotonin also affects digestion and metabolism, which can help the patient's decreased appetite and sleep difficulty. The answer is (D).\n\nQ: A 42-year-old man comes to the office for preoperative evaluation prior to undergoing adrenalectomy scheduled in 2 weeks. One month ago, he received care in the emergency department for pain over his right flank following a motor vehicle collision. At that time, blood pressure was 160/100 mm Hg and CT scan of the abdomen showed an incidental 10-cm left adrenal mass. Results of laboratory studies, including complete blood count, serum electrolyte concentrations, and liver function tests, were within the reference ranges. The patient otherwise had been healthy and had never been told that he had elevated blood pressure. He takes no medications. A follow-up visit in the office 2 weeks ago disclosed elevated urinary normetanephrine and metanephrine and plasma aldosterone concentrations. The patient was referred to a surgeon, who recommended the adrenalectomy. Today, vital signs are temperature 36.6\u00b0C (97.9\u00b0F), pulse 100/min, respirations 14/min, and blood pressure 170/95 mm Hg. Physical examination discloses no significant findings. Initial preoperative preparation should include treatment with which of the following?\n(A) Labetalol (B) A loading dose of potassium chloride (C) Nifedipine (D) Phenoxybenzamine\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The symptoms and the adrenal mass suggested pheochromocytoma, and the blood pressure indicates hypertension. Phenoxybenzamine is used to treat hypertension caused by pheochromocytoma. The answer is (D).\n\n", "professional_psychology": "The following are multiple choice questions (with answers) about professional psychology.\n\nQ: In the construction of a multiple regression equation for purposes of prediction, the optimal combination of measures is one in which the predictors\n(A) are uncorrelated with each other but are moderately correlated with the criterion (B) have low correlations with each other and low correlations with the criterion (C) are highly intercorrelated with each other and moderately correlated with the criterion (D) have low correlations with the criterion bur are moderately correlated with each other\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. The basis of multiple regression is to assess the relationship between one continuous variable and a set of independent variables. So the predictors should be uncorrelated with each other but are moderately correlated with the criterion. The answer is (A).\n\nQ: There are three ways to measure the Central Tendency: the Mean, the Median and the Mode. From your knowledge about them, what is the mode?\n(A) less sensitive to extreme scores than the mean (B) more useful for skewed distributions (C) sensitive to extreme values and highly skewed distributions (D) the most frequently occurring number\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. The definition of mode is the most frequently occurring number. The answer is (D).\n\nQ: Carl Jung believed that a client's transference:\n(A) is a fantasy that distracts the client from reality. (B) represents \u201cmixed feelings\u201d toward the therapist. (C) \"is a form of \"\"acting out.\"\"\" (D) reflects the client\u2019s personal and collective unconscious.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Transference is a phenomenon that a person's feelings are unconsciously redirected, so it reflects the client's personal and collective unconscious. The answer is (D).\n\nQ: In terms of Hofstede\u2019s (1980) five cultural dimensions, the United States scores at the top of the scale on:\n(A) individualism. (B) individualism and power distance. (C) power distance and masculinity. (D) uncertainty avoidance.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. US scores highest on individualism among the five cultural dimensions. The answer is (A).\n\nQ: One of your therapy clients asks your advice about a good weight- reduction program. You have investigated the programs in the community and are enrolled in the one you consider the best. This program offers a $50 bonus to its patrons for each new person they bring into the program. Under these circumstances, your most appropriate response would be to\n(A) tell your client the pros and cons of each program you know about except for the one in which you are enrolled (B) recommend to your client the program in which you are enrolled and explain the $50 bonus you will receive (C) recommend to your client the program in which you are enrolled and offer to have the $50 bonus credited to your client's account in the program (D) tell your client the pros and cons of each program you know about, but do not claim the $50 bonus if your client enrolls in your program\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Based on the circumstances, you should tell your client about the pros and cons of each program, but it would be inappropriate to receive the bonus, so you should not claim the $50 bonus. The answer is (D).\n\n", "public_relations": "The following are multiple choice questions (with answers) about public relations.\n\nQ: Earth Hour was a campaign launched by which organization?\n(A) Greenpeace (B) The UN (C) Oxfam (D) World Wildlife Fund\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. Earth Hour is a worldwide movement oragnized launched by the World Wildlife Fund. The answer is (D).\n\nQ: In issues management, what is the most proactive approach to addressing negative or misleading information posted online about your organization?\n(A) Buy domain names that could be used by opposition groups. (B) Post anonymous comments on blogs to combat this information. (C) Prepare a news release that discredits the inaccurate information. (D) Make policy changes to address complaints highlighted on these sites.\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. In issues management, the most proactive approach to addressing negative or misleading information posted online is to make policy changes to address complaints highlighted on those sites. The answer is (D).\n\nQ: At which stage in the planning process would a situation analysis be carried out?\n(A) Defining the program (B) Planning the program (C) Taking action and implementing ideas (D) Evaluation of the program\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. Situation analyses are typically carried out during the planning process stage of defining the program. The answer is (A).\n\nQ: Which of these statements is true of the Vatican in 2010 at the time of the accusations of child abuse cover-ups?\n(A) There was a coordinated media response. (B) Consistent messages were communicated. (C) Criticisms were taken as attacks on the Catholic Church. (D) The credibility of the Vatican was upheld.\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. In 2010 when there were accusations of child abuse cover-ups, the Vatican took those criticisms as attacks on the Catholic Church. The answer is (C).\n\nQ: What should a public relations media practitioner do if she does not know the answer to a reporter's question?\n(A) Give the reporter other information she is certain is correct. (B) Say that the information is 'off the record' and will be disseminated later. (C) Say 'I don't know' and promise to provide the information later. (D) Say 'no comment,' rather than appear uninformed.\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. If a public relations media practitioner does not know the answer to a reporter's question, they should say 'I don't know' and offer to provide the information later. The answer is (C).\n\n", "security_studies": "The following are multiple choice questions (with answers) about security studies.\n\nQ: What are the frameworks of analysis within which terrorism has been considered (as of 2020)?\n(A) Competition between larger nations has resulted in some countries actively supporting terrorist groups to undermine the strength of rival states. Terrorist networks are extended patronage clubs maintained and paid for by their donor states and are conceptualised as being like state actors, to be dealt with using military force. (B) Globalization has enabled the internationalization of terrorist activities by opening up their operational space, although coordination is still managed from a geographical base. This suggests that terrorist groups are nationally structured which means that terrorism cannot be considered in terms of a war to be defeated militarily without having serious implications on the indigenous population. (C) Terrorism can be viewed as a problem to be resolved by military means (war on terrorism), by normal police techniques (terrorism as crime), or as a medical problem with underlying causes and symptoms (terrorism as disease). (D) Terrorism is viewed as a criminal problem. The criminalization of terrorism has two important implications. Firstly, it suggests that terrorism can be eradicated - terrorists can be caught and brought to trial by normal judicial proceedings thereby removing the threat from society - and secondly, it suggests that preventative crime techniques are applicable to prevent its development.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. (A) is wrong because it is not competition between larger nations that causes terrorism. \n(B) is wrong because globalization is not the cause of terrorism.\n(C) is correct because the US undertook the war on terrorism. \n(D) is wrong because preventative crime techniques will likely not end terrorism. The answer is (C).\n\nQ: Which of the following is the best lens through which to investigate the role of child soldiers?\n(A) Child soldiers are victims of combat that need re-education and rehabilitation. (B) Children and their mothers are not active subjects in warfare and are best considered as subjects in the private sphere. (C) Children are most often innocent bystanders in war and are best used as signifiers of peace. (D) Children have political subjecthood that is missed when they are considered as passive victims of warfare.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. Child soliders as a political topic can be missed when they are considered passive victims of warfare. The answer is (D).\n\nQ: How can we best describe the relationship between the state-centric approach and the concept of human security?\n(A) There are such wide divisions within the human security framework regarding the nature of threats and referent objects that no widely applicable comparisons between state-centric approaches and human security can be drawn. (B) By adopting the framework of human security, the limitations of the realist state-centric approach become evident. Whilst human security defines the referent object as the person or population, state-centric approaches prioritise the security of the state, de-prioritizing the pursuit of human security. (C) The state-centric approach to security is a faction of human security, usually defined within the broad school of human security. By being state-centric this approach prioritises the individual as the referent object in security studies. (D) Both the state-centric and human-centric approaches to security are mutually exclusive and offer a sufficient analytic framework with which to understand the international security system. It is therefore the role of security analysts to determine which of these substantial concepts is correct, and which should be discarded.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. Human security focuses on a person or population whereas state-centric approaches focus on the state while deprioritizing human security. The answer is (B).\n\nQ: In order to become securitized, a threat must be presented in which of these ways?\n(A) As an existential threat that requires immediate and extraordinary action, posing a threat to the survival of the state or to societal security. (B) As requiring immediate and extraordinary action by the state, threatening the survival of a referent object and therefore warranting the use of measures not normally employed in the political realm. (C) As an urgent threat to the survival of the referent object, so serious that it legitimises the employment of extraordinary action in response. (D) As an urgent threat to the survival of the audience that requires extraordinary or emergency measures.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. To be securitized, a threat must be an urgent threat to the survival of the referent object. The answer is (C).\n\nQ: What distinguishes coercive diplomacy from military force?\n(A) Compellence is another term for coercive diplomacy, but covering a narrower set of criteria; compellence covers those threats aimed at initiating adversary action. A threat to coerce a state to give up part of its territory would count as coercive diplomacy, as long as that threat proactively initiates action before reactive diplomacy is taken. (B) Coercive diplomacy constitutes the threats of limited force to induce adversary's incentive to comply with the coercer's demands. It is an influence strategy that is intended to obtain compliance: the use of force to defeat an opponent first does not count. It leaves an element of choice with the target to comply, or to continue. (C) Military force, or the threat of military force, utilises fear to achieve strategic objectives. Coercive diplomacy is differentiated from this approach, because it does not use fear as a tool for coercing an adversary. (D) Coercive diplomacy is employed to use force but to limit its effects on the international community. Coercive diplomacy is an aggressive strategy that is intended to obtain compliance through defeat. It does not leave an element of choice with the target, the target either being forced to comply or engage in conflict. It seeks to control by imposing compliance by removing any opportunity for negotiation or concession.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. Coercive diplomacy uses the threat of force to induce the opponent to comply with demands. The answer is (B).\n\n", "sociology": "The following are multiple choice questions (with answers) about sociology.\n\nQ: Which of the following is not a problem associated with official statistics on strike action?\n(A) most strikes go unnoticed by employers and the mass media (B) not all industrial disputes will be reported by the employer (C) the definition of strikes excludes those that involve fewer than ten workers or last less than one day (D) it is hard to compare strikes that were measured in different ways\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. Official statistics on strike action can be problematic because not all industrial disputes will be reported by employers, the definition of strikes excludes those that involves fewer than ten workers or last less than one day, and it is hard to compare strikes that were measured in different ways. Thus, (A) is not a problem associated with official statistics on strike action. The answer is (A).\n\nQ: What does Berger (1963) describe as a metaphor for social reality?\n(A) a fairground ride (B) a circus (C) a puppet theatre (D) a ballet\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. Berger describes social reality using the metaphor of a puppet theatre. The answer is (C).\n\nQ: The term 'hegemony' refers to:\n(A) the tendency for the working class not to realize their own interests (B) a dominant ideology that legitimates economic, political and cultural power (C) a form of dual consciousness based on ideology and everyday experiences (D) a mode of payment given for outstanding topiary\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. Hegemony refers to a dominant ideology that legitimates economic, policital, and cultural power. The answer is (B).\n\nQ: The shift from 'civil religion' to 'common religion' means that:\n(A) the increasing bureaucracy of the state has made religion only a marginal part of our lives (B) despite the weakening of traditional authority, our everyday lives and 'common sense' remain shaped by religious beliefs and values (C) religious participation in collective worship may have declined, but people still practise their faiths in private (D) people are much more likely to discuss their religious beliefs in public, informal settings\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. The shift from civil religion to common religion means that despite the weakening of traditional authority, our everyday lives and common sense remain shaped by religious beliefs and values. The answer is (B).\n\nQ: Which of the following did the post-war welfare state of 1948 not aim to provide:\n(A) free health care and education for all (B) a minimum wage (C) full employment (D) universal welfare\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. The post-war welfare state of 1948 aimed to provide free healthcare and education, full employment, and universal welfare. But it did not aim to provide a minimum wage. The answer is (B).\n\n", "us_foreign_policy": "The following are multiple choice questions (with answers) about us foreign policy.\n\nQ: How did Donald Trump attack globalization in the 2016 campaign?\n(A) Globalization had made men like him too rich (B) Globalization only benefited certain American states, such as New York (C) Liberal elites had encouraged globalization, while 'ordinary Americans' lost jobs because of it (D) Globalization encouraged damaging trade wars\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. Trump attacked globalization because he believed ordinary Americans lost jobs due to it, and so he wanted to blame liberals who had encouraged it. The answer is (C).\n\nQ: How did NSC-68 change U.S. strategy?\n(A) It globalized containment. (B) It militarized containment. (C) It called for the development of the hydrogen bomb. (D) All of the above\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. NSC-68 outlined a variety of courses of action, including globalization of containment, militarization of contaiment, and the development of the hydrogen bomb. The answer is (D).\n\nQ: How do Defensive Realism and Offensive Realism differ in their explanation of state behaviour?\n(A) Defensive realists place greater emphasis on the role of international institutions (B) Defensive realists place less emphasis on geographical factors (C) Offensive realists give more priority to the national interest than Defensive realists. (D) Defensive realists believe states are security maximizers, while Offensive realists believe states to be power maximizers\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. While defensive realism advocates that states are security maximizers, offensive realists think of states as power maximizers. The answer is (D).\n\nQ: The realm of policy decisions concerned primarily with relations between the United States and the rest of the world is known as\n(A) terrorism policy. (B) economic policy. (C) foreign policy. (D) international policy.\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. The topic of policy decisions concerns with relations between the US and the rest of the world is known as foreign policy. The answer is (C).\n\nQ: How did the 2008 financial crisis affect America's international reputation?\n(A) It damaged support for the US model of political economy and capitalism (B) It created anger at the United States for exaggerating the crisis (C) It increased support for American global leadership under President Obama (D) It reduced global use of the US dollar\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. The 2008 financial crisis damanged the international reputation of the American model of political economy and capitalism. The answer is (A).\n\n", "virology": "The following are multiple choice questions (with answers) about virology.\n\nQ: The median survival time to AIDS and death was established by following:\n(A) Seroprevalent HIV-infected individuals (B) Seronegatives (C) Seroconverters (D) High-risk seronegatives\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. The median survival time to AIDS and death was established as a result of the development of seroconverters. The answer is (C).\n\nQ: Which of the following is a morphological characteristic of the paramyxoviruses.\n(A) Fragile viruses often visualised with RNA spewing from the inside (B) Elongate viruses (C) Icosahedral viruses with envelope (D) Very large viruses\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. Paramyxoviruses are fragile viruses often visualised with RNA spewing from the inside. The answer is (A).\n\nQ: The most important goal of a behavioral intervention is:\n(A) Change in behavior (B) Comprehensive coverage (C) Effective use of behavioral theory (D) Sustained behavior change\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. The prim goal of a behavioral intervention is to cause sustained behavior change. The answer is (D).\n\nQ: A key factor facilitating the application of nested case-control studies from the MACS was:\n(A) Data collection (B) Establishment of a repository of biologic specimens (C) Participant interest (D) Administration of the questionnaire by staff\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. The Multicenter AIDS Cohort Study's use of nested case-control studies was facilitated by the establishment of a repository of biologic specimens. The answer is (B).\n\nQ: Why are parvoviruses a highly impactful parasite?\n(A) Because they have no nucleic acid (B) They require a helper virus (C) Only replicate in dividing cells (D) Can integrate into host chromosomes\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. Paroviruses are highly impactful because they do not have nucleic acid. The answer is (A).\n\n", "world_religions": "The following are multiple choice questions (with answers) about world religions.\n\nQ: How can the Upanishads be characterized?\n(A) Ritual texts (B) Philosophical texts (C) Hymns (D) Origin stories\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The Upanishads are the most recent part of Vedas (the oldest scriptures in Hinduism) and supplied the basis of later Hindu philosophy. So they are philosophical texts. The answer is (B).\n\nQ: What is the Second Gem in Buddhism?\n(A) The Dharma (B) The Sangha (C) The Buddha (D) The Bodhisattva\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The Second Gem in Buddhism is The Dharma. The answer is (A).\n\nQ: Which Japanese government promoted a kind of national cult based on the emperor and his associations with kami?\n(A) Honen (B) Tanaka (C) Tokugawa (D) Meiji\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The promotion of a national cult based on the emperor and his associations with Kami happened during the reign of Emperor Meiji (1852-1912). The answer is (D).\n\nQ: In which dynasty was the \"Mandate of Heaven\" developed to legitimatize the new rulers?\n(A) Shang (B) Zhou (C) Han (D) Xia\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The \"Mandate of Heaven\" was developed as an ancient Chinese philosophical concept during the Zhou Dynasty (1046-256 BCE). The answer is (B).\n\nQ: What is the sign of the covenant for Jewish males?\n(A) The rainbow (B) Circumcision (C) A son (D) Bar mitzvah\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. In Judaism, the most distinctive sign of the covenant is circumcision (brit milah). The answer is (B).\n\n"} diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6c1c6a19dc7638adfa630ce80b58294f5b351b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml @@ -0,0 +1,34 @@ +group: mmlu_flan_cot_fewshot +group_alias: mmlu (flan style, fewshot cot) +task: + - group: stem + task: + - mmlu_flan_cot_fewshot_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_cot_fewshot_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_cot_fewshot_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_cot_fewshot_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: True + filter_list: get-answer +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfbf222e5ba5892a5e26113d382ea86ec1300ce0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml @@ -0,0 +1,30 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +validation_split: validation +test_split: test +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "{% if choices is defined%}Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step.{% else %}Q: {{ question.strip() }}\nA:{% endif %}" +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer] if answer is defined else target}}" +filter_list: + - name: "get-answer" + filter: + - function: "regex" + regex_pattern: "(?<=answer is )(.*)(?=.)" + - function: "take_first" +generation_kwargs: + until: + - "" + do_sample: false + temperature: 0.0 +num_fewshot: 4 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6235d5c0997558a123258cba3dfdb4b844a2fb60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml @@ -0,0 +1,59 @@ +dataset_name: abstract_algebra +description: The following are multiple choice questions (with answers) about abstract + algebra. +fewshot_config: + sampler: first_n + samples: + - question: 'Statement 1 | Every element of a group generates a cyclic subgroup of + the group. Statement 2 | The symmetric group S_10 has 10 elements. + + (A) True, True (B) False, False (C) True, False (D) False, True' + target: Let's think step by step. A cyclic group is a group that is generated + by a single element. Hence a subgroup generated by a single element of a group + is cyclic and Statement 1 is True. The answer is (C). + - question: 'The symmetric group $S_n$ has $ + + actorial{n}$ elements, hence it is not true that $S_{10}$ has 10 elements. + + Find the characteristic of the ring 2Z. + + (A) 0 (B) 3 (C) 12 (D) 30' + target: Let's think step by step. A characteristic of a ring is R is $n$ if the + statement $ka = 0$ for all $a\in 2Z$ implies that $k$ is a multiple of $n$. + Assume that $ka = 0$ for all $a\in 2Z$ for some $k$. In particular $2k = 0$. + Hence $k=0$ and $n=0$. The answer is (A). + - question: 'Statement 1| Every function from a finite set onto itself must be one + to one. Statement 2 | Every subgroup of an abelian group is abelian. + + (A) True, True (B) False, False (C) True, False (D) False, True' + target: "Let's think step by step. Statement 1 is true. Let $S$ be a finite set.\ + \ If $f:S \nightarrow S$ is a onto function, then $|S| = |f(S)|$. If $f$ was\ + \ not one to one, then for finite domain $S$ the image would have less than\ + \ $S$ elements, a contradiction.\nStatement 2 is true. Let $G$ be an abelian\ + \ group and $H$ be a subgroup of $G$. We need to show that $H$ is abelian. Let\ + \ $a,b \\in H$. Then $a,b \\in G$ and $ab=ba$. Since $G$ is abelian, $ab=ba$.\ + \ Since $H$ is a subgroup of $G$, $ab \\in H$. Therefore, $ab=ba$ and $H$ is\ + \ abelian. The answer is (A)." + - question: 'Statement 1 | If aH is an element of a factor group, then |aH| divides + |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G. + + (A) True, True (B) False, False (C) True, False (D) False, True' + target: Let's think step by step. Statement 2 is false. Let $H$ be a subgroup + of $S_3$ generated by the cycle $(1,2)$ and $K$ be a subgroup of $S_3$ generated + by the cycle $(1,3)$. Both $H$ and $K$ have two elements, the generators and + the identity. However $HK$ contains cycles (1,2), (1,3) and (2,3,1), but the + inverse of (2,3,1) is (2,1,3) and it does not belong to HK, hence HK is not + a subgroup. The answer is (B). + - question: 'Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field. + + (A) 0 (B) 1 (C) 2 (D) 3' + target: 'Let''s think step by step. Z_3[x]/(x^2 + c) is a field if and only if + x^2 + c does not have roots in Z_3. That is x^2 + c != 0 for every x in Z_3. + If c = 0, then x^2 + c = x^2 has root 0. If c = 1 then x^2 + c = x^2 + 1 = 0 + + 1 for x = 0, 1 + 1 = 2 for x = 1 and 1 + 1 = 2 for x = 2, hence x^2 + 1 does + not have any roots. For c = 2 the polynomial x^2 + 2 has two roots at x = 1 + and x = 2. Hence Z_3[x]/(x^2 + c) is a field if and only if c = 1. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6521bdebf5efa653c7bc798fa7c4ecb985e4166 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml @@ -0,0 +1,75 @@ +dataset_name: anatomy +description: The following are multiple choice questions (with answers) about anatomy. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is the body cavity that contains the pituitary + gland? + + (A) Abdominal (B) Cranial (C) Pleural (D) Spinal' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. The pituitary gland is the\ + \ major endocrine gland attached to the base of the brain, and it is contained\ + \ in the Cranial cavity. The answer is (B)." + - question: 'Which of these branches of the trigeminal nerve contain somatic motor + processes? + + (A) The supraorbital nerve (B) The infraorbital nerve (C) The mental nerve (D) + None of the above' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. \nWe know the following:\ + \ (A) The supraorbital nerve (also known as the frontal nerve) is the largest\ + \ branch of the ophthalmic nerve and branch of ophthalmic division of the trigeminal\ + \ nerve. (B) The infraorbital nerve is a branch of the maxillary division of\ + \ the trigeminal nerve. (C) The mental nerve is a branch of the mandibular division\ + \ of the trigeminal nerve. Because all these nerves are purely sensory nerves\ + \ and do not contain any somatic motor processes. Therefore, the answer should\ + \ be none of the above, which is (D). The answer is (D)." + - question: 'In Angle''s Class II Div 2 occlusion there is + + (A) excess overbite of the upper lateral incisors. (B) negative overjet of the + upper central incisors. (C) excess overjet of the upper lateral incisors. (D) + excess overjet of the upper central incisors.' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. This is a question related\ + \ to anatomy and orthodontics. Excess overjet is associated with Class II occlusions;\ + \ therefore, we can safely eliminate (B) from the list, as negative overjet\ + \ is often associated with Class III occlusions. Now, we need to determine the\ + \ location of the excess overjet, and that would be the upper (maxillary) lateral\ + \ incisors. Only (C) has the correct information. The answer is (C)." + - question: 'The pleura + + (A) have no sensory innervation. (B) are separated by a 2 mm space. (C) extend + into the neck. (D) are composed of respiratory epithelium.' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. First, recall that the pleura\ + \ refers to the thin layer of tissue that covers the lungs and lines the interior\ + \ wall of the chest cavity. Now, let\u2019s look at each option:\nOption (A):\ + \ \u201CThe pleura have no sensory innervation.\u201D This information is not\ + \ correct. The pleura do have a sensory innervation.\nOption (B): \u201CThe\ + \ pleura are separated by a 2 mm space.\u201D This information is not correct.\ + \ There is a very thin \u201Cpotential\u201D space between the layers of the\ + \ pleura; however, it is typically filled with serous pleural fluid. \nOption\ + \ (C): \u201CThe pleura extend into the neck.\u201D This information is actuakky\ + \ true. The cervical pleura, also known as the dome of the pleuradome of the\ + \ pleura, lines the extendsiton of the pleural cavity into the neck.\nOption\ + \ (D): \u201CThe pleura are composed of respiratory epithelium.\u201D This information\ + \ is not correct. The pleaura are composed of connective tissue (CT).\nBecause\ + \ (A), (B), and (D) are all incorrect, (D) is the only correct answer. The answer\ + \ is (C)." + - question: 'What is the embryological origin of the hyoid bone? + + (A) The first pharyngeal arch (B) The first and second pharyngeal arches (C) + The second pharyngeal arch (D) The second and third pharyngeal arches' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. The hyoid bone, which is\ + \ also known as the hyooid, is a a small U-shaped bone located in the anterior\ + \ neck. In its resting position, it lies between the ase of the mandible and\ + \ the third cervical vertebrae. We know that the second and the third pharyngeal\ + \ arches give rise to the horns of the hyoid bone; therefore, the embryological\ + \ origin of the hyoid bone are the second and the third pharyngeal arches\u2014\ + this information is covered in the last option (D). Therefore, we conclude that\ + \ (D) must be the correct answer. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b89974588e8c83db8aedc80b607e25212a676592 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml @@ -0,0 +1,70 @@ +dataset_name: astronomy +description: The following are multiple choice questions (with answers) about astronomy. +fewshot_config: + sampler: first_n + samples: + - question: 'Where do most short-period comets come from and how do we know? + + (A) The Kuiper belt; short period comets tend to be in the plane of the solar + system just like the Kuiper belt. (B) The Kuiper belt; short period comets tend + to come from random directions indicating a spherical distribution of comets + called the Kuiper belt. (C) The asteroid belt; short period comets have orbital + periods similar to asteroids like Vesta and are found in the plane of the solar + system just like the asteroid belt. (D) The Oort cloud; short period comets + tend to be in the plane of the solar system just like the Oort cloud.' + target: Let's think step by step. Most short-period comets come from the Kuiper + belt, and we know because short period coments tend to be in the plane of the + solar system, just like the Kuiper belt is. The answer is (A). + - question: 'You are pushing a truck along a road. Would it be easier to accelerate + this truck on Mars? Why? (Assume there is no friction) + + (A) It would be harder since the truck is heavier on Mars. (B) It would be easier + since the truck is lighter on Mars. (C) It would be harder since the truck is + lighter on Mars. (D) It would be the same no matter where you are.' + target: "Let's think step by step. If we assume that there is no friction, the\ + \ force needed to accelerate the truck is by Newton\u2019s second law only dependent\ + \ on the mass of the truck. Hence (A), (B) and (C) are incorrect since it doesn\u2019\ + t matter that it\u2019s on Mars, and (D) is the correct answer. The answer is\ + \ (D)." + - question: 'Say the pupil of your eye has a diameter of 5 mm and you have a telescope + with an aperture of 50 cm. How much more light can the telescope gather than + your eye? + + (A) 10000 times more (B) 100 times more (C) 1000 times more (D) 10 times more' + target: Let's think step by step. The amount of light is proportional to the aperture + area $A = \pi D^2/4$ for a lens with diameter $D$, so the relative amounts of + light between the eye with diameter 5mm and the telescope with diameter 50mm + is $(50 cm)^2/(5mm)^2 = 10000$. The answer is (A). + - question: 'Why isn''t there a planet where the asteroid belt is located? + + (A) A planet once formed here but it was broken apart by a catastrophic collision. + (B) There was not enough material in this part of the solar nebula to form a + planet. (C) There was too much rocky material to form a terrestrial planet but + not enough gaseous material to form a jovian planet. (D) Resonance with Jupiter + prevented material from collecting together to form a planet.' + target: "Let's think step by step. The asteroid belt is a stellar disc consisting\ + \ of a large number of asteroids between Mars and Jupiter's orbits. The asteroids\ + \ in this belt are affected by the gravitational pull from both other asteroids\ + \ and nearby planets. Due to the strong gravitational force of Jupiter there\ + \ are resonances that give rise to low density regions of asteroids known as\ + \ the Kirkwood gap. So (B) and (C) are not correct since it\u2019s not a lack\ + \ of material that prevents a planet from being formed, and (A) is incorrect\ + \ because the Kirkwood gap would have prevented a planet from forming in the\ + \ first place, and (D) is the correct option. The answer is (D)." + - question: 'Why is Mars red? + + (A) Because the surface is covered with heavily oxidized ("rusted") minerals. + (B) Because the atmosphere scatters more light at bluer wavelengths transmitting + mostly red light. (C) Because Mars is covered with ancient lava flows which + are red in color. (D) Because flowing water on Mars''s surface altered the surface + minerals several billion years ago.' + target: 'Let''s think step by step. Option (B) is not correct because if the red + color was caused by the scattering off the atmosphere, then the earth with a + much thicker atmosphere would also look red. Options (C) and (D) are not specific + enough about why the color of the surface would be red, while (A) is correct + because it explains that the surface is red due to the rusted materials on the + surface and the red color comes from the rust. So the correct option is (A). + The answer is (A).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c83d4bc8c06531a9375f52d8688f8b4a7cdb974 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml @@ -0,0 +1,75 @@ +dataset_name: business_ethics +description: The following are multiple choice questions (with answers) about business + ethics. +fewshot_config: + sampler: first_n + samples: + - question: 'In contrast to _______, _______ aim to reward favourable behaviour by + companies. The success of such campaigns have been heightened through the use + of ___________, which allow campaigns to facilitate the company in achieving + _________ . + + (A) Buycotts, Boycotts, Blockchain technology, Charitable donations (B) Buycotts, + Boycotts, Digital technology, Increased Sales (C) Boycotts, Buyalls, Blockchain + technology, Charitable donations (D) Boycotts, Buycotts, Digital technology, + Increased Sales' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CIn contrast to *boycotts*, *buycotts* aim to reward favourable behavior\ + \ by companies. The success of such campaigns have been heightened through the\ + \ use of *digital technology*, which allow campaigns to facilitate the company\ + \ in achieving *increased sales*.\u201D The answer is (D)." + - question: '_______ is the direct attempt to formally or informally manage ethical + issues or problems, through specific policies, practices and programmes. + + (A) Corporate social responsibility (B) Business ethics management (C) Sustainability + (D) Environmental management' + target: Let's think step by step. We refer to Wikipedia articles on business ethics + for help. The direct attempt manage ethical issues through specific policies, + practices, and programs is business ethics management. The answer is (B). + - question: 'Three contrasting tactics that CSO''s can engage in to meet their aims + are ________ which typically involves research and communication, ________, + which may involve physically attacking a company''s operations or ________, + often involving some form of _______. + + (A) Non-violent direct action, Violent direct action, Indirect action, Boycott + (B) Indirect action, Instrumental action, Non-violent direct action, Information + campaign (C) Indirect action, Violent direct action, Non-violent direct-action + Boycott (D) Non-violent direct action, Instrumental action, Indirect action, + Information campaign' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CThree contrasting tactics that CSO's can engage in to meet their aims\ + \ are *indirect action*, which typically involves research and communication,\ + \ *violent direct action*, which may involve physically attacking a company's\ + \ operations or *non-violent direct action*, often involving some form of *boycott*.\u201D\ + \ The answer is (C)." + - question: 'To ensure the independence of the non-executive board members, there are + a number of steps which can be taken, which include non-executives being drawn + from _______ the company, being appointed for a _________ time period as well + as being appointed _________. + + (A) Outside, Limited, Independently (B) Inside, Limited, Intermittently (C) + Outside, Unlimited, Intermittently (D) Inside, Unlimited, Independently' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CTo ensure the independence of the non-executive board members, there\ + \ are a number of steps which can be taken, which include non-executives being\ + \ draw from *outside* the company, being appointed for a *limited* time period\ + \ as well as being imported *independently*. The answer is (A)." + - question: 'Beyond the business case for engaging in CSR there are a number of moral + arguments relating to: negative _______, the _______that corporations possess + and the ________ of business and society. + + (A) Externalities, Power, Independence (B) Publicity, Insubstantial resources, + Mutual dependence (C) Publicity, Power, Independence (D) Externalities, Power, + Mutual dependence' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CBeyond the business case for engaging the CSR there are a number of\ + \ moral arguments relating to: negative *externalities*, the *power* that corporations\ + \ possess and the *mutual independence* of business and society. The answer\ + \ is (D).\n\n" +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..008d2f870ad82e3ebee126511418c67c787c7b3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml @@ -0,0 +1,48 @@ +dataset_name: clinical_knowledge +description: The following are multiple choice questions (with answers) about clinical + knowledge. +fewshot_config: + sampler: first_n + samples: + - question: 'Glycolysis is the name given to the pathway involving the conversion of: + + (A) glycogen to glucose-1-phosphate. (B) glycogen or glucose to fructose. (C) + glycogen or glucose to pyruvate or lactate. (D) glycogen or glucose to pyruvate + or acetyl CoA.' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. Glycolysis is the name given to the pathway involving conversion of + glycogen or glucose to pyruvate or lactate. The answer is (C). + - question: 'What is the difference between a male and a female catheter? + + (A) Male and female catheters are different colours. (B) Male catheters are + longer than female catheters. (C) Male catheters are bigger than female catheters. + (D) Female catheters are longer than male catheters.' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. The difference between a male and female catheter is that male catheters + tend to be longer than female catheters. The answer is (B). + - question: 'How many attempts should you make to cannulate a patient before passing + the job on to a senior colleague, according to the medical knowledge of 2020? + + (A) 4 (B) 3 (C) 2 (D) 1' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. According to the medical protocol as of 2020, you should make two + attempts to cannulate a patient before passing the job on to a more-senior practitioner. + The answer is (C). + - question: 'In the assessment of the hand function which of the following is true? + + (A) Abduction of the thumb is supplied by spinal root T2 (B) Opposition of the + thumb by opponens policis is supplied by spinal root T1 (C) Finger adduction + is supplied by the median nerve (D) Finger abduction is mediated by the palmar + interossei' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. Of all the options, it is only true that the opposition of the thumb + by opponens pollicis is supplied by spinal root T1. The answer is (B). + - question: 'The energy for all forms of muscle contraction is provided by: + + (A) ATP. (B) ADP. (C) phosphocreatine. (D) oxidative phosphorylation.' + target: 'Let''s think step by step. We refer to Wikipedia articles on clinical + knowledge for help. The energy for muscular contraction is provided by ATP (adenosine + triphosphate), which is the powerhouse of the cell. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28f7f989b86b28cccc5e9128268e8ea6bc80e662 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml @@ -0,0 +1,75 @@ +dataset_name: college_biology +description: The following are multiple choice questions (with answers) about college + biology. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following represents an accurate statement concerning arthropods? + + (A) They possess an exoskeleton composed primarily of peptidoglycan. (B) They + possess an open circulatory system with a dorsal heart. (C) They are members + of a biologically unsuccessful phylum incapable of exploiting diverse habitats + and nutrition sources. (D) They lack paired, jointed appendages.' + target: Let's think step by step. Peptidoglycan is known to comprise the plasma + membrane of most bacteria, rather than the exoskeleton of arthropods, which + is made of chitin, which rules out (A). The answer (C) is false because arthropods + are a highly successful phylum. Likewise, arthropods have paired, jointed appendages, + which rules out (D). The only remaining option is (B), as arthropods have an + open circulatory system with a dorsal tubular heart. The answer is (B). + - question: 'In a given population, 1 out of every 400 people has a cancer caused by + a completely recessive allele, b. Assuming the population is in Hardy-Weinberg + equilibrium, which of the following is the expected proportion of individuals + who carry the b allele but are not expected to develop the cancer? + + (A) 1/400 (B) 19/400 (C) 20/400 (D) 38/400' + target: "Let's think step by step. According to the Hardy Weinberg Law, $p^2 +\ + \ 2 p q + q^2 = 1$, and $p + q = 1$ where $p$ is the frequency of the dominant\ + \ allele, $q$ is the frequency of the recessive allele, and $p^2$, $q^2$, and\ + \ $2pq$ are the frequencies of dominant homozygous, recessive homozygous, and\ + \ heterozygous individuals, respectively. \u200BThe frequency of the recessive\ + \ allele (q) is $\\sqrt{\frac{1}{400}} = 0.05$. We have $p = 1 - q = 0.95$.\ + \ The frequency of heterozygous individuals is $2pq = 2 \\cdot 0.05 \\cdot 0.95\ + \ = 0.095$. The number of heterozygous individuals is equal to the frequency\ + \ of heterozygous individuals times the size of the population, or $0.095 *\ + \ 400 = 38$. So we end up with 38/400. The answer is (D)." + - question: 'According to the pressure-flow model of movement of phloem contents, photosynthate + movement from source to sink is driven by + + (A) an ATP-dependent pressure-flow pump (B) a water-pressure potential gradient + (C) transpiration (D) apoplastic diffusion' + target: Let's think step by step. It is a gradient in water pressure that induces + the movement of phloem content, which refers to answer (B). The mechanism of + movement does not rely on metabolism, which rules out (A). Transpiration refers + to the exhalation of water vapor through plant stomata, and is also not related, + which rules out (C). While the apoplastic pathway is one of two main pathways + for water transport in plants, it is not central to the pressure flow model, + which rules out (D). The answer is (B). + - question: 'Which of the following contain DNA sequences required for the segregation + of chromosomes in mitosis and meiosis? + + (A) Telomeres (B) Centromeres (C) Nucleosomes (D) Spliceosomes' + target: Let's think step by step. The genetic material in Telomeres is not used, + which rules out (A). Nucleosomes are the repeating subunit that comprises chromatin + packed in a cell nucleus, and do not specifically refer to DNA sequences necessary + for segregating chromosomes in cell division, which rules out (C). A spliceosome + is a large ribonucleoprotein that removes introns from transcribed pre-mRNA + rather than governing chromosome segregation. Centromeres are directly responsible + for segregating chromosomes in cell division. The answer is (B). + - question: 'The presence of homologous structures in two different organisms, such + as the humerus in the front limb of a human and a bird, indicates that + + (A) the human and bird are polyphyletic species (B) a human''s and bird''s evolution + is convergent (C) the human and bird belong to a clade (D) the human and bird + developed by analogy' + target: 'Let''s think step by step. Polyphyletic species are organisms that are + grouped due to having similar characteristics but which do not have a common + ancestor. This is not the case for humans and birds, which rules out (A). Convergent + evolution refers to the indepdendent development of similar features in different + species at different periods, which is also not the case for humans and birds, + which rules out (B). Analogy refers to the superficial resemblance of structures + that have different origins, which is not the case for the human and bird forearms, + which rules out (D). Humans and birds do belong to the same clade - a group + of organisms composed of a common ancestor. The answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a8cfc9e4436f1dc50b75efdf84a0b6f0625f2bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml @@ -0,0 +1,49 @@ +dataset_name: college_chemistry +description: The following are multiple choice questions (with answers) about college + chemistry. +fewshot_config: + sampler: first_n + samples: + - question: "3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq) +\ + \ 4 Cr3+(aq) + 10 H2O(l). In the reaction shown above, Cl\u2212(aq) behaves\ + \ as\n(A) an acid (B) a base (C) a catalyst (D) a reducing agent" + target: Let's think step by step. A molecule that behaves as a base accepts an + H+ ion (or proton) from another molecule, whereas a molecule that behaves as + an acid donates an H+ ion (or proton) to another molecule. Neither of these + is the case for Cl in this reaction, which rules out (A) and (B). A catalyst + is a substance that only accelerates a reaction without itself undergoing chemical + change, which is not the case here. This rules out (C). Instead, the $Cl^{-} + molecules carry a negative charge, which they donate in the reaction to form + 3 HClO2. This is the behavior of a reducing agent, or (D). The answer is (D). + - question: 'Which of the following statements about the lanthanide elements is NOT + true? + + (A) The most common oxidation state for the lanthanide elements is +3. (B) Lanthanide + complexes often have high coordination numbers (> 6). (C) All of the lanthanide + elements react with aqueous acid to liberate hydrogen. (D) The atomic radii + of the lanthanide elements increase across the period from La to Lu.' + target: Let's think step by step. The atomic radii of the lanthanide elements + in fact decrease across the period from La to Lu. Options (A), (B), and (C) + are all true. This means that only (D) is NOT true. The answer is (D). + - question: 'Which of the following lists the hydrides of group-14 elements in order + of thermal stability, from lowest to highest? + + (A) PbH4 < SnH4 < GeH4 < SiH4 < CH4 (B) PbH4 < SnH4 < CH4 < GeH4 < SiH4 (C) + CH4 < SiH4 < GeH4 < SnH4 < PbH4 (D) CH4 < PbH4 < GeH4 < SnH4 < SiH4' + target: Let's think step by step. The thermal stability of group-14 hydrides decreases + as we move from the top of group 14 to the bottom. The order of elements in + the group from top to bottom is C, Si, Ge, Sn, Pb. Therefore in order of increasing + thermal stability we have PbH4, SnH4, GeH4, SiH4, and CH4, or answer (A). The + answer is (A). + - question: "Predict the number of lines in the EPR spectrum of a solution of 13C-labelled\ + \ methyl radical (13CH3\u2022), assuming the lines do not overlap.\n(A) 4 (B)\ + \ 3 (C) 6 (D) 24 (E) 8" + target: "Let's think step by step. The electron paramagnetic resonance spectrum\ + \ will be split by two forms of interactions. The first is the hyperfine interaction\ + \ with the 13C (nuclear spin $I = \nrac{1}{2}$) which will split the spectrum\ + \ into 2 lines. This will be further split into 4 lines by the interaction with\ + \ three equivalent 1H nuclei. The total number of lines is therefore $2 \\cdot\ + \ 4 = 8$. The answer is (E).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5eccde7c6be9620f63bbf3b1de42f32c8e121539 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml @@ -0,0 +1,180 @@ +dataset_name: college_computer_science +description: The following are multiple choice questions (with answers) about college + computer science. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following regular expressions is equivalent to (describes + the same set of strings as) (a* + b)*(c + d)? + + (A) a*(c + d)+ b(c + d) + + (B) a*(c + d)* + b(c + d)* + + (C) a*(c + d)+ b*(c + d) + + (D) (a + b)*c +(a + b)*d' + target: 'Let''s think step by step. We know that: + + 1. (X* + Y)* = (X + Y)* + + 2. X(Y + Z)? = XY + XZ + + Using equation 1 we can rewrite (a* + b)*(c + d)? as: + + 3. (a + b)*(c + d)? + + Using equation 2 we can rewrite equation 3 as: + + (a + b)*c + (a + b)*d The answer is (D).' + - question: 'The Singleton design pattern is used to guarantee that only a single instance + of a class may be instantiated. Which of the following is (are) true of this + design pattern? + + I. The Singleton class has a static factory method to provide its instance. + + II. The Singleton class can be a subclass of another class. + + III. The Singleton class has a private constructor. + + (A) I only + + (B) II only + + (C) III only + + (D) I, II, and III' + target: 'Let''s think step by step. Statement I is a correct statement about a + Singleton, because a Singleton restricts instantiation to a single, static method. + Statement II is also correct, because there is no inherent restriction regarding + the inheritance of a Singleton. Statement III is also correct, because a Singletons + must be instantiated only once, so its constructor is made private to prevent + any construction except via its static factory method. + + Given these facts, statements I, II, and III are all correct. The answer is + (D).' + - question: 'A certain pipelined RISC machine has 8 general-purpose registers R0, R1, + . . . , R7 and supports the following operations: + + ADD Rs1, Rs2, Rd (Add Rs1 to Rs2 and put the sum in Rd) + + MUL Rs1, Rs2, Rd (Multiply Rs1 by Rs2 and put the product in Rd) + + An operation normally takes one cycle; however, an operation takes two cycles + if it produces a result required by the immediately following operation in an + operation sequence. + + Consider the expression AB + ABC + BC, where variables A, B, C are located in + registers R0, R1, R2. If the contents of these three registers must not be modified, + what is the minimum number of clock cycles required for an operation sequence + that computes the value of AB + ABC + BC? + + (A) 5 (B) 6 (C) 7 (D) 8' + target: 'Let''s think step by step. First, we are given that A is in R0, B is + in R1, and C is in R2. + + Next, we can see that we must compute three multiplies (AB, BC, and ABC) and + two adds (AB + ABC, (AB + ABC) + BC) to compute our final answer, resulting + in a minimum of five clock cycles. + + Next, we can see that there is no way to avoid at least one pipeline stall when + computing our final answer, because to compute our final sum we must wait at + least one cycle for the results from the previous stage to be ready. Thus, our + minimum number of cycles must be 6. + + We can verify that we can create a solution that requires only six cycles as + follows: + + compute AB: MUL R0, R1, R3 + + compute BC: MUL R1, R2, R4 + + compute ABC: MUL R3, R4, R5 + + compute AB + BC: ADD R3, R4, R6 + + STALL + + compute AB + ABC + BC: ADD R5, R6, R7 + + So there are 6 cycles. The answer is (B).' + - question: 'A compiler generates code for the following assignment statement. + + G := (A + B) * C - (D + E) * F + + The target machine has a single accumulator and a single-address instruction + set consisting of instructions load, store, add, subtract, and multiply. For + the arithmetic operations, the left operand is taken from the accumulator and + the result appears in the accumulator. The smallest possible number of instructions + in the resulting code is + + (A) 5 (B) 6 (C) 7 (D) 9' + target: 'Let''s think step by step. We can compute the final answer with the following + sequence of operations: + + 1. LOAD D (accumulator = D) + + 2. ADD E (accumulator = D+E) + + 3. MUL F (accumulator = (D+E)*F) + + 4. STORE X (X = (D+E)*F) + + 5. LOAD A (accumulator = A) + + 6. ADD B (accumulator = A+B) + + 7. MUL C (accumulator = (A+B)*C) + + 8. SUB X (accumulator = (A+B)*C - (D+E)*F) + + 9. STORE G (G = (A+B)*C - (D+E)*F) + + This sequence takes 9 instructions. The answer is (D).' + - question: 'Consider a computer design in which multiple processors, each with a private + cache memory, share global memory using a single bus. This bus is the critical + system resource. Each processor can execute one instruction every 500 nanoseconds + as long as memory references are satisfied by its local cache. When a cache + miss occurs, the processor is delayed for an additional 2,000 nanoseconds. During + half of this additional delay, the bus is dedicated to serving the cache miss. + During the other half, the processor cannot continue, but the bus is free to + service requests from other processors. On average, each instruction requires + 2 memory references. On average, cache misses occur on 1 percent of references. + What proportion of the capacity of the bus would a single processor consume, + ignoring delays due to competition from other processors? + + (A) 1/50 (B) 1/27 (C) 1/25 (D) 2/27' + target: 'Let''s think step by step. We know that each instruction requires two + memory references per instruction, and that there is an average cache miss rate + of one percent. + + Thus a given processor has: + + (1 cache miss / 100 references) * (2 references / instruction) = + + (2 cache misses / 100 instructions), so: + + misses_per_instruction = 1 cache miss / 50 instructions. + + Next, we know that each instruction requires 500 nanoseconds when there is no + cache miss, and 500 + 2000 = 2500 nanoseconds when there is a cache miss. Thus: + + 50 instructions / (49 * 500) + (1 * 2500) nanoseconds, so: + + instructions_per_ns = 50 instructions / 27000 nanoseconds. + + Now, we know that each cache miss locks the bus for half of the 2000 nanosecond + cache miss delay, or 1000 nanoseconds, so: + + lock_ns_per_miss = 1000 nanoseconds / cache miss. + + Thus we can see that on average a single processor will lock the bus for: + + lock_ns_per_miss * misses_per_instruction * instructions_per_ns = + + (1000 nanoseconds / cache miss) * (1 cache miss / 50 instructions) * (50 instructions + / 27000 nanoseconds) = 1000 * (1/50) * (50/27000) = 1000/27000 = 1/27. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5552cc35bc55ffab1b53538ee2605778e6f215d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml @@ -0,0 +1,73 @@ +dataset_name: college_mathematics +description: The following are multiple choice questions (with answers) about college + mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Let V be the set of all real polynomials p(x). Let transformations T, + S be defined on V by T:p(x) -> xp(x) and S:p(x) -> p''(x) = d/dx p(x), and interpret + (ST)(p(x)) as S(T(p(x))). Which of the following is true? + + (A) ST = 0 (B) ST = T (C) ST = TS (D) ST - TS is the identity map of V onto + itself.' + target: "Let's think step by step. For a given polynomial $p$ we have\n\\[ST(p)\ + \ = (xp(x))\u2019 = p(x) + xp\u2019(x)\\]\nand\n\\[TS(p) = xp\u2019(x).\\]\n\ + Hence \\[ST(p) - TS(p) = p(x) + xp\u2019(x) - xp\u2019(x).\\] The answer is\ + \ (D)." + - question: 'Suppose that f(1 + x) = f(x) for all real x. If f is a polynomial and + f(5) = 11, then f(15/2) + + (A) -11 (B) 0 (C) 11 (D) 33/2' + target: Let's think step by step. The only polynomial so that $f(1 + x) = f(x)$ + is a constant polynomial. Hence $f(5) = 11 = f(15/2)$. The answer is (C). + - question: 'Let A be a real 2x2 matrix. Which of the following statements must be + true? + + I. All of the entries of A^2 are nonnegative. + + II. The determinant of A^2 is nonnegative. + + III. If A has two distinct eigenvalues, then A^2 has two distinct eigenvalues. + + (A) I only (B) II only (C) III only (D) II and III only' + target: 'Let''s think step by step. We have \[ det(A^2) = (det(A))^2 \geq 0,\] + hence II holds. + + III is false: as a counterexample take a diagonal matrix with -1 and 1 on the + diagonal. Then $A^2$ is the identity matrix. The answer is (B).' + - question: 'Let A be the set of all ordered pairs of integers (m, n) such that 7m + + 12n = 22. What is the greatest negative number in the set B = {m + n : (m, + n) \in A}? + + (A) -5 (B) -4 (C) -3 (D) -2' + target: Let's think step by step. We have 12n = 22 - 7m and one of the solutions + is $m = -2$, $n = 3$. Then $m + n = 1$, hence we need to look for smaller $m$ + in order to make $m + n$ negative. The next solution is $m = -14$ and $n = 10$. + For smaller $m$ we have $m + n$ smaller than $-4$. The answer is (B). + - question: 'A tank initially contains a salt solution of 3 grams of salt dissolved + in 100 liters of water. A salt solution containing 0.02 grams of salt per liter + of water is sprayed into the tank at a rate of 4 liters per minute. The sprayed + solution is continually mixed with the salt solution in the tank, and the mixture + flows out of the tank at a rate of 4 liters per minute. If the mixing is instantaneous, + how many grams of salt are in the tank after 100 minutes have elapsed? + + (A) 2 (B) 2 - e^-2 (C) 2 + e^-2 (D) 2 + e^-4' + target: "Let's think step by step. For all $t \\in \\mathbb{R}$, let $s(t)$ denote\ + \ the number grams of salt in the tank at the $t$ minute mark. Then $s(0) =\ + \ 3$.\nWe use $s$ and $s(t)$ interchangeably. We also use $s^{\\prime}$ and\ + \ $s^{\\prime}(t)$ interchangeably. The solution sprayed into the tank adds\ + \ $(0.02) 4=2 / 25$ grams of salt per minute. There are always 100 liters of\ + \ liquid in the tank, containing $s$ grams of salt. So the density of salt in\ + \ the tank is $s / 100$ grams per liter. The flow of water out of the tank therefore\ + \ subtracts $4(s / 100)=s / 25$ grams of salt per minute. Then, for all $t \\\ + in \\mathbb{R}$, we have $s^{\\prime}(t)=(2 / 25)-(s / 25)=(2-s) / 25$, and\ + \ so $[s(t)=2] \\Rightarrow\\left[s^{\\prime}(t)=0\right]$. For all $t \\in\ + \ \\mathbb{R}$,\n$$\n\frac{d}{d t}[\\ln (s-2)]=\frac{s^{\\prime}}{s-2}=\frac{-1}{25}=\f\ + rac{d}{d t}\\left[-\frac{t}{25}\right] .\n$$\nChoose $C \\in \\mathbb{R}$ such\ + \ that, for all $t \\in \\mathbb{R}, \\ln ((s(t)-2))=-[t / 25]+C$. Let $K:=e^{C}$.\ + \ Then, for all $t \\in \\mathbb{R}$, we have $(s(t))-2=K e^{-t / 25}$, and\ + \ so $s(t)=2+K e^{-t / 25}$. Then $3=s(0)=2+K e^{0}=2+K$, so $K=1$. Then $s(100)=2+K\ + \ e^{-100 / 25}=2+1 \\cdot e^{-4}=2+e^{-4}$. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7eac0bab3f9286469b44e815c8a2090fc5ff0832 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml @@ -0,0 +1,68 @@ +dataset_name: college_medicine +description: The following are multiple choice questions (with answers) about college + medicine. +fewshot_config: + sampler: first_n + samples: + - question: 'An expected side effect of creatine supplementation is: + + (A) muscle weakness. (B) gain in body mass. (C) muscle cramps. (D) loss of electrolytes.' + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. Creatine supplementation is a dietary supplement that results in body + mass gain. The answer is (B). + - question: 'Which of the following is not a true statement? + + (A) Muscle glycogen is broken down enzymatically to glucose-1-phosphate (B) + Elite endurance runners have a high proportion of Type I fibres in their leg + muscles (C) Liver glycogen is important in the maintenance of the blood glucose + concentration (D) Insulin promotes glucose uptake by all tissues in the body' + target: "Let's think step by step. We refer to Wikipedia articles on medicine\ + \ for help. Let\u2019s solve this step by step and go over each choice: \n(A)\ + \ \u201CMuscle glycogen is broken down enzymatically to glucose-1-phosphate\u201D\ + : This is a correct statement.\n(B) \u201CElite endurance runners have a high\ + \ proportion of Type I fibres in their leg muscles\u201D: This is a correct\ + \ statement.\n(C) \u201CLiver glycogen is important in the maintenance of the\ + \ blood glucose concentration\u201D: This is a correct statement. \n(D) \u201C\ + Insulin promotes glucose uptake by all tissues in the body\u201D: This is not\ + \ a correct statement, because insulin promotes glucose uptake by the liver,\ + \ adipose tissue, and muscle, but not all tissues. For instance, the tissues\ + \ in the brain and red blood cells are not affected by insulin. The answer is\ + \ (D)." + - question: "A high school science teacher fills a 1 liter bottle with pure nitrogen\ + \ and seals the lid. The pressure is 1.70 atm, and the room temperature is 25\xB0\ + C. Which two variables will both increase the pressure of the system, if all\ + \ other variables are held constant?\n(A) Increasing temperature, increasing\ + \ moles of gas (B) Increasing temperature, increasing volume (C) Decreasing\ + \ volume, decreasing temperature (D) Decreasing moles of gas, increasing volume" + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. The relevant equation for this is the ideal gas law: PV=nRT. To increase + the pressure of the system (P), then either n (number of moles of the gas) or + T (temperature) have to increase. The answer is (A).' + - question: 'In a genetic test of a newborn, a rare genetic disorder is found that + has X-linked recessive transmission. Which of the following statements is likely + true regarding the pedigree of this disorder? + + (A) All descendants on the maternal side will have the disorder. (B) Females + will be approximately twice as affected as males in this family. (C) All daughters + of an affected male will be affected. (D) There will be equal distribution of + males and females affected.' + target: "Let's think step by step. We refer to Wikipedia articles on medicine\ + \ for help. Let\u2019s solve this step by step. Let's recall first that females\ + \ have two X chromosomes, while males have one X and one Y chromosome. This\ + \ is an important fact we need to know before answering this question. \nBecause\ + \ a male can only pass his only one X chromosome to a daughter, if he is affected\ + \ by this rare genetic disorder, then we know for sure that he will pass this\ + \ rare genetic disorder to all his future-born daughters. Therefore, \u201C\ + (C): All daughters of an affected male will be affected\u201D is a correct statement.\ + \ The answer is (C)." + - question: 'Glucose is transported into the muscle cell: + + (A) via protein transporters called GLUT4. (B) only in the presence of insulin. + (C) via hexokinase. (D) via monocarbylic acid transporters.' + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. Glucose (also known as the blood sugar) is the main sugar found in + the human body. It is transported into the muscle cell via diffusion through + protein transporters called GLUT4. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa158a9f3c3e9e420608970a3ea91e744a042a6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml @@ -0,0 +1,61 @@ +dataset_name: college_physics +description: The following are multiple choice questions (with answers) about college + physics. +fewshot_config: + sampler: first_n + samples: + - question: 'A refracting telescope consists of two converging lenses separated by + 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification + of the telescope is + + (A) 4 (B) 5 (C) 6 (D) 20' + target: Let's think step by step. In a refracting telescope, if both lenses are + converging, the focus of both lenses must be between the two lenses, and thus + the focal lengths of the two lenses must add up to their separation. Since the + focal length of one lens is 20 cm, the focal length of the other must be 80 + cm. The magnification is the ratio of these two focal lengths, or 4. The answer + is (A). + - question: 'The muon decays with a characteristic lifetime of about 10^-6 second into + an electron, a muon neutrino, and an electron antineutrino. The muon is forbidden + from decaying into an electron and just a single neutrino by the law of conservation + of + + (A) charge (B) mass (C) energy and momentum (D) lepton number' + target: Let's think step by step. Lepton number must be conserved, meaning the + total number of leptons minus the number of antileptons. If a muon decays into + an electron and a single neutrino, the total lepton number would go from one + to two, violating lepton number conservation. The answer is (D). + - question: 'One end of a Nichrome wire of length 2L and cross-sectional area A is + attached to an end of another Nichrome wire of length L and cross- sectional + area 2A. If the free end of the longer wire is at an electric potential of 8.0 + volts, and the free end of the shorter wire is at an electric potential of 1.0 + volt, the potential at the junction of the two wires is most nearly equal to + + (A) 2.4 V (B) 3.3 V (C) 4.5 V (D) 5.7 V' + target: Let's think step by step. This is a simple voltage divider problem, where + the longer wire has a resistance four times that of the shorter end. So the + voltage divider ratio is 1 / 5, meaning that the potential in the middle is + 1.0 V + (8.0 V - 1.0 V) * 1/5 = 2.4 V. The answer is (A). + - question: 'A refracting telescope consists of two converging lenses separated by + 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification + of the telescope is + + (A) 4 (B) 5 (C) 6 (D) 20' + target: Let's think step by step. In a refracting telescope, if both lenses are + converging, the focus of both lenses must be between the two lenses, and thus + the focal lengths of the two lenses must add up to their separation. Since the + focal length of one lens is 20 cm, the focal length of the other must be 80 + cm. The magnification is the ratio of these two focal lengths, or 4. The answer + is (A). + - question: 'For which of the following thermodynamic processes is the increase in + the internal energy of an ideal gas equal to the heat added to the gas? + + (A) Constant temperature (B) Constant volume (C) Constant pressure (D) Adiabatic' + target: 'Let''s think step by step. Heat added to the gas can go into the gases + internal energy or work done against an external force. However, if the volume + of the gas container is constant, no work will be done (since work is pressure + times change in volume). So, at constant volume, all of the heat goes into the + internal energy. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a7c5cb2d5d1a2943b5b1ad6c17bf5ce0a324f5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml @@ -0,0 +1,50 @@ +dataset_name: computer_security +description: The following are multiple choice questions (with answers) about computer + security. +fewshot_config: + sampler: first_n + samples: + - question: 'SHA-1 has a message digest of + + (A) 160 bits (B) 512 bits (C) 628 bits (D) 820 bits' + target: Let's think step by step. Since SHA-1 is a hash function which takes an + question and produces a 160-bit (20-byte) hash value, its message digest is 160 + bits. The answer is (A). + - question: "_____________ can modify data on your system \u2013 so that your system\ + \ doesn\u2019t run correctly or you can no longer access specific data, or it\ + \ may even ask for ransom in order to give your access.\n(A) IM \u2013 Trojans\ + \ (B) Backdoor Trojans (C) Trojan-Downloader (D) Ransom Trojan" + target: Let's think step by step. The system is asking for trojans, which are + for ransom, which means ransom trojan. The answer is (D). + - question: 'What is ethical hacking? + + (A) "Hacking" ethics so they justify unintended selfish behavior (B) Hacking + systems (e.g., during penetration testing) to expose vulnerabilities so they + can be fixed, rather than exploited (C) Hacking into systems run by those whose + ethics you disagree with (D) A slang term for rapid software development, e.g., + as part of hackathons' + target: Let's think step by step. Ethical hacking is a process of detecting vulnerabilities + in an application, system, or organization's infrastructure that an attacker + can use to exploit an individual or organization. They use this process to prevent + cyberattacks and security breaches by lawfully hacking into the systems and + looking for weak points. The answer is (B). + - question: 'The ____________ is anything which your search engine cannot search. + + (A) Haunted web (B) World Wide Web (C) Surface web (D) Deep Web' + target: "Let's think step by step. The search engine searches on the Surface Web,\ + \ which is the portion of the world wide web which is visible so (B,C) are wrong.\ + \ The Haunted Web doesn\u2019t correspond to an internet concept. The Deep Web\ + \ is the part of the World Wide Web which is not indexed. The answer is (D)." + - question: 'Exploitation of the Heartbleed bug permits + + (A) overwriting cryptographic keys in memory (B) a kind of code injection (C) + a read outside bounds of a buffer (D) a format string attack' + target: 'Let''s think step by step. The Heartbleed Bug is a serious vulnerability + in the popular OpenSSL cryptographic software library. Heartbleed resulted from + improper question validation (due to a missing bounds check) in the implementation + of the TLS heartbeat extension. The vulnerability was classified as a buffer + over-read, a situation where more data can be read than should be allowed. The + answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4757faf1d490e312ea70a3d4dbe291459366471 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml @@ -0,0 +1,49 @@ +dataset_name: conceptual_physics +description: ' + + The following are multiple choice questions (with answers) about conceptual physics.' +fewshot_config: + sampler: first_n + samples: + - question: 'Colors in a soap bubble result from light + + (A) converted to a different frequency (B) deflection (C) interference (D) polarization' + target: Let's think step by step. In a soap bubble film, the light bounces between + the two soap-air interfaces many times, interfering with itself constructively + or destructively depending on the width of the film. This results in different + colors being visible. The answer is (C). + - question: 'Compared with the mass of a uranium atom undergoing fission, the combined + masses of the products after fission are + + (A) less (B) more (C) the same (D) zero' + target: Let's think step by step. Fission releases energy, which comes from the + rest mass of its initial nucleus. Thus the mass of the products is less than + the mass of the reactant uranium nucleus. The answer is (A). + - question: 'Things that are equivalent according to the equivalence principle are + + (A) space and time. (B) a traveling twin and a stay-at-home twin. (C) gravity + and acceleration. (D) mass and energy.' + target: "Let's think step by step. Einstein\u2019s famous equivalence principle\ + \ states that gravity and acceleration are equivalent. The answer is (C)." + - question: 'Which of these three elements has the most mass per nucleon? + + (A) Hydrogen (B) Iron (C) Uranium (D) Same in each' + target: Let's think step by step. Due to nuclear binding energy, the mass of an + atomic nucleus is less than the sum of individual masses of the free constituent + protons and neutrons; this is known as the mass defect. Hydrogen has no mass + defect because it has only a single nucleon, so it will have the most mass per + nucleon. The answer is (A). + - question: 'A model airplane flies slower when flying into the wind and faster with + wind at its back. When launched at right angles to the wind a cross wind its + groundspeed compared with flying in still air is + + (A) the same (B) greater (C) less (D) either greater or less depending on wind + speed' + target: "Let's think step by step. The plane\u2019s speed in the direction of\ + \ the wind is greater than it would be in the absence of wind, and its direction\ + \ orthogonal to the wind is the same as it would be in the absence of the wind.\ + \ The total speed, which is these two components added in quadrature, is thus\ + \ greater than the speed in still air. The answer is (B).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4282345ce0f2e8bab97a80413fbd2b796a7fe3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml @@ -0,0 +1,87 @@ +dataset_name: econometrics +description: The following are multiple choice questions (with answers) about econometrics. +fewshot_config: + sampler: first_n + samples: + - question: 'Suppose now that a researcher wishes to use information criteria to determine + the optimal lag length for a VAR. 500 observations are available for the bi-variate + VAR, and the values of the determinant of the variance-covariance matrix of + residuals are 0.0336, 0.0169, 0.0084, and 0.0062 for 1, 2, 3, and 4 lags respectively. + What is the optimal model order according to Akaike''s information criterion? + + (A) 1 lag (B) 2 lags (C) 3 lags (D) 4 lags' + target: "Let's think step by step. We refer to Wikipedia articles on econometrics\ + \ for help. Let\u2019s solve this problem step by step. First of all, let\u2019\ + s recall that for a given set of data, Akaike's information criterion (AIC)\ + \ allows us to measure how well a statistical model fits the data; it is an\ + \ estimator of prediction error. Here in this problem we will need to use the\ + \ formula ln(det(sigma_hat)) + (2 * k / T) to determine the values of Akaike\u2019\ + s criterion, where ln denotes the natural log function, det the determinant\ + \ function, k the total number of parameters in total (across both equations),\ + \ and T the number of observations (which, in this case, is equal to 500). For\ + \ 1 lag, the number of parameters in total is equal to 6; for 2 lags, it is\ + \ 10; for 3 lags, it is 14; and for 4 lags, it is 18. Now, let\u2019s calculate\ + \ the values of the criterion for each lag:\n(A) 1 lag: ln(0.0336) + (2 * 6\ + \ / 500) = ln(0.0336) + (12 / 500) = -3.369\n(B) 2 lags: ln(0.0169) + (2 * 10\ + \ / 500) = ln(0.0169) + (20 / 500) = -4.040\n(C) 3 lags: ln(0.0084) + (2 * 14\ + \ / 500) = ln(0.0084) + (28 / 500) =-4.724\n(D) 4 lags: ln(0.0062) + (2 * 18\ + \ / 500) = ln(0.0062) + (36 / 500) =-5.011\nBecause the optimal model order\ + \ according to AIC minimizes the information criterion, the answer should be\ + \ the one with the lowest value. In this case, (D) has the lowest value. The\ + \ answer is (C)." + - question: 'Consider the following AR(1) model with the disturbances having zero mean + and unit variance + + yt = 0.2 + 0.4 yt-1 + ut + + The (unconditional) mean of y will be given by + + (A) 0.2 (B) 0.4 (C) 0.5 (D) 0.33' + target: "Let's think step by step. We refer to Wikipedia articles on econometrics\ + \ for help. Let\u2019s solve this problem step by step. If we have a an AR(1)\ + \ model with the disturbances having zero mean and unit variance, then the unconditional\ + \ mean of y is equal to the following:\nunconditional mean of y = (the intercept\ + \ term) / (1 - autoregressive coefficient)\nWe know that the intercept term\ + \ is 0.2 and the autoregressive coefficient is 0.4; thus, we have:\nunconditional\ + \ mean of y = (0.2) / (1 - 0.4) = (0.2) / (0.6) = 2 / 6 = 1 / 3, which is approximately\ + \ 0.33. That means that the answer should be (D) 0.33. The answer is (D)." + - question: 'What would be then consequences for the OLS estimator if heteroscedasticity + is present in a regression model but ignored? + + (A) It will be biased (B) It will be inconsistent (C) It will be inefficient + (D) All of (a), (b) and (c) will be true.' + target: Let's think step by step. We refer to Wikipedia articles on econometrics + for help. Heteroscedasticity refers to the condition where the variance of the + error terms is not constant across multiple observations. If heteroscedasticity + is present in a regression model, then the coefficient estimates in the OLS + estimator will be not only unbiased and consistent but also inefficient. Because + (A) and (B) are incorrect choices and (C) is a correct choice, (D) cannot be + the right answer. Ultimately, (C) is the only true choice. The answer is (C). + - question: 'Suppose that a test statistic has associated with it a p-value of 0.08. + Which one of the following statements is true? + + (i) If the size of the test were exactly 8%, we would be indifferent between + rejecting and not rejecting the null hypothesis + + (ii) The null would be rejected if a 10% size of test were used + + (iii) The null would not be rejected if a 1% size of test were used + + (iv) The null would be rejected if a 5% size of test were used. + + (A) (ii) and (iv) only (B) (i) and (iii) only (C) (i), (ii), and (iii) only + (D) (i), (ii), (iii), and (iv).' + target: "Let's think step by step. We refer to Wikipedia articles on econometrics\ + \ for help. Let\u2019s reason about each of the options.\n(i) is a true statement.\n\ + (ii) is a true statement.\n(iii) is a true statement.\n(iv) is not a true statement.\ + \ Thus, (i), (ii), and (iii) are true. The answer is (C)." + - question: 'For a stationary autoregressive process, shocks will + + (A) Eventually die away (B) Persist indefinitely (C) Grow exponentially (D) + Never occur' + target: 'Let''s think step by step. We refer to Wikipedia articles on econometrics + for help. This is a formal logic problem about stationally process. For a stationary + autoregressive process, shocks will eventually die away. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..305d2340c5ffa69761ef8dc2ab128849e571bbf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml @@ -0,0 +1,47 @@ +dataset_name: electrical_engineering +description: ' + + The following are multiple choice questions (with answers) about electrical engineering.' +fewshot_config: + sampler: first_n + samples: + - question: "A point pole has a strength of 4\u03C0 * 10^-4 weber. The force in newtons\ + \ on a point pole of 4\u03C0 * 1.5 * 10^-4 weber placed at a distance of 10\ + \ cm from it will be\n(A) 15 N. (B) 20 N. (C) 7.5 N. (D) 3.75 N." + target: "Let's think step by step. The force between two point poles is given\ + \ by m_1m_2/(mu_0 4 \\pi r^2), in analogy to Coulomb\u2019s law. Plugging in\ + \ the values given in the question, we calculate that the force is approximately\ + \ 15 N. The answer is (A)." + - question: 'The coil of a moving coil meter has 100 turns, is 40 mm long and 30 mm + wide. The control torque is 240*10-6 N-m on full scale. If magnetic flux density + is 1Wb/m2 range of meter is + + (A) 1 mA. (B) 2 mA. (C) 3 mA. (D) 4 mA.' + target: Let's think step by step. The torque on a coil in a uniform magnetic field + is given by BANI, where B is the magnetic flux density, A is the area of the + coil, N is the number of turns, and I is the current. So we have that I = (Torque)/(BAN), + or 240e-6/(1200e-6 * 100 * 1) = 2e-3. The answer is (B). + - question: 'In an SR latch built from NOR gates, which condition is not allowed + + (A) S=0, R=0 (B) S=0, R=1 (C) S=1, R=0 (D) S=1, R=1' + target: Let's think step by step. An SR latch is a set-reset latch; in the case + where S=1 and R=1, the circuit has no stable state; instead a race condition + will be produced within the circuit, so the device will be in an undefined state. + So S=1, R=1 is an illegal question. The answer is (D). + - question: 'Two long parallel conductors carry 100 A. If the conductors are separated + by 20 mm, the force per meter of length of each conductor will be + + (A) 100 N. (B) 0.1 N. (C) 1 N. (D) 0.01 N.' + target: Let's think step by step. The magnetic force-per-length between two current-carrying + conductors is given by \mu_0 I_1 I_2 / (2 \pi r), where $r$ is the separation + distance and I_1 and I_2 are the currents. Plugging in 100 A for I_1 and I_2, + and 20 mm for r, gives 0.1 N. The answer is (B). + - question: "In a 2 pole lap winding dc machine , the resistance of one conductor is\ + \ 2\u03A9 and total number of conductors is 100. Find the total resistance\n\ + (A) 200\u03A9 (B) 100\u03A9 (C) 50\u03A9 (D) 10\u03A9" + target: 'Let''s think step by step. In lap winding, effectively two resistors + are connected in parallel, so the actual resistance of each pair is 1 Ohm. Since + we have 50 pairs, we get a total resistance of 50 Ohms. The answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fb9eef860424b49adf108566edadaf1500dbf5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml @@ -0,0 +1,77 @@ +dataset_name: elementary_mathematics +description: The following are multiple choice questions (with answers) about elementary + mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Olivia used the rule "Add 11" to create the number pattern shown below. + 10, 21, 32, 43, 54. Which statement about the number pattern is true? + + (A) The 10th number in the pattern will be an even number. + + (B) The number pattern will never have two even numbers next to each other. + + (C) The next two numbers in the pattern will be an even number then an odd number. + + (D) If the number pattern started with an odd number then the pattern would + have only odd numbers in it.' + target: Let's think step by step. Choice A is incorrect because every even-numbered + term in the pattern is odd, and 10 is an even number. Choice B is correct, because + adding an odd number (in this case 11) to an odd number produces an even number, + and adding an odd number to an even number produces an odd number. Thus the + terms in the pattern will alternate between odd and even, so there will never + be two even numbers next to each other. Choice C is incorrect because the last + term in the example is even (54), and we know that the terms will alternate + between even and odd. Choice D is incorrect because the terms in the pattern + will alternate between odd and even, regardless of the value of the first term. + The answer is (B). + - question: 'The population of the city where Michelle was born is 145,826. What is + the value of the 5 in the number 145,826? + + (A) 5 thousands + + (B) 5 hundreds + + (C) 5 tens + + (D) 5 ones' + target: 'Let''s think step by step. Choice A is correct, because there are three + digits following the 5, so + + the 5 is in the thousands place. Thus the other choices are incorrect. The answer + is (A).' + - question: "A store sells 107 different colors of paint. They have 25 cans of each\ + \ color in storage. The number of cans of paint the store has in storage can\ + \ be found using the expression below. 107 \xD7 25. How many cans of paint does\ + \ the store have in storage?\n(A) 749\n(B) 2,675\n(C) 2,945\n(D) 4,250" + target: Let's think step by step. We can calculate 107 x 25 = (100 x 25) + (7 + x 25) = 2500 + 175 = 2675. The answer is (B). + - question: 'A total of 30 players will play basketball at a park. There will be exactly + 5 players on each team. Which statement correctly explains how to find the number + of teams needed? + + (A) Add 5 to 30 to find 35 teams. + + (B) Divide 30 by 5 to find 6 teams. + + (C) Multiply 30 and 5 to find 150 teams. + + (D) Subtract 5 from 30 to find 25 teams.' + target: Let's think step by step. We want to find the number of teams. We know + that there are 5 players/team, and 30 players. Thus to get the number of teams + we divide players by players/team, so 30 players / 5 players/team = 6 teams. + The answer is (B). + - question: 'Which expression is equivalent to 5 x 9? + + (A) (5 x 4) x (6 x 5) + + (B) (5 x 5) + (5 x 4) + + (C) (5 x 5) + (5 x 9) + + (D) (5 x 9) x (6 x 9)' + target: 'Let''s think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5 + + 4) = (5 x 5) + (5 x 4). The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fbc73c3d24f9cef06a41bbcfddea55aec1b424a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml @@ -0,0 +1,70 @@ +dataset_name: formal_logic +description: The following are multiple choice questions (with answers) about formal + logic. +fewshot_config: + sampler: first_n + samples: + - question: "Which of the given formulas of PL is the best symbolization of the following\ + \ sentence?\nTurtles live long lives and are happy creatures, unless they are\ + \ injured.\n(A) (L \u2022 H) \u2261 I (B) (L \u2022 H) \u2228 I (C) L \u2022\ + \ (H \u2228 I) (D) L \u2022 (H \u2283 R)." + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CL\u201D denote \u201C\ + living long\u201D, H \u201Cbeing happy\u201D, and \u201CI\u201D \u201Cbeing\ + \ injured\u201D. Now, consider each choice:\n(A) means (living long AND being\ + \ happy) is equivalent to (being injured). \n(B) means (living long AND being\ + \ happy) OR (being injured). \n(C) means (living long) AND (being happy OR being\ + \ injured). \n(D) means (living long) AND (being happy implies being R), but\ + \ what R denotes is not clear.\nObviously, (B) is the best symbolization of\ + \ the original sentence. The answer is (B)." + - question: 'Select the best translation into predicate logic.George borrows Hector''s + lawnmower. (g: George; h: Hector; l: Hector''s lawnmower; Bxyx: x borrows y + from z). + + (A) Blgh (B) Bhlg (C) Bglh (D) Bghl' + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. We are told that \u201CBxyx\u201D\ + \ means \u201Cx borrows y from z\u201D. We can rewrite \u201CGeorge borrows\ + \ Hector's lawnmower\u201D as \u201CGeorge borrows a lawnmower from Hector\u201D\ + , which can then be translated into predicate logic as \u201CBglh\u201D. The\ + \ answer \u201CBglh\u201D appears in (C); therefore, (C) must be the correct\ + \ answer. The answer is (C)." + - question: "\nSelect the best English interpretation of the given arguments in predicate\ + \ logic.\nDm\n(\u2200x)(Wx \u2283 ~Dx). \n(\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax\n\ + (A) Marina is a dancer. Some weaklings are not dancers. Either everything is\ + \ a weakling or Georgia plays volleyball. So something plays volleyball. (B)\ + \ Marina is a dancer. No weakling is a dancer. Everything is either a weakling\ + \ or plays volleyball. So something plays volleyball. (C) Marina is a dancer.\ + \ Some weaklings are not dancers. Everything is either a weakling or plays volleyball.\ + \ So something plays volleyball. (D) Marina is a dancer. No weakling is a dancer.\ + \ Either everything is a weakling or Georgia plays volleyball. So something\ + \ plays volleyball." + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CD\u201D denote \u201C\ + being a dancer\u201D, \u201Cm\u201D denote \u201CMaria\u201D, \u201Cg\u201D\ + \ denote \u201CGeorgia\u201D, \u201CW\u201D denote \u201Cweakling\u201D, \u201C\ + A\u201D denote \u201Cplaying volleyball\u201D. Then, we have the following:\n\ + 1. Dm \u2192 Maria is a dance.\n2. (\u2200x)(Wx \u2283 ~Dx). \u2192 For all\ + \ x, if x is a weakling, then x is not a dancer. In other words, no weakling\ + \ is a dancer.\n3. (\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax \u2192 For all x, x\ + \ is a weakling or Georgia plays volleyball. So there exists an x that plays\ + \ volleyball. \nOptions (A) and (C) do claim that some weaklings are not dancers,\ + \ but the second argument strongly states that no weakling is a dancer. Thus,\ + \ we can eliminate them. Option (B) omits the important detail about Georgia\ + \ playing volleyball. Option (D) has all the details presented in the arguments\ + \ and is the best English interpretation of the arguments. The answer is (D)." + - question: "Select the best translation into predicate logic: No people drive on Mars.\n\ + (A) ~Pd (B) (\u2200x)(Px \u2228 ~Dx) (C) (\u2200x)(Px \u2283 ~Dx) (D) ~Dp" + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CP\u201D denote \u201C\ + being on Mars\u201D and \u201CD\u201D denote \u201Cdriving on Mars\u201D. Then\ + \ let\u2019s consider each option:\nOption (A): ~Pd \u2192 d is not on Mars.\n\ + Option (B): (\u2200x)(Px \u2228 ~Dx) \u2192 For all x, x is on Mars and x do\ + \ not drive on Mars.\nOption (C): (\u2200x)(Px \u2283 ~Dx) \u2192 For all x,\ + \ x is on Mars implies that x do not drive on Mars.\nOption (D): ~Dp: \u2192\ + \ p do not drive on Mars.\nOf all these options, Option (C) appears to be the\ + \ best and most meaningful interpretation of the argument \u201CNo people drive\ + \ on Mars.\u201D The answer is (C).\n\n" +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..739c00106536c8df64213d55b83c053db1314124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml @@ -0,0 +1,49 @@ +dataset_name: global_facts +description: The following are multiple choice questions (with answers) about global + facts. +fewshot_config: + sampler: first_n + samples: + - question: "As of 2017, how many of the world\u2019s 1-year-old children today have\ + \ been vaccinated against some disease? *\n(A) 80% (B) 60% (C) 40% (D) 20%" + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. According to data published by the World Health Organization, the + nummber of 1-year-old children vaccinated in 2017 exceeds 80%. The answer is + (A). + - question: 'As of 2019, about what percentage of Americans agree that the state is + run for the benefit of all the people? + + (A) 31% (B) 46% (C) 61% (D) 76%' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. In 2019, about 46% percentage of Americans agree that the state is + run for the benefit of all the people. The answer is (B). + - question: 'As of 2019, about what percentage of Russians say it is very important + to have free media in our country without government/state censorship? + + (A) 38% (B) 53% (C) 68% (D) 83%' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. As of 2019, about 38% of Russians say it is very important to have + free media in our country. The answer is (A). + - question: 'As of 2015, since 1990 forests have ____ in Europe and have ____ in Africa + and the Americas. + + (A) increased, increased (B) increased, decreased (C) decreased, increased (D) + decreased, decreased' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. As of 2015, since 1990 forests have increased in Europe and have decreased + in Africa and the Americas. The answer is (B). + - question: 'Which of the following pairs of statements are both true (as of 2019)? + + (A) People tend to be optimistic about their own future and the future of their + nation or the world. (B) People tend to be optimistic about their own future + but pessimistic about the future of their nation or the world. (C) People tend + to be pessimistic about their own future but optimistic about the future of + their nation or the world. (D) People tend to be pessimistic about their own + future and the future of their nation or the world.' + target: 'Let''s think step by step. We refer to Wikipedia articles on global facts + for help. As of 2019, most people tend to be optimistic about their own future + but pessimistic about the future of their nation or the world. The answer is + (B).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dfb19f924761c6dd56cc6b3b9ada38b5bf473e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml @@ -0,0 +1,69 @@ +dataset_name: high_school_biology +description: The following are multiple choice questions (with answers) about high + school biology. +fewshot_config: + sampler: first_n + samples: + - question: "In animal cells, which of the following represents the most likely pathway\ + \ that a secretory protein takes as it is synthesized in a cell?\n(A) Plasma\ + \ membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough\ + \ ER (B) Ribosome\u2013Golgi apparatus\u2013rough ER\u2013secretory vesicle\u2013\ + plasma membrane (C) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013\ + secretory vesicle\u2013rough ER (D) Ribosome\u2013rough ER\u2013Golgi apparatus\u2013\ + secretory vesicle\u2013plasma membrane" + target: Let's think step by step. Protein synthesis starts at the ribosome, so + we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum + and moves from there to the Golgi apparatus, where it is modified and packaged + into a vesicle. The vesicle then floats to the plasma membrane and is secreted. + The answer is (D). + - question: "A mutation in a bacterial enzyme changed a previously polar amino acid\ + \ into a nonpolar amino acid. This amino acid was located at a site distant\ + \ from the enzyme\u2019s active site. How might this mutation alter the enzyme\u2019\ + s substrate specificity?\n(A) By changing the enzyme\u2019s pH optimum (B) By\ + \ changing the enzyme\u2019s location in the cell (C) By changing the shape\ + \ of the protein (D) An amino acid change away from the active site cannot alter\ + \ the enzyme\u2019s substrate specificity." + target: Let's think step by step. A change in an amino acid leads to a change + in the primary structure of the protein. A change in the primary structure may + lead to a change in the secondary and the tertiary structure of the protein. + A change in the tertiary structure means a change in the shape of the protein, + so (C) has to be correct. Since the change does not affect the active site of + the enzyme, we do not expect the activity of the enzyme to be affected. The + answer is (C). + - question: 'Which of the following is not a way to form recombinant DNA? + + (A) Translation (B) Conjugation (C) Specialized transduction (D) Transformation' + target: 'Let''s think step by step. The introduction of foreign DNA or RNA into + bacteria or eukaryotic cells is a common technique in molecular biology and + scientific research. There are multiple ways foreign DNA can be introduced into + cells including transformation, transduction, conjugation, and transfection. + In contrast, (A) is not a way to form DNA: during translation the ribosomes + synthesize proteins from RNA. The answer is (A).' + - question: 'Homologous structures are often cited as evidence for the process of natural + selection. All of the following are examples of homologous structures EXCEPT + + (A) the wings of a bird and the wings of a bat (B) the flippers of a whale and + the arms of a man (C) the pectoral fins of a porpoise and the flippers of a + seal (D) the forelegs of an insect and the forelimbs of a dog' + target: "Let's think step by step. \u200B\u200BHomologous structures are similar\ + \ physical features in organisms that share a common ancestor \u200B\u200Bbut\ + \ different functions. Comparisons (B) and (C) are clearly homologous because\ + \ they share a common ancestor and the structures serve different purposes.\ + \ Bat wings and birg wings are also homologous, while they are both wings, the\ + \ forelimbs serve different purposes. Insects and dogs are very far ancestors\ + \ since one is vertebrate while the other is invertebrate and the forelimbs\ + \ serve the same purpose, so they are not homologous. The answer is (D)." + - question: 'Which of the following is not known to be involved in the control of cell + division? + + (A) Cyclins (B) Protein kinases (C) Checkpoints (D) Fibroblast cells' + target: 'Let''s think step by step. Normal cells move through the cell cycle in + a regulated way. At the checkpoint stage, they use information about their own + internal state and cues from the environment around them to decide whether to + proceed with cell division. Cues like these act by changing the activity of + core cell cycle regulators inside the cell. The most common regulators are cyclins + and cyclin-dependent kinases. Fibroblast cells do not play any role in cell + division. The answer is (D).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..948c375824506cc188ed821c4512725bfdc92d06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml @@ -0,0 +1,66 @@ +dataset_name: high_school_chemistry +description: The following are multiple choice questions (with answers) about high + school chemistry. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is considered an acid anhydride? + + (A) HCl (B) H2SO3 (C) SO2 (D) Al(NO3)3' + target: Let's think step by step. An acid anhydride is a compound that is derived + by removing water from an acid. The chemical formula for water is H2O, which + means that we need to determine which of these options, when combined with H2O, + forms an acid. SO2, or Sulfur dioxide, when combined with H2O, makes H2SO4, + or sulfuric acid. The answer is (C). + - question: 'Which of the following is expected to be a polar molecule? + + (A) PCl4F (B) BF3 (C) CO2 (D) Si(CH3)4' + target: Let's think step by step. A polar molecule is one that has a slightly + positive charge on one end of the molecule and a slightly negative charge on + the other end. Boron trifluoride (BF3) has Boron as the center atom and three + fluorine atoms attached to it; it is trigonal planar and symmetric, so it is + nonpolar. Carbon Dioxide (CO2) has Carbon as the central atom with double bonds + to two Oxygen atoms - this is also symmetrical and therefore nonpolar. The same + is the case for tetramethyl silane (SI(CH3)4), which is a Silicon atom surrounded + by four methyl groups. The structure of PCL4F is that Phosphorus is the central + atom, attached to four chlorines and one fluorine atom. This is asymmetrical, + and therefore has a net dipole and is expected to be a polar molecule. The answer + is (A). + - question: 'From the solubility rules, which of the following is true? + + (A) All chlorides, bromides, and iodides are soluble (B) All sulfates are soluble + (C) All hydroxides are soluble (D) All ammonium-containing compounds are soluble' + target: Let's think step by step. The chlorides, bromides, and iodides of lead, + silver, and mercury are not soluble in water. This rules out (A). The sulfates + of lead, barium, and calcium are not soluble in water, which rules out (B). + The hydroxides of any metal besides sodium, potassium, ammonium, calcium, and + barium are insoluble. This rules out (C). Typically ammonium ions indicate a + soluble ionic substance. The answer is (D). + - question: 'A new compound is synthesized and found to be a monoprotic acid with a + molar mass of 248 g/mol. When 0.0050 mol of this acid are dissolved in 0.500 + L of water, the pH is measured as 3.89. What is the pKa of this acid? + + (A) 3.89 (B) 7.78 (C) 5.78 (D) 2.33' + target: "Let's think step by step. Recall that $[A] = [H^{+}]$. Here, this is\ + \ equal to $$10^{-3.89}$. Then we have $K_{a} = $\nrac{[H^{+}][A^{-}]}{[HA]}\ + \ = \nrac{10^{-3.89} \\cdot 10^{-3.89}}{10^{-2}}. The resulting exponent is\ + \ $-3.89 + (-3.89) - (-2) = 5.78$, therefore $K_a = 10^{-5.78}$. The $pK_a$\ + \ is the negative log of $K_a$, which is equal to $5.78$. The answer is (C)." + - question: 'A solution contains 2.00 mole of acetic acid, CH3COOH, and 1.00 mole of + calcium acetate, Ca(CH3COO)2. The solution is able to resist the addition of + a small amount of strong acid or strong base with only minor changes in the + pH of the solution. Larger quantities of strong acid or strong base can cause + a significant change in pH. How many moles of nitric acid, HNO3, may be added + before the pH begins to change significantly? + + (A) 0.500 mole (B) 1.00 mole (C) 2.00 mole (D) 3.00 mole' + target: "Let's think step by step. We would like to compute the buffer capacity\ + \ of this solution. First we write the equation for the ionization of the weak\ + \ acid, in this case of acetic acid. $CH_{3}COOH (aq) + H_{2}O \nightarrow H_{3}O^{+}\ + \ + CH3COO^{-}$. The conjugate base is therefore the acetate ion. The added\ + \ strong acid, Nitric acid, will react with the conjugate base. Therefore the\ + \ maximum amount of acid that can be added will be equal to the amount of acetate\ + \ ion, or 2 moles. The answer is (C).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f21030ae880ec0d6d42f0e2618c312b55b82549 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml @@ -0,0 +1,84 @@ +dataset_name: high_school_computer_science +description: The following are multiple choice questions (with answers) about high + school computer science. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is an example of the use of a device on the Internet + of Things (IoT) ? + + (A) A car alerts a driver that it is about to hit an object. (B) A hiker uses + a G P S watch to keep track of her position. (C) A refrigerator orders milk + from an online delivery service when the milk in the refrigerator is almost + gone. (D) A runner uses a watch with optical sensors to monitor his heart rate.' + target: Let's think step by step. The term Internet of Things (IoT) refers to + common devices which are connected to the internet, enabling new functionality. + Choice A is incorrect because it does not describe an internet connected device. + In choice B, the watch is only described as having GPS functionality but no + internet connectivity. Choice C describes a common device (a refrigerator) which + has internet connectivity enabling new functionality (online ordering). Choice + D does not mention internet connectivity for the watch, only optical sensors. + The answer is (C). + - question: 'Many Web browsers allow users to open anonymous windows. During a browsing + session in an anonymous window, the browser does not record a browsing history + or a list of downloaded files. When the anonymous window is exited, cookies + created during the session are deleted. Which of the following statements about + browsing sessions in an anonymous window is true? + + (A) The activities of a user browsing in an anonymous window will not be visible + to people who monitor the user''s network, such as the system administrator. + (B) Items placed in a Web store''s shopping cart for future purchase during + the anonymous browsing session will not be saved on the user''s computer. (C) + A user will not be able to log in to e-mail or social media accounts during + the anonymous browsing session. (D) A user browsing in an anonymous window will + be protected from viruses launched from any web sites visited or files downloaded.' + target: "Let's think step by step. Choice A is incorrect as it only describes\ + \ network traffic, which an anonymous browser does not change. Choice B is correct\ + \ as it correctly describes how an anonymous browser will prevent saving data\ + \ on the user\u2019s computer after the session is ended. Choice C is incorrect\ + \ because an anonymous browser will not prevent logging in to email or social\ + \ media accounts. Choice D is incorrect because an anonymous browser in itself\ + \ performs no virus protection. The answer is (B)." + - question: "In the program below, the initial value of X is 5 and the initial value\ + \ of Y is 10.\nIF (X < 0){\n DISPLAY (\"Foxtrot\")\n} ELSE {\n IF (X > Y){\n\ + \ DISPLAY (\"Hotel\")\n } ELSE {\n IF (Y > 0){\n DISPLAY (\"November\")\n\ + \ } ELSE {\n DISPLAY (\"Yankee\")\n }\n }\n}\nWhat is displayed as a result\ + \ of running the program?\n(A) Foxtrot (B) Hotel (C) November (D) Yankee" + target: Let's think step by step. Because X has the value 5, the first conditional + IF (X < 0) is false, so we move to the first ELSE clause. Because X is 5 and + Y is 10, the second conditional IF (X > Y) is false, so we move to the following + ELSE clause. Since Y is 10, the conditional IF (Y > 0) is true, so the command + DISPLAY ("November") is executed. The answer is (C). + - question: 'What is the output of "abc"[::-1] in Python 3? + + (A) Error (B) abc (C) cba (D) c' + target: Let's think step by step. We know that the slicing operator [::-1] takes + all of the elements in the string in reverse order, so we reverse the order + of the string "abc", resulting in "cba". The answer is (C). + - question: "A list of numbers has n elements, indexed from 1 to n. The following algorithm\ + \ is intended to display the number of elements in the list that have a value\ + \ greater than 100. The algorithm uses the variables count and position. Steps\ + \ 3 and 4 are missing.\n Step 1: Set count to 0 and position to 1.\n Step 2:\ + \ If the value of the element at index position is greater than 100, increase\ + \ the value of count by 1.\n Step 3: (missing step)\n Step 4: (missing step)\n\ + \ Step 5: Display the value of count.\nWhich of the following could be used\ + \ to replace steps 3 and 4 so that the algorithm works as intended?\n(A) Step\ + \ 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until\ + \ the value of count is greater than 100.\n(B) Step 3: Increase the value of\ + \ position by 1.\n Step 4: Repeat steps 2 and 3 until the value of position\ + \ is greater than n.\n(C) Step 3: Repeat step 2 until the value of count is\ + \ greater than 100.\n Step 4: Increase the value of position by 1.\n(D) Step\ + \ 3: Repeat step 2 until the value of position is greater than n.\n Step 4:\ + \ Increase the value of count by 1." + target: 'Let''s think step by step. Choice A is incorrect, because its Step 4 + has an incorrect termination condition, stopping when count is greater than + 100. We need to stop after inspecting all elements in the list. Choice B is + correct because it correctly increments both count and position, and correctly + repeats these steps and terminates when all elements in the list have been inspected. + Choice C is incorrect because it incorrectly increments the variable count until + its value is greater than 100, regardless of the elements in the list. Choice + D is incorrect because its step 3 does not increment the value of position, + so it will repeat forever. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4707a1857f44fad0ef67313b5f1901b5b6c869b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml @@ -0,0 +1,199 @@ +dataset_name: high_school_european_history +description: The following are multiple choice questions (with answers) about high + school european history. +fewshot_config: + sampler: first_n + samples: + - question: 'This question refers to the following information. + + Albeit the king''s Majesty justly and rightfully is and ought to be the supreme + head of the Church of England, and so is recognized by the clergy of this realm + in their convocations, yet nevertheless, for corroboration and confirmation + thereof, and for increase of virtue in Christ''s religion within this realm + of England, and to repress and extirpate all errors, heresies, and other enormities + and abuses heretofore used in the same, be it enacted, by authority of this + present Parliament, that the king, our sovereign lord, his heirs and successors, + kings of this realm, shall be taken, accepted, and reputed the only supreme + head in earth of the Church of England, called Anglicans Ecclesia; and shall + have and enjoy, annexed and united to the imperial crown of this realm, as well + the title and style thereof, as all honors, dignities, preeminences, jurisdictions, + privileges, authorities, immunities, profits, and commodities to the said dignity + of the supreme head of the same Church belonging and appertaining; and that + our said sovereign lord, his heirs and successors, kings of this realm, shall + have full power and authority from time to time to visit, repress, redress, + record, order, correct, restrain, and amend all such errors, heresies, abuses, + offenses, contempts, and enormities, whatsoever they be, which by any manner + of spiritual authority or jurisdiction ought or may lawfully be reformed, repressed, + ordered, redressed, corrected, restrained, or amended, most to the pleasure + of Almighty God, the increase of virtue in Christ''s religion, and for the conservation + of the peace, unity, and tranquility of this realm; any usage, foreign land, + foreign authority, prescription, or any other thing or things to the contrary + hereof notwithstanding. + + English Parliament, Act of Supremacy, 1534 + + From the passage, one may infer that the English Parliament wished to argue + that the Act of Supremacy would + + (A) give the English king a new position of authority (B) give the position + of head of the Church of England to Henry VIII alone and exclude his heirs (C) + establish Calvinism as the one true theology in England (D) end various forms + of corruption plaguing the Church in England' + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The Act of Supremacy states that it grants authority to the king "to + repress and extirpate all errors, heresies, and other enormities and abuses", + referring to the corruption in the Church of England. The answer is (D). + - question: "This question refers to the following information.\nRead the following\ + \ excerpt.\nThe revolutionary seed had penetrated into every country and spread\ + \ more or less. It was greatly developed under the r\xE9gime of the military\ + \ despotism of Bonaparte. His conquests displaced a number of laws, institutions,\ + \ and customs; broke through bonds sacred among all nations, strong enough to\ + \ resist time itself; which is more than can be said of certain benefits conferred\ + \ by these innovators.\nThe monarchs will fulfil the duties imposed upon them\ + \ by Him who, by entrusting them with power, has charged them to watch over\ + \ the maintenance of justice, and the rights of all, to avoid the paths of error,\ + \ and tread firmly in the way of truth. Placed beyond the passions which agitate\ + \ society, it is in days of trial chiefly that they are called upon to despoil\ + \ realities of their false appearances, and to show themselves as they are,\ + \ fathers invested with the authority belonging by right to the heads of families,\ + \ to prove that, in days of mourning, they know how to be just, wise, and therefore\ + \ strong, and that they will not abandon the people whom they ought to govern\ + \ to be the sport of factions, to error and its consequences, which must involve\ + \ the loss of society.\nUnion between the monarchs is the basis of the policy\ + \ which must now be followed to save society from total ruin. . . .\nLet them\ + \ not confound concessions made to parties with the good they ought to do for\ + \ their people, in modifying, according to their recognized needs, such branches\ + \ of the administration as require it.\nLet them be just, but strong; beneficent,\ + \ but strict.\nLet them maintain religious principles in all their purity, and\ + \ not allow the faith to be attacked and morality interpreted according to the\ + \ social contract or the visions of foolish sectarians.\nLet them suppress Secret\ + \ Societies; that gangrene of society.\n\u2014Klemens von Metternich, Political\ + \ Confession of Faith, 1820\nWhich of the following was the greatest cause of\ + \ the fears expressed by Metternich in the document above?\n(A) The ideas of\ + \ personal liberty and nationalism conceived during the Enlightenment resulted\ + \ in radical revolutions that could spread throughout Europe. (B) The conquest\ + \ of Europe by Napoleon led to the creation of new factions and shifted the\ + \ European balance of power. (C) The power of monarchs had grown to the point\ + \ where it needed to be checked by other powers within each nation or domination\ + \ of civilians would occur. (D) The rising and falling economic cycle of the\ + \ newly emerging capitalist economy could lead to civilian unrest that must\ + \ be suppressed." + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The fears of revolution in early 19th century Europe expressed by + Klemens von Metternich, a conservative Austrian statesman, were a direct result + of the age of Enlightenment, a period of European history where the absolute + power of the monarchy was challenged with ideas of individual liberty and nationalism, + leading to the French revolution and its effects all over Europe. The answer + is (A). + - question: 'This question refers to the following information. + + The excerpts below are from the Navigation Acts of 1651. + + [A]fter the first day of December, one thousand six hundred fifty and one, and + from thence forwards, no goods or commodities whatsoever of the growth, production + or manufacture of Asia, Africa or America, or of any part thereof; or of any + islands belonging to them, or which are described or laid down in the usual + maps or cards of those places, as well of the English plantations as others, + shall be imported or brought into this Commonwealth of England, or into Ireland, + or any other lands, islands, plantations, or territories to this Commonwealth + belonging, or in their possession, in any other ship or ships, vessel or vessels + whatsoever, but only in such as do truly and without fraud belong only to the + people of this Commonwealth, or the plantations thereof, as the proprietors + or right owners thereof; and whereof the master and mariners are also of the + people of this Commonwealth, under the penalty of the forfeiture and loss of + all the goods that shall be imported contrary to this act, , , , + + [N]o goods or commodities of the growth, production, or manufacture of Europe, + or of any part thereof, shall after the first day of December, one thousand + six hundred fifty and one, be imported or brought into this Commonwealth of + England, or any other lands or territories to this Commonwealth belonging, or + in their possession, in any ship or ships, vessel or vessels whatsoever, but + in such as do truly and without fraud belong only to the people of this Commonwealth, + and in no other, except only such foreign ships and vessels as do truly and + properly belong to the people of that country or place, of which the said goods + are the growth, production or manufacture. + + Which of the following best describes the outcome of the Navigation Acts of + 1651? + + (A) They served as a catalyst for the growth of English shipping and overseas + trade, but did little to limit the prospects of the Dutch in the seventeenth + century. (B) They brought about almost immediate hardships for the Dutch economy + as their dominance of overseas trade quickly ended. (C) They were rescinded + during the restoration of the Stuarts as they sought normal diplomatic relations + with the Dutch so not as to need Parliament''s financial support for war. (D) + They led to nearly a century of recurrent war between England and the Netherlands, + which would not end until after American independence.' + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The Navigation Acts of 1651 helped English shipping by restricting + the ability of ships from other European countries, especially the Dutch, to + transport goods from colonies in Asia and Africa into England. The answer is + (A). + - question: "This question refers to the following information.\nIn Russia there was\ + \ nothing going on well, and [Souvarine] was in despair over the news he had\ + \ received. His old companions were all turning to the politicians; the famous\ + \ Nihilists who made Europe tremble-sons of village priests, of the lower middle\ + \ class, of tradesmen-could not rise above the idea of national liberation,\ + \ and seemed to believe that the world would be delivered-when they had killed\ + \ their despot&\u2026\n\"Foolery! They'll never get out of it with their foolery.\"\ + \nThen, lowering his voice still more, in a few bitter words he described his\ + \ old dream of fraternity. He had renounced his rank and his fortune; he had\ + \ gone among workmen, only in the hope of seeing at last the foundation of a\ + \ new society of labour in common. All the sous in his pockets had long gone\ + \ to the urchins of the settlement; he had been as tender as a brother with\ + \ the colliers, smiling at their suspicion, winning them over by his quiet workmanlike\ + \ ways and his dislike of chattering. But decidedly the fusion had not taken\ + \ place.\nHis voice changed, his eyes grew bright, he fixed them on \xE9tienne,\ + \ directly addressing him:\n\"Now, do you understand that? These hatworkers\ + \ at Marseilles who have won the great lottery prize of a hundred thousand francs\ + \ have gone off at once and invested it, declaring that they are going to live\ + \ without doing anything! Yes, that is your idea, all of you French workmen;\ + \ you want to unearth a treasure in order to devour it alone afterwards in some\ + \ lazy, selfish corner. You may cry out as much as you like against the rich,\ + \ you haven't got courage enough to give back to the poor the money that luck\ + \ brings you. You will never be worthy of happiness as long as you own anything,\ + \ and your hatred of the bourgeois proceeds solely from an angry desire to be\ + \ bourgeois yourselves in their place.\"\n\xE9mile Zola, French writer, Germinal,\ + \ 1885\nThe passage displays the direct concern for the welfare of the working\ + \ classes that was typically a part of which movement?\n(A) Capitalist (B) Scientific\ + \ (C) Communist (D) Existentialist" + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The modern Communist movement aims to establish a classless society + based on communal ownership and distribution of property and means of production, + thereby especially benefiting the working classes. The answer is (C). + - question: "This question refers to the following information.\nThe following excerpt\ + \ is from a pamphlet.\nYou will do me the justice to remember, that I have always\ + \ strenuously supported the Right of every man to his own opinion, however different\ + \ that opinion might be to mine. He who denies to another this right, makes\ + \ a slave of himself to his present opinion, because he precludes himself the\ + \ right of changing it.\nThe most formidable weapon against errors of every\ + \ kind is Reason. I have never used any other, and I trust I never shall.\n\ + The circumstance that has now taken place in France of the total abolition of\ + \ the whole national order of priesthood, and of everything appertaining to\ + \ compulsive systems of religion, and compulsive articles of faith, has not\ + \ only precipitated my intention, but rendered a work of this kind exceedingly\ + \ necessary, lest in the general wreck of superstition, of false systems of\ + \ government, and false theology, we lose sight of morality, of humanity, and\ + \ of the theology that is true.\nI believe in one God, and no more; and I hope\ + \ for happiness beyond this life.\nI believe in the equality of man; and I believe\ + \ that religious duties consist in doing justice, loving mercy, and endeavoring\ + \ to make our fellow-creatures happy.\nI do not believe in the creed professed\ + \ by the Jewish church, by the Roman church, by the Greek church, by the Turkish\ + \ church, by the Protestant church, nor by any church that I know of. My own\ + \ mind is my own church.\nAll national institutions of churches, whether Jewish,\ + \ Christian or Turkish, appear to me no other than human inventions, set up\ + \ to terrify and enslave mankind, and monopolize power and profit.\nI do not\ + \ mean by this declaration to condemn those who believe otherwise; they have\ + \ the same right to their belief as I have to mine.\n\u2014Thomas Paine, The\ + \ Age of Reason, 1794\u20131795\nWhich of the following Enlightenment philosophes\ + \ designed a system of checks and balances for government to avoid abuses of\ + \ power?\n(A) Jean Jacques Rousseau (B) Baron Montesquieu (C) Mary Wollstonecraft\ + \ (D) Adam Smith" + target: 'Let''s think step by step. We refer to Wikipedia articles on european + history for help. Baron Montesquieu was a 18th centrury French philsopher who + wrote extensively against the monoplization of power and advocated for a system + of checks and balances in government to prevent the rise of despotism. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96f4b365af04a3dc5c754def2899a5291ada1072 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml @@ -0,0 +1,53 @@ +dataset_name: high_school_geography +description: The following are multiple choice questions (with answers) about high + school geography. +fewshot_config: + sampler: first_n + samples: + - question: 'Which one of the following items is an example of nonmaterial culture? + + (A) Dove soap (B) Dove candy bar (C) Dove symbol (D) A dove (bird).' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. Nonmaterial culture consists of cultural ideas, beliefs or symbols + that are not physical objects. The answer is (C). + - question: 'During the third stage of the demographic transition model, which of the + following is true? + + (A) Birth rates increase and population growth rate is less rapid. (B) Birth + rates decline and population growth rate is less rapid. (C) Birth rates increase + and population growth rate increases. (D) Birth rates decrease and population + growth rate increases.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. The demographic transition model models the five different stages + of population growth as a country goes through economic development, where the + third stage refers to a period of declining birth rates and lower population + growth. The answer is (B). + - question: 'The practice of hiring a foreign third-party service provider to run an + operation is called + + (A) outsourcing. (B) offshoring. (C) maquiladoras. (D) locational interdependence.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. "Offshoring" literally means to move or base some of the activities + or processes of a company to a foreign country. The answer is (B). + - question: 'Which of the following statements is NOT accurate regarding the services + provided by local governments in the United States? + + (A) Duplication of efforts occurs often. (B) Social problems of the central + city spill over into the surrounding residential suburbs. (C) Inefficiency in + providing services occurs often. (D) One neighborhood''s efforts to reduce pollution + are always supported by neighboring communities.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. There may be economic, social or political reasons for two neighboring + communities and their local governments not agreeing to pollution reduction + efforts initiated by one of them. The answer is (D). + - question: 'The rate of natural increase of a population is found by subtracting the + + (A) crude death rate from the crude birth date. (B) crude birth rate from the + crude death rate. (C) doubling time from the crude birth rate. (D) fertility + rate from the crude death rate.' + target: 'Let''s think step by step. We refer to Wikipedia articles on geography + for help. The difference between number of births and deaths gives the population + increase at any given time. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c11772183c0266288f95f4d273ebbdf32c0dba1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,61 @@ +dataset_name: high_school_government_and_politics +description: The following are multiple choice questions (with answers) about high + school government and politics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following best states an argument made by James Madison in + The Federalist number 10? + + (A) Honest politicians can prevent factions from developing. (B) Factions are + more likely to occur in large republics than in small ones. (C) The negative + effects of factionalism can be reduced by a republican government. (D) Free + elections are the people''s best defense against factionalism.' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. In the Federalist number 10, James Madison advocated + for a representative republican form of government to guard against factionalism. + The answer is (C). + - question: 'The term "budget deficit" refers to the + + (A) annual increase in federal spending on the military (B) amount of interest + on the national debt (C) difference between the initial budget proposals made + by the president and Congress (D) amount the government spends in excess of + its revenues' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. When the goverment spends more than it earns, their difference + is the budget deficit. The answer is (D). + - question: 'Which of the following statements about cabinet departments is FALSE? + + (A) They are established by the legislative branch. (B) Their members often + don''t have much influence over presidential decisions. (C) They cannot all + be run by leaders who belong to the same political party the president does. + (D) Not every federal agency is a cabinet department.' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. There is no law stipulating that some cabinet department + leaders have to belong to a political party different from that of the president. + The answer is (C). + - question: 'Which of the following cases established the precedent that a defendant + must be informed of the right to remain silent, the right to a lawyer, and protection + from self-incrimination? + + (A) Weeks v. United States (B) Betts v. Brady (C) Mapp v. Ohio (D) Miranda v. + Arizona' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. In the landmark Miranda v. Arizona in 1966, the US Supreme + Court, based on the Fifth and Sixth Amendment of the US Constitution, guaranteed + a defendant's right to an attorney and protection from self-incrimination. The + answer is (D). + - question: 'Uncertainty over the limits to presidential power is caused primarily + by the fact that + + (A) the constitutional definition of those powers is broad and unspecific (B) + most people agree that the Constitution places too many limits on presidential + power (C) the Supreme Court consistently refuses to rule on cases concerning + presidential powers (D) constitutional amendments have greatly increased presidential + powers' + target: 'Let''s think step by step. We refer to Wikipedia articles on government + and politics for help. The US Constitution is not very specific about the powers + of the president, leading to uncertainty over its limits. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5700d5df946539608bbf10555e0f06ca07672b09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,53 @@ +dataset_name: high_school_macroeconomics +description: The following are multiple choice questions (with answers) about high + school macroeconomics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following policies best describes supply-side fiscal policy? + + (A) An increase in the money supply (B) Increased government spending (C) Lower + taxes on research and development of new technology (D) Higher taxes on household + income' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. Supply-side fiscal policy stimulates the economy by encouraging more + production of goods and services through reduction in taxes and deregulation. + The answer is (C). + - question: 'The short-run Phillips curve indicates a + + (A) direct relation between unemployment and inflation (B) direct relation between + price and quantity demanded (C) inverse relation between price and quantity + demanded (D) inverse relation between unemployment and inflation' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. The short-run Phillips curve shows that whenever unemployment decreases + below a natural level, the inflation starts increasing, and vice-versa. The + answer is (D). + - question: 'Holding all else equal which of the following monetary policies would + be used to boost U.S. exports? + + (A) Increasing the discount rate (B) Increasing the reserve ratio (C) Buying + government securities (D) Lowering tariffs' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. Buying government securities leads to reduction in demand for US dollars + from foreign buyers, thereby making it cheaper and hence making US exports more + attractive. The answer is (C). + - question: 'A federal deficit occurs when + + (A) exports exceed imports. (B) imports exceed exports. (C) federal tax collections + exceed spending. (D) federal spending exceeds federal tax revenues.' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. A federal deficit occurs when federal spending exceeds federal income + which is primarily from tax revenues. The answer is (D). + - question: 'Which of the following is not included in the U.S. GDP? + + (A) The U.S. military opens a new base in a foreign country with 1000 U.S. personnel. + (B) Japanese consumers buy thousands of CDs produced in the United States. (C) + An American pop singer performs a sold-out concert in Paris. (D) A French theatrical + production tours dozens of American cities.' + target: 'Let''s think step by step. We refer to Wikipedia articles on macroeconomics + for help. The economic transactions related to the performance of the American + pop-singer in Paris happens entirely outside the U.S. and hence is not included + in the GDP numbers. The answer is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3e05795561a11dca35dd342bc88b4794584809c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml @@ -0,0 +1,51 @@ +dataset_name: high_school_mathematics +description: The following are multiple choice questions (with answers) about high + school mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Simplify and write the result with a rational denominator: $$\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}$$ + + (A) \frac{3\sqrt{3}}{3} (B) \frac{1}{3} (C) \sqrt{3} (D) \frac{\sqrt{3}}{3}' + target: Let's think step by step. Factoring $729=3^6$ and combining the roots + $\frac{1}{2}\frac{1}{3}\frac{1}{2}=\frac{1}{12}$, we get that $\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}=\left(\frac{1}{3^6}\right)^{\frac{1}{12}}=\frac{1}{3^{\frac{1}{2}}}=\frac{3}{\sqrt{3}}$ + The answer is (D). + - question: 'Five thousand dollars compounded annually at an $x\%$ interest rate takes + six years to double. At the same interest rate, how many years will it take + $\$300$ to grow to $\$9600$? + + (A) 12 (B) 1 (C) 30 (D) 5' + target: Let's think step by step. To go from $\$300$ to $\$9600$, the value must + go up by a factor of $9600/300=32=2^5$. Since at this interest rate it takes + six years for it to double, it will take $5*6=30$ years to grow to $\$9600$. + The answer is (C). + - question: "Ten students take a biology test and receive the following scores: 45,\ + \ 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students\u2019\ + \ test scores?\n(A) 55 (B) 60 (C) 62 (D) 65" + target: Let's think step by step. There are 10 students and the sum of their scores + is $45 + 55 + 50 + 70 + 65 + 80 + 40 + 90 + 70 + 85 = 650$, the mean is $650/10=65$. + The answer is (D). + - question: 'The variable $x$ varies directly as the square of $y$, and $y$ varies + directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is + the value of $x$ when $z$ equals $\frac{1}{2}$? + + (A) -1 (B) 16 (C) -\frac{1}{256} (D) \frac{1}{16}' + target: Let's think step by step. We know that $x \propto y^2$ and $y \propto + z^3$, so $x = k z^6$ for some constant $k$. Plugging in for $x=-16$ and $z=2$, + the constant value is $k=\frac{x}{z^6}=\frac{-16}{64}=-\frac{1}{4}$. So, when + $z=\frac{1}{2}$, the value of $x$ is $x=kz^6=-\frac{1}{4}\frac{1}{2^6}=-\frac{1}{256}$. + The answer is (C). + - question: 'Joe was in charge of lights for a dance. The red light blinks every two + seconds, the yellow light every three seconds, and the blue light every five + seconds. If we include the very beginning and very end of the dance, how many + times during a seven minute dance will all the lights come on at the same time? + (Assume that all three lights blink simultaneously at the very beginning of + the dance.) + + (A) 3 (B) 15 (C) 6 (D) 5' + target: 'Let''s think step by step. The least common multiple of 2, 3 and 5 is + 30, so during a 7 minute dance, all the three lights will come on at the same + time $2*7+1=15$ times. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ead78be898187a7c08c292eaeebfe6b067ef4413 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,56 @@ +dataset_name: high_school_microeconomics +description: The following are multiple choice questions (with answers) about high + school microeconomics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is necessarily a characteristic of oligopoly? + + (A) Free entry into and exit from the market (B) A few large producers (C) One + producer of a good with no close substitutes (D) A homogenous product' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. An oligopoly is when a market is dominated by just one or a few number + of sellers or producers. To get oligopoly, the market should have high barriers + to new entry, and the product has differentiation. The answer is (B). + - question: 'If the government subsidizes producers in a perfectly competitive market, + then + + (A) the demand for the product will increase (B) the demand for the product + will decrease (C) the consumer surplus will increase (D) the consumer surplus + will decrease' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. (A) and (B) are wrong because the demand curve does not change at + all. If the government subsidizes producers, the supply will increase, and thus + the consumer surplus also increases. The answer is (C). + - question: 'Which of the following is true of a price floor? + + (A) The price floor shifts the demand curve to the left. (B) An effective floor + creates a shortage of the good. (C) The price floor shifts the supply curve + of the good to the right. (D) To be an effective floor, it must be set above + the equilibrium price.' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. Price floor does not shift the demand or shift curve. An effective + price floor should be set above the equilibrium price, otherwise the market + bears and the floor does not have effective effect. The answer is (D). + - question: 'The concentration ratio for a monopoly is + + (A) 0 (B) 5 (C) 10 (D) 100' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. The concentration ratio is calculated as the sum of market share of + a specific number of largest companies. Monopoly means one company or entity + controls the entire market, therefore, the concentration ratio is 100 percent. + The answer is (D). + - question: 'In a competitive labor market for housepainters, which of the following + would increase the demand for housepainters? + + (A) An effective minimum wage imposed on this labor market. (B) An increase + in the price of gallons of paint. (C) An increase in the construction of new + houses. (D) An increase in the price of mechanical painters so long as the output + effect exceeds the substitution effect.' + target: 'Let''s think step by step. We refer to Wikipedia articles on microeconomics + for help. An increase in the construction of new houses means an increase demand + of in-house painting, thus increases the demand for housepainters. The answer + is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d2166b8595529213797cf12efd67f90ca4e9e61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml @@ -0,0 +1,50 @@ +dataset_name: high_school_physics +description: The following are multiple choice questions (with answers) about high + school physics. +fewshot_config: + sampler: first_n + samples: + - question: 'A microwave oven is connected to an outlet, 120 V, and draws a current + of 2 amps. At what rate is energy being used by the microwave oven? + + (A) 10 W (B) 30 W (C) 60 W (D) 240 W' + target: Let's think step by step. Rate of energy usage is known as power; in an + dissipative electrical circuit, power is given by voltage times current. So + in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D). + - question: "A point charge, Q = +1 mC, is fixed at the origin. How much work is required\ + \ to move a charge, Q = +8 \xB5C, from the point (0, 4 meters) to the point\ + \ (3 meters, 0)?\n(A) 3.5 J (B) 6.0 J (C) 22.5 J (D) 40 J" + target: "Let's think step by step. To calculate the work required to move a charge\ + \ from one location to another in a fixed electric field, it is enough to calculate\ + \ the potential difference between the two locations. Here, the potential only\ + \ depends on the distance between the charges; it\u2019s $k q_1 q_2 / r$, where\ + \ $k$ is Coulomb\u2019s constant. Plugging in values $q_1 = $ 1 mC, $q_2 = 8\ + \ \\mu$ C, gives the answer as 5.992 J, which rounds to 6 J. The answer is (B)." + - question: 'Which of the following conditions will ensure that angular momentum is + conserved? I. Conservation of linear momentum II. Zero net external force III. + Zero net external torque + + (A) I and II only (B) I and III only (C) II and III only (D) III only' + target: Let's think step by step. Torque is defined as the change in angular momentum; + if there is zero external torque, angular momentum is conserved. The answer + is (D). + - question: "A photocell of work function \u03D5 = 2eV is connected to a resistor in\ + \ series. Light of frequency f = 1 \xD7 10^15 Hz hits a metal plate of the photocell.\ + \ If the power of the light is P = 100 W, what is the current through the resistor?\n\ + (A) 2:00 AM (B) 6:00 AM (C) 12:00 AM (D) 24 A" + target: Let's think step by step. The only answer above which has units of current + is D, 24 A. The answer is (D). + - question: "A pipe full of air is closed at one end. A standing wave is produced in\ + \ the pipe, causing the pipe to sound a note. Which of the following is a correct\ + \ statement about the wave\u2019s properties at the closed end of the pipe?\n\ + (A) The pressure is at a node, but the particle displacement is at an antinode.\ + \ (B) The pressure is at an antinode, but the particle displacement is at a\ + \ node. (C) The pressure and the particle displacement are both at nodes. (D)\ + \ The pressure and the particle displacement are both at antinodes." + target: 'Let''s think step by step. At the closed end of the pipe, the particles + cannot have any net displacement because the pipe closure stops them. So the + particle displacement is at a node. This closure also causes the pressure to + be maximal, i.e. an antinode. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..553fe18de24f14765d14d0fb2dbdfaf72449af32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml @@ -0,0 +1,64 @@ +dataset_name: high_school_psychology +description: The following are multiple choice questions (with answers) about high + school psychology. +fewshot_config: + sampler: first_n + samples: + - question: 'Pascale is interested in the processing strategies children use to learn + new information. Pascale would best be classified as what type of psychologist? + + (A) sociocultural (B) clinical (C) cognitive (D) behaviorist' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Sociocultural psychologist focuses on the effect of societal factors + on people. Clinical psychologist focuses on people with mental issues. Cognitive + psychologist focuses on how people think and learn, including the processing + strategies. Behaviorist focuses more on the environment and experience effect + on people. The answer is (C). + - question: 'According to Caplan''s model of consultee-centered case consultation, + the consultant is primarily interested in + + (A) identifying the causes and solutions of the client''s presenting problems + (B) identifying and eliminating the causes of the consultee''s difficulties + in handling a problem (C) establishing a hierarchy of authority to enable effective + decision making (D) presenting a single, well-defined and unambiguous course + of action for the consultant to overcome skills deficits' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Caplan defines two type of consultation. Client-centered case consultation + aims to handle client's problems, while consultee-centered case consultation + aims to identify the reason of client's difficulty to solve problems. The answer + is (B). + - question: 'According to the Individuals with Disabilities Education Improvement Act, + which of the following must an educational agency do before it changes the educational + placement of a student with a disability? + + (A) Give the child a trial period in the new environment (B) Notify the parents + in writing (C) Obtain school board approval (D) Obtain parental consent' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. When the decision to change the educational placement of a student + with a disability is made, the educational agency must notify the parents in + writing on that date. The answer is (B). + - question: 'While swimming in the ocean, Ivan is frightened by a dark shadow in the + water even before he has the chance to identify what the shadow is. The synaptic + connections taking place during this incident of fright are best described by + which of the following? + + (A) Messages are sent from the thalamus directly to the amygdala. (B) Messages + are sent from the thalamus to the "what" and "where" pathways. (C) Messages + are sent from the parasympathetic nervous system to the cerebral cortex. (D) + Messages are sent from the frontal lobes to the pituitary gland.' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Our neural system has a mechanism that can respond immediate emotional + signal before going to the thought center. In the Ivan's case, messages travel + directly from thalamus to amygdala. The answer is (A). + - question: 'Ani believes that her attitudes and behavior play a central role in what + happens to her. Such a belief is likely to be associated with + + (A) a strong superego. (B) low self-esteem. (C) low self-efficacy. (D) an internal + locus of control.' + target: 'Let''s think step by step. We refer to Wikipedia articles on psychology + for help. People with an external locus of control believes fate and luck play + an important role in their lives, while people with an internal locus of control + believes they control their lives. The answer is (D).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66ed702a6b9b57dc69719a13923e04ebc8218ea9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml @@ -0,0 +1,81 @@ +dataset_name: high_school_statistics +description: The following are multiple choice questions (with answers) about high + school statistics. +fewshot_config: + sampler: first_n + samples: + - question: 'A new smartwatch is manufactured in one part of a factory, then secured + for shipping in another, independent part of the factory. The weight of the + smartwatch has a mean of 62 grams and a standard deviation of 1.0 grams. The + weight of the packaging (box, user''s guide, bubble wrap, etc.) has a mean of + 456 grams and a standard deviation of 6 grams. Together, the distribution of + the weight of the smartwatch and its packaging would have the following mean + and standard deviation: + + (A) Mean 518 grams; standard deviation 7.0 grams (B) Mean 518 grams; standard + deviation 3.5 grams (C) Mean 518 grams; standard deviation 6.1 grams (D) Mean + 394 grams; standard deviation 6.1 grams' + target: Let's think step by step. Since the weight of the watch and the weight + of the packaging are independent random variables, the mean and variance of + their sum is equal to the sum of their individual means and variances. So the + mean is 62 + 456 = 518 grams, and the variances is 1.0^2 + 6.0^2 = 37, leading + to a standard deviation of 6.1 grams. The answer is (C). + - question: 'After a frost warning was issued, the owner of a large orange grove asked + his workers to spray all his trees with water. The water was supposed to freeze + and form a protective covering of ice around the orange blossom. Nevertheless, + the owner suspected that some trees suffered considerable damage due to the + frost. To estimate the proportion of trees that suffered more than 50 percent + damage due to the frost, he took a random sample of 100 trees from his grove. + What is the response variable in this experiment? + + (A) The proportion of trees that suffered more than 50 percent damage due to + frost. (B) The number of trees affected by the frost. (C) The number of trees + sampled from the grove. (D) For each sampled tree, whether it suffered more + than 50 percent damage or at most 50 percent damage.' + target: Let's think step by step. In this experiment, the response variable is + what is measured. For each tree, what is measured is whether or not it suffered + more than 50 percent damage due to the frost. The answer is (D). + - question: 'Suppose X and Y are random variables with E(X) = 37, var(X) = 5, E(Y) + = 62, and var(Y) = 12. What are the expected value and variance of the random + variable X + Y? + + (A) E(X + Y) = 99, var(X + Y) = 8.5 (B) E(X + Y) = 99, var(X + Y) = 13 (C) E(X + + Y) = 99, var(X + Y) = 17 (D) There is insufficient information to answer this + question.' + target: Let's think step by step. While means of sums of random variables add + (regardless of whether the variables are independent) in order to determine + the variance of a sum of random variables, we need to know not just their individual + variances but the covariance of the two variables, which is not given in this + problem. The answer is (D). + - question: 'Which of the following sets has the smallest standard deviation? Which + has the largest? + + I: {1,2,3} + + II: {-10,10} + + III: {100} + + (A) I, II (B) II, III (C) III, I (D) III, II' + target: Let's think step by step. The variance of distribution I is the expected + squared deviation from its mean (which is 2), so the variance is 2/3 . The variance + of distribution II is 10^2 (because both elements are 10 away from the mean + of zero). The variance of distribution III is 0, since it has a single entry. + So distribution III has the smallest standard deviation and distribution II + has the largest. The answer is (D). + - question: 'Which of the following is a correct statement about correlation? + + (A) If the slope of the regression line is exactly 1, then the correlation is + exactly 1. (B) If the correlation is 0, then the slope of the regression line + is undefined. (C) Switching which variable is called x and which is called y + changes the sign of the correlation. (D) The correlation r is equal to the slope + of the regression line when z-scores for the y-variable are plotted against + z-scores for the x-variable.' + target: 'Let''s think step by step. Statement A is false because the slope of + the regression line being exactly 1 can occur even when the two variables are + not perfectly correlated. Statement B is false because uncorrelated variables + regression lines can have slope zero. Statement C is false because correlation + is symmetric in the two random variables. The answer is (D).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cea5109f6570086dce3cf1815dc50f1889d80ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml @@ -0,0 +1,156 @@ +dataset_name: high_school_us_history +description: The following are multiple choice questions (with answers) about high + school us history. +fewshot_config: + sampler: first_n + samples: + - question: "This question refers to the following information.\nI come not to urge\ + \ personal claims, nor to seek individual benefits; I appear as the advocate\ + \ of those who cannot plead their own cause; I come as the friend of those who\ + \ are deserted, oppressed, and desolate. In the Providence of God, I am the\ + \ voice of the maniac whose piercing cries from the dreary dungeons of your\ + \ jails penetrate not your Halls of Legislation. I am the Hope of the poor crazed\ + \ beings who pine in the cells, and stalls, and cages, and waste rooms of your\ + \ poor-houses. I am the Revelation of hundreds of wailing, suffering creatures,\ + \ hidden in your private dwellings, and in pens and cabins\u2014shut out, cut\ + \ off from all healing influences, from all mind-restoring cares.\u2026 Could\ + \ their melancholy histories be spread before you as revealed to my grieved\ + \ spirit during the last three months, how promptly, how earnestly would you\ + \ search out the most approved means of relief; how trifling, how insignificant,\ + \ by comparison, would appear the sacrifices you are asked to make; how would\ + \ a few dimes and dollars, gathered from each citizen, diminish in value as\ + \ a possession, compared with the certain benefits and vast good to be secured\ + \ for the suffering insane...by the consecration and application of a sufficient\ + \ fund to the construction of a suitable hospital.\u2026\n\u2014Dorothea Dix,\ + \ Memorial Soliciting a State Hospital for the Protection and Cure of the Insane,\n\ + Submitted to the General Assembly of North Carolina, November 1848\nDorothea\ + \ Dix can best be compared to whom?\n(A) Abigail Adams (B) Clara Barton (C)\ + \ Shirley Temple (D) Hillary Clinton" + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. Both Dorothea Dix and Clara barton are American nurses. The answer + is (B). + - question: "This question refers to the following information.\n\"As our late Conduct\ + \ at the Conestoga Manor and Lancaster have occasioned much Speculation & a\ + \ great diversity of Sentiments in this and neighboring Governments; some vindicating\ + \ & others condemning it; some charitably alleviating the Crime, & others maliciously\ + \ painting it in the most odious & detestable Colours, we think it our duty\ + \ to lay before the Publick, the whole Matter as it appeared, & still appears,\ + \ to us. . . .\n\"If these things are not sufficient to prove an unjustifiable\ + \ Attachment in the Quakers to the Indians Savages, a fixed Resolution to befriend\ + \ them & an utter insensibility to human Distresses, let us consider a few more\ + \ recent Facts. When we found the last Summer that we were likely to get no\ + \ Assistance from the Government, some Volunteers went out at our own Expense,\ + \ determined to drive our Enemies from our Borders; & when we came near to the\ + \ great Island, we understood that a Number of their Warriors had gone out against\ + \ our Frontiers. Upon this we returned and came up with them and fought with\ + \ them at the Munfey Hill where we lost some of our Men & killed some of their\ + \ Warriors & thereby saved our Frontiers from this Story in another Expedition.\ + \ But no sooner had we destroyed their Provisions on the great Island, & ruined\ + \ their trade with the good People at Bethlehem, but these very Indians, who\ + \ were justly suspected of having murdered our Friends in Northampton County,\ + \ were by the Influence of some Quakers taken under the Protection of the Government\ + \ to screen them from the Resentments of the Friends and Relations of the Murdered,\ + \ & to support them thro the Winter.\"\n\u2014\"Apology of the Paxton Boys\"\ + \ (pamphlet), 1764 (Note: \"apology\" in this context should be read as an explanation,\ + \ not an admission of guilt or regret.\nThe sentiments expressed in the explanation\ + \ above reflect which of the ongoing tensions during the colonial period of\ + \ American history?\n(A) Tensions between British policies and the aspirations\ + \ of North American colonists. (B) Tensions between American Indians allied\ + \ with the French and those allied with the British. (C) Tensions between freed\ + \ African Americans and white planters. (D) Tensions between backcountry settlers\ + \ and elites within colonial America." + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. After the French and Indian War, the Scotch-Irish settlers attacked + American Indians. After the attacks on the Conestoga, about 250 Paxton Boys + present their grievances to the Pennsylvania legislature. As mentioned in the + information, the Paxton Boys cited resentiment at local elites. The answer is + (D). + - question: "This question refers to the following information.\nOur leaders talk about\ + \ stopping aggression from the north, but this was a struggle among groups of\ + \ Vietnamese until we intervened. We seem bent upon saving the Vietnamese from\ + \ Ho Chi Minh even if we have to kill them and demolish their country to do\ + \ it. As the native people survey bombed-out villages, women and children burned\ + \ by napalm, rice crops destroyed and cities overrun with our military personnel,\ + \ they are doubtless saying secretly of the Vietcong guerillas and of the American\ + \ forces, \"A plague on both your houses.\" \u2026 Stop the bombing, north and\ + \ south, end search and destroy offensive sweeps, and confine our military action\ + \ to holding operations on the ground. Bombing the north has failed to halt\ + \ or seriously check the flow of troops to the south and may, in fact, have\ + \ prompted a much greater war effort by Hanoi.\n\u2014Senator George McGovern,\ + \ \"The Lessons of Vietnam,\" April 25, 1967\nWhich of the following opinions\ + \ from the 1960s most directly reflects the perspective of George McGovern's\ + \ speech?\n(A) Americans must maximize their technological edge in Vietnam.\ + \ (B) American bombing in Vietnam is step by step leading to progress in the\ + \ war. (C) American bombing in Vietnam is a failure. (D) America must not give\ + \ in to defeatism about the war in Vietnam." + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. "Stop the bombing" and "Bombing the north has failed to halt or seriously + check the flow of troops to the south" indicate that the perspective of George + McGovern's speech is that Amerian bombing in Vietnam is a failure. The answer + is (C). + - question: "This question refers to the following information.\n\"In the new Code\ + \ of Laws which I suppose it will be necessary for you to make I desire you\ + \ would Remember the Ladies, and be more generous and favorable to them than\ + \ your ancestors. Do not put such unlimited power into the hands of the Husbands.\ + \ Remember all Men would be tyrants if they could. If particular care and attention\ + \ is not paid to the Ladies we are determined to foment a Rebellion, and will\ + \ not hold ourselves bound by any Laws in which we have no voice, or Representation.\"\ + \nAbigail Adams, in a letter to John Adams, 1776\n\"Special legislation for\ + \ woman has placed us in a most anomalous position. Women invested with the\ + \ rights of citizens in one section\u2014voters, jurors, office-holders\u2014\ + crossing an imaginary line, are subjects in the next. In some States, a married\ + \ woman may hold property and transact business in her own name; in others,\ + \ her earnings belong to her husband. In some States, a woman may testify against\ + \ her husband, sue and be sued in the courts; in others, she has no redress\ + \ in case of damage to person, property, or character. In case of divorce on\ + \ account of adultery in the husband, the innocent wife is held to possess no\ + \ right to children or property, unless by special decree of the court. But\ + \ in no State of the Union has the wife the right to her own person, or to any\ + \ part of the joint earnings of the co-partnership during the life of her husband.\ + \ In some States women may enter the law schools and practice in the courts;\ + \ in others they are forbidden. In some universities girls enjoy equal educational\ + \ advantages with boys, while many of the proudest institutions in the land\ + \ deny them admittance, though the sons of China, Japan and Africa are welcomed\ + \ there. But the privileges already granted in the several States are by no\ + \ means secure.\"\nSusan B. Anthony, \"Declaration of Rights for Women,\" July\ + \ 4, 1876\nThe sentiments expressed in the second excerpt by Susan B. Anthony\ + \ are most likely in support of\n(A) the Equal Rights Amendment (B) universal\ + \ suffrage (C) states' rights (D) prohibition" + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. The above information mentioned that women are in an anomalous position + in terms of legislation. Women's earnings do not belong to themselves, or they + cannot testify against her husbands. Susan believes women should have equal + legal rights as men. The answer is (B). + - question: 'This question refers to the following information. + + "Society in every state is a blessing, but government even in its best state + is but a necessary evil; in its worst state an intolerable one; for when we + suffer, or are exposed to the same miseries by a government, which we might + expect in a country without government, our calamity is heightened by reflecting + that we furnish the means by which we suffer. Government, like dress, is the + badge of lost innocence; the palaces of kings are built on the ruins of the + bowers of paradise. For were the impulses of conscience clear, uniform, and + irresistibly obeyed, man would need no other lawgiver; but that not being the + case, he finds it necessary to surrender up a part of his property to furnish + means for the protection of the rest; and this he is induced to do by the same + prudence which in every other case advises him out of two evils to choose the + least. Wherefore, security being the true design and end of government, it unanswerably + follows that whatever form thereof appears most likely to ensure it to us, with + the least expense and greatest benefit, is preferable to all others." + + Thomas Paine, Common Sense, 1776 + + Which of the following "miseries" alluded to above were most condemned by Anti-Federalists + of the post-Revolutionary era? + + (A) Organized response to Bacon''s Rebellion (B) Federal response to Shays''s + Rebellion (C) Federal response to the Whiskey Rebellion (D) Federal response + to Pontiac''s Rebellion' + target: 'Let''s think step by step. We refer to Wikipedia articles on us history + for help. Anti-Federalists do not believe centralized government power, and + suspect Washington''s military response to Whiskey Rebellion. Bacon''s Rebellion + and Pontiac''s Rebellion happen before the Revolution and they can be ruled + out. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fbdaf05c137270f4ff4207e7c6ce81c2a34d30c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml @@ -0,0 +1,100 @@ +dataset_name: high_school_world_history +description: The following are multiple choice questions (with answers) about high + school world history. +fewshot_config: + sampler: first_n + samples: + - question: "This question refers to the following information.\n\"At least one of\ + \ the [world's] societies would have to somehow enormously increase its productivity\ + \ [in order to achieve global hegemony]. That quantum jump would have to be\ + \ made before the various scientific, technological, agricultural, and industrial\ + \ revolutions on which our post-quantum-leap world rests. It could only be accomplished\ + \ by exploiting the ecosystems, mineral resources, and human assets of whole\ + \ continents outside the lands of the society making the jump. Western Europe\ + \ did just that by means of its brutality and guns and, more important, by geographical\ + \ and ecological luck.\"\nCopyright \xA9 2015 Cambridge University Press.\n\ + Alfred Crosby, historian, Ecological Imperialism, 2004\nThe \"quantum jump\"\ + \ mentioned in the passage most directly contributed to which of the following\ + \ developments in the period 1450\u20131750 C.E.?\n(A) A breakdown in trade\ + \ routes through the collapse of the established state structure (B) An increase\ + \ in the population of the world through more plentiful supplies of food (C)\ + \ The spread of Chinese and Indian belief systems across the world (D) An increase\ + \ in social unrest" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The "quantum jump" mentioned in the passage refers to the conquest + of the New World and the Columbian Exchange. Choice (A) and (C) did not happen + in history. Choice (C) refers to the human assets. The answer is (B). + - question: "This question refers to the following information.\n\"The struggle against\ + \ neo-colonialism is not aimed at excluding the capital of the developed world\ + \ from operating in less developed countries. It is aimed at preventing the\ + \ financial power of the developed countries being used in such a way as to\ + \ impoverish the less developed.\nNon-alignment, as practiced by Ghana and many\ + \ other countries, is based on co-operation with all States whether they be\ + \ capitalist, socialist or have a mixed economy. Such a policy, therefore, involves\ + \ foreign investment from capitalist countries, but it must be invested in accordance\ + \ with a national plan drawn up by the government of the non-aligned State with\ + \ its own interests in mind. The issue is not what return the foreign investor\ + \ receives on his investments\u2026The question is one of power. A State in\ + \ the grip of neo-colonialism is not master of its own destiny.\"\nKwame Nkrumah,\ + \ Neo-Colonialism, 1965\nWhich of the following provides the best context for\ + \ Nkrumah's writings?\n(A) The Industrial Revolution (B) Decolonization (C)\ + \ Regional Free Trade Associations (D) Autarky" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage expresses a point that the successful fight against neo-colonialism + were in danger and the newly independent nations like Ghana may be re-colonized + via financial power of the developed countries. The answer is (B). + - question: "This question refers to the following information.\n\"Indeed, as both\ + \ the fatwas of distinguished [scholars] who base their opinion on reason and\ + \ tradition alike and the consensus of the Sunni community agree that the ancient\ + \ obligation of extirpation, extermination, and expulsion of evil innovation\ + \ must be the aim of our exalted aspiration, for \"Religious zeal is a victory\ + \ for the Faith of God the Beneficent\"; then, in accordance with the words\ + \ of the Prophet (Peace upon him!) \"Whosoever introduces evil innovation into\ + \ our order must be expelled\" and \"Whosoever does aught against our order\ + \ must be expelled,\" action has become necessary and exigent\u2026\"\nLetter\ + \ from Ottoman Sultan Selim I to Safavid Shah Ismail I, 1514\nThe letter from\ + \ Selim I is most clearly an example of which of the following?\n(A) The maintenance\ + \ of military supremacy at all costs (B) Expanding tensions between religious\ + \ sects (C) Factors that brought about the collapse of the Ottoman Empire (D)\ + \ Peacemaking efforts among the Islamic empires" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage is an example of expanding tensions between Selim and + Ismail. In the passage the Selim references the fatwa and the consensus of the + Sunni community to against whosoever introduces evil. The answer is (B). + - question: 'This question refers to the following information. + + "The real grievance of the worker is the insecurity of his existence; he is + not sure that he will always have work, he is not sure that he will always be + healthy, and he foresees that he will one day be old and unfit to work. If he + falls into poverty, even if only through a prolonged illness, he is then completely + helpless, exam_ins to his own devices, and society does not currently recognize + any real obligation towards him beyond the usual help for the poor, even if + he has been working all the time ever so faithfully and diligently. The usual + help for the poor, however, leaves a lot to be desired, especially in large + cities, where it is very much worse than in the country." + + Otto von Bismarck, 1884 + + Otto von Bismarck likely made this speech in reaction to which of the following + issues? + + (A) Social acceptance of child labor (B) Declining life expectancy in Germany + (C) Criticisms of German trade tariffs (D) Negative effects attributed to industrial + capitalism' + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage talks about the grievance of the work under the industrial + capitalism. The answer is (D). + - question: "This question refers to the following information.\nHe contains all works\ + \ and desires and all perfumes and all tastes. He enfolds the whole universe\ + \ and in silence is loving to all. This is the Spirit that is in my heart, this\ + \ is Brahman. To him I shall come when I go beyond this life, and to him will\ + \ come he who has faith and doubts not.\n\u2014The Upanishads, India, c. 1000\ + \ BCE\nTo which religion does the speaker most likely belong?\n(A) Hinduism\ + \ (B) Buddhism (C) Shintoism (D) Zoroastrianism" + target: 'Let''s think step by step. We refer to Wikipedia articles on world history + for help. Brahman refers to the ultimate reality of all things in the Hindu + religion. In contrast, Buddhism does not have a concept of supreme God. The + answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eec010845fa68ad974bdb7cd922a0028365d96e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml @@ -0,0 +1,42 @@ +dataset_name: human_aging +description: The following are multiple choice questions (with answers) about human + aging. +fewshot_config: + sampler: first_n + samples: + - question: 'All other things being equal, which of the following persons is more likely + to show osteoporosis? + + (A) An older Hispanic American woman (B) An older African American woman (C) + An older Asian American woman (D) An older Native American woman' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. Although osteoporosis can occur at any age, the risk is higher for + older people. It is most common in Asian and non-Hispanic white women. The answer + is (C). + - question: 'The finding that adults tend to remember events from their adolescence + better than from other periods in their lives is referred to as the + + (A) Adolescence advantage (B) Reminiscence bump (C) Memorial memorial (D) Quadratic + retrieval spike' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. Reminiscence bump is a phenomenon that older adults tend to recollect + events during their young ages. People usually have a period of childhood amnesia + from birth to around age 5, and a reminiscence bump between 10 and 30. The answer + is (B). + - question: 'Which element in tobacco smoke is responsible for cancers? + + (A) Nicotine (B) Tar (C) Carbon monoxide (D) Smoke particles' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. The benzene, acrylamide and acrylonitrile in tar interact with the + lungs and cause DNA mutations in cells of the lungs, and lead to cancer. The + answer is (B). + - question: 'When older adults move to a new state after retirement, which of the following + is the more likely destination? + + (A) Texas (B) California (C) Hawaii (D) Vermont' + target: 'Let''s think step by step. We refer to Wikipedia articles on human aging + for help. Texas does not have state tax, and has low cost of living compared + with the other three options. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dab78f0c5ec1042d23240bb71f59b212885585aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml @@ -0,0 +1,50 @@ +dataset_name: human_sexuality +description: The following are multiple choice questions (with answers) about human + sexuality. +fewshot_config: + sampler: first_n + samples: + - question: 'The most common disorder among men who seek sexual therapy is: + + (A) premature ejaculation (B) inhibited ejaculation (C) erectile disorder (D) + ejaculatory disorder' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. The most common type of arousal disorder for men is erectile dysfunction, + meaning that a person is not able to get any physical satisfaction from sexual + activity although he may be interested in it. The answer is (C). + - question: 'A woman who knows she has active herpes and untreated syphilis but continues + to have sex without informing her partners of her condition has, in psychoanalytic + terms: + + (A) a strong ego (B) a weak superego (C) a weak id (D) a strong superego' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. A person with weak superego tends to be delinquent, criminal or have + antisocial personality. The action of the woman who knows she has active venereal + disease but still have sex with her partners indicate she may has antisocial + personality. The answer is (B). + - question: 'The nature of homosexual activities that occur during preadolescence include + all but which of the following? + + (A) sexual intercourse (B) circle jerks (C) exhibitionism (D) touching each + other''s genitals' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. Sexual intercourse is prohibited during preadolescence. The answer + is (A). + - question: 'Women''s ability to have multiple orgasms is primarily due to: + + (A) the fact that they do not have a refractory period. (B) the response of + the inner layers of the vagina. (C) having alternating orgasms in different + locations. (D) the G-Spot.' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. The refractory period is the time when a person is not able to be + erect or is not interested in sex. The answer is (A). + - question: 'Morning sickness is typically a problem: + + (A) during the first trimester (B) during the second trimester (C) during the + third trimester (D) all through the pregnancy' + target: 'Let''s think step by step. We refer to Wikipedia articles on human sexuality + for help. Morning sickness usually begins by nine weeks after conception, corresponding + to the first trimester. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99341f395352d7b6a9a8d1a71005ca821ac9b723 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml @@ -0,0 +1,70 @@ +dataset_name: international_law +description: The following are multiple choice questions (with answers) about international + law. +fewshot_config: + sampler: first_n + samples: + - question: 'How the consent to be bound of a State may be expressed? + + (A) The consent of a State to be bound is expressed only by ratification (B) + The consent of a state to be bound by a treaty may be expressed by signature, + ratification, acceptance, approval or accession (C) The consent of a State to + be bound is expressed by signature (D) The consent of a State to be bound is + expressed by whatever means they choose' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. Article 11 of Vienna Convention on the Law of Treaties signed + in 1969 states that "the consent of a State to be bound by a treaty may be expressed + by signature, exchange of instruments constituting a treaty, ratification, acceptance, + approval or accession, or by any other means if so agreed." (B) is the most + precise and accurate answer. The answer is (B). + - question: 'What is the judge ad hoc? + + (A) If a party to a contentious case before the ICJ does not have a national + sitting as judge, it is entitled to nominate someone as a judge solely for that + case, with the title of judge ad hoc (B) Judge ad hoc is the member of the bench + of the ICJ with a casting vote (C) Judge ad hoc is a surrogate judge, in case + a judge is disqualified or passes away (D) Judge ad hoc is the judge that each + party will always nominate in every contentious case' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. As "ad hoc" implies, a judge ad hoc is appointed only for a specific + case or period, when a party to a contentious case before the International + Court of Justice does not have a regular national sitting as judge. The answer + is (A). + - question: 'When ''consent'' can serve as a circumstance precluding the wrongfulness + of a State conduct? + + (A) Consent can serve as a circumstance precluding the wrongfulness whenever + it is given (B) Consent can never serve as a circumstance precluding wrongfulness + (C) Consent can serve as a circumstance precluding wrongfulness, provided the + consent is valid and to the extent that the conduct remains within the limits + of the consent given (D) Consent can always serve as a circumstance precluding + wrongfulness, no matter which organ of the State gives it' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. Valid consent can serve as a circumstance precluding the wrongfulness + of a State conduct if the conduct remains within the limits of that consent, + according to Chapter V of the Responsibility of States for Internationally Wrongful + Acts, 2001, United Nations. The answer is (C). + - question: 'Would a reservation to the definition of torture in the ICCPR be acceptable + in contemporary practice? + + (A) This is an acceptable reservation if the reserving country''s legislation + employs a different definition (B) This is an unacceptable reservation because + it contravenes the object and purpose of the ICCPR (C) This is an unacceptable + reservation because the definition of torture in the ICCPR is consistent with + customary international law (D) This is an acceptable reservation because under + general international law States have the right to enter reservations to treaties' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. For it contravenes the object and purpose of the ICCPR, this is + an unacceptable reservation in contemporary practice. The answer is (B). + - question: 'What types of force does Article 2(4) of the UN Charter prohibit? + + (A) Article 2(4) encompasses only armed force (B) Article 2(4) encompasses all + types of force, including sanctions (C) Article 2(4) encompasses all interference + in the domestic affairs of States (D) Article 2(4) encompasses force directed + only against a State''s territorial integrity' + target: 'Let''s think step by step. We refer to Wikipedia articles on international + law for help. Article 2(4) of the UN Charter prohibits states from using armed + forces in their international relations. The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c714f7e595a48d28b750479a46183a02fc24dc0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml @@ -0,0 +1,59 @@ +dataset_name: jurisprudence +description: The following are multiple choice questions (with answers) about jurisprudence. +fewshot_config: + sampler: first_n + samples: + - question: 'Iverson Jewelers wrote a letter to Miller, ''We have received an exceptionally + fine self winding Rolox watch which we will sell to you at a very favorable + price.'' + + (A) The letter is an offer to sell (B) A valid offer cannot be made by letter. + (C) The letter contains a valid offer which will terminate within a reasonable + time. (D) The letter lacks one of the essential elements of an offer.' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. An offer shows the intent to enter into a mutually-beneficial contract + with specific terms. An offer can be made by a letter. While this letter indicates + the willingness to sell, the lack of specific terms, such as transaction price + and offer expiration date, makes it an incomplete offer. The answer is (D). + - question: 'Functions of the law include all but which of the following? + + (A) maximizing individual freedom (B) providing a basis for compromise (C) keeping + the peace (D) promoting the principles of the free enterprise system' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Laws are fundamentally about helping resolve disputes between individuals, + and therefore essential for maximizing individual freedom, providing a basis + for compromise, and keeping the peace. The answer is (D). + - question: 'The ________ School of jurisprudence postulates that the law is based + on what is "correct." + + (A) Natural Law (B) Analytical (C) Historical (D) Sociological' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Natural Law School of jurisprudence focuses on the laws of nature, + and states that the law should be based on ethics, morals, and what is "correct". + Analytical deals with the law as it already exists, Historical postulates that + the law was found and not made, and Sociological studies how the law and society + impact each other. The answer is (A). + - question: 'Which word best summarizes Weber''s explanation of the development of + formally rational law? + + (A) Authority. (B) Charisma. (C) Co-operation. (D) Capitalism.' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Weber explained the development of formal rationality in laws as how + the modern society moved from tradition to rationality, where people decide + actions based less on how they were culturally done and more on expected utilities. + How rational individuals optimize efficiency of accomplishing tasks for higher + rewards is a core principle of Capitalism. The answer is (D). + - question: 'Which position does Rawls claim is the least likely to be adopted by the + POP (people in the original position)? + + (A) The POP would choose equality above liberty. (B) The POP would opt for the + ''maximin'' strategy. (C) The POP would opt for the ''difference principle''. + (D) The POP would reject the ''system of natural liberty.''' + target: 'Let''s think step by step. We refer to Wikipedia articles on jurisprudence + for help. The POP would opt for the ''maximin'' strategy, opt for the ''difference + principle'', and reject the ''system of natural liberty'', but the POP would + not choose equality above liberty, since the POP assume both equal and free + citizens. The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd72cab8cf8ea0553a633810e59c0ba4059c2176 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml @@ -0,0 +1,61 @@ +dataset_name: logical_fallacies +description: The following are multiple choice questions (with answers) about logical + fallacies. +fewshot_config: + sampler: first_n + samples: + - question: 'When an arguer causes confusion during refutation because of real or feigned + lack of an ability to engage in refutation, that arguer may have committed the + fallacy of + + (A) poor sportsmanship (B) appeal to compassion (C) argument against the person + (D) ignorance of refutation' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. Ignorance of refutation, one of Aristotle's original list of logical + fallacies in his Organon, is when someone causes confusion in an argument through + real or feigned inability to engage in refutation, in order to win the argument. + The answer is (D). + - question: 'The complex question fallacy consists of + + (A) arguing something is inferior just because it doesn''t do something it was + never intended to do. (B) including more than one claim in the proposition and + treating proof for one claim as proof for all the claims. (C) drawing a conclusion + before examining the evidence, and only considering evidence that supports that + conclusion. (D) asking a question that includes either an unproven assumption + or more than one question, thus making a straightforward yes or no answer meaningless.' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. The complex question fallacy is when someone makes a single yes or + no answer to a question meaningless, by including either an unproven assumption + or many questions. The latter is also known as the many questions fallacy. The + answer is (D). + - question: 'Arguing that what is true of the parts must be true of the whole is the + fallacy of... + + (A) Division (B) Composition (C) Appeal to the person (D) Appeal to ignorance' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. Fallacy of composition occurs when someone argues what is true of + the parts must be true of the whole. The answer is (B). + - question: 'Which of the following is true of a valid categorical syllogism? + + (A) The minor premise must deny the antecedent (B) The major premise must affirm + the consequent (C) The middle term must be used in at least one premise in a + universal or unqualified sense (D) All of the above' + target: 'Let''s think step by step. We refer to Wikipedia articles on logical + fallacies for help. A valid categorical syllogism must satisfy several conditions: + (1) the syllogism must have exactly three terms (2) every term of the syllogism + must be used twice exactly, (3) a term may be used only once in any premise, + and (4) the middle term must be used in at least one premise in a universal + or unqualified sense, etc. Only (C) is true. The answer is (C).' + - question: 'If someone attacks the character of an opposing arguer, instead of responding + to that opponent''s arguments, the first person has probably committed which + of the following fallacies? + + (A) tu quoque (B) horse laugh (C) argument against the person (D) ignoratio + elenchi' + target: 'Let''s think step by step. We refer to Wikipedia articles on logical + fallacies for help. The argument against the person fallacy occurs when someone + irrelevantly attacks the character of an opposing arguer, instead of addressing + that opponent''s arguments. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33622ac4e7291eb380b6e58382e4fe84052a5bdc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml @@ -0,0 +1,74 @@ +dataset_name: machine_learning +description: The following are multiple choice questions (with answers) about machine + learning. +fewshot_config: + sampler: first_n + samples: + - question: 'Which image data augmentation is most common for natural images? + + (A) random crop and horizontal flip (B) random crop and vertical flip (C) posterization + (D) dithering' + target: Let's think step by step. Data augmentation is used to increase the diversity + of images in the training dataset. It is important that natural images are kept + natural after being augmented. Vertical flips of images are not natural, so + (B) is false. Posterization makes the image look like a poster and and dithering + increases color depth. None of these two preserve the natural property. The + only natural data augmentation technique is (A). The answer is (A). + - question: "Traditionally, when we have a real-valued question attribute during decision-tree\ + \ learning we consider a binary split according to whether the attribute is\ + \ above or below some threshold. Pat suggests that instead we should just have\ + \ a multiway split with one branch for each of the distinct values of the attribute.\ + \ From the list below choose the single biggest problem with Pat\u2019s suggestion:\n\ + (A) It is too computationally expensive. (B) It would probably result in a decision\ + \ tree that scores badly on the training set and a testset. (C) It would probably\ + \ result in a decision tree that scores well on the training set but badly on\ + \ a testset. (D) It would probably result in a decision tree that scores well\ + \ on a testset but badly on a training set." + target: "Let's think step by step. Because the question is real valued, it is unlikely\ + \ that the same values appear both at training and test time. This means that\ + \ while such a decision tree could yield good performance on the training data,\ + \ when evaluated on the test data it will perform badly because the decision\ + \ tree won\u2019t know what to do with numbers that did not appear in the training\ + \ data. The answer is (C)." + - question: "You are reviewing papers for the World\u2019s Fanciest Machine Learning\ + \ Conference, and you see submissions with the following claims. Which ones\ + \ would you consider accepting?\n(A) My method achieves a training error lower\ + \ than all previous methods! (B) My method achieves a test error lower than\ + \ all previous methods! (Footnote: When regularisation parameter \u03BB is chosen\ + \ so as to minimise test error.) (C) My method achieves a test error lower than\ + \ all previous methods! (Footnote: When regularisation parameter \u03BB is chosen\ + \ so as to minimise cross-validaton error.) (D) My method achieves a cross-validation\ + \ error lower than all previous methods! (Footnote: When regularisation parameter\ + \ \u03BB is chosen so as to minimise cross-validaton error.)" + target: "Let's think step by step. In machine learning, we train with some data\ + \ and fixed hyperparameters and the training error can be arbitrarily low, so\ + \ (A) can\u2019t be right. Then, one compares different hyperparameters by selecting\ + \ the model with the lowest cross-validation error, this means that (B) and\ + \ (D) are not the right procedure. The only relevant number after these is the\ + \ test error and thus (C) is the right answer. The answer is (C)." + - question: 'A 6-sided die is rolled 15 times and the results are: side 1 comes up + 0 times; side 2: 1 time; side 3: 2 times; side 4: 3 times; side 5: 4 times; + side 6: 5 times. Based on these results, what is the probability of side 3 coming + up when using Add-1 Smoothing? + + (A) 2.0/15 (B) 1.0/7 (C) 3.0/16 (D) 1.0/5' + target: 'Let''s think step by step. Add-1 smoothing adds the value of one to the + different counts and then normalizes the probabilities accordingly. The counts + after adding one will be: side 1 comes up 1 time; side 2: 2 times; side 3: 3 + times; side 4: 4 times; side 5: 5 times; side 6: 6 times. The number of sum + one die rolls will be 21, so the probability of drawing a three is 3/21 = 1/7. + The answer is (B).' + - question: 'To achieve an 0/1 loss estimate that is less than 1 percent of the true + 0/1 loss (with probability 95%), according to Hoeffding''s inequality the IID + test set must have how many examples? + + (A) around 10 examples (B) around 100 examples (C) between 100 and 500 examples + (D) more than 1000 examples' + target: "Let's think step by step. By the Hoeffding\u2019s inequality, we expect\ + \ that with 95% probability the in-sample and out-of-sample errors differ by\ + \ epsilon when we have N samples if 2 exp(-2 epsilon^2 N)<0.05, this implies\ + \ that N > -1/(2*epsilon**2) log ( 0.05/2 )= log (40)*5000. Since log(40)>1,\ + \ we have that one needs more than 1000 examples. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87d9ba8c9aa31733a5849695213d97deff9c2ded --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml @@ -0,0 +1,46 @@ +dataset_name: management +description: The following are multiple choice questions (with answers) about management. +fewshot_config: + sampler: first_n + samples: + - question: 'How can organisational structures that are characterised by democratic + and inclusive styles of management be described? + + (A) Hierarchical (B) Bureaucratic (C) Flat (D) Functional' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. Flat organizational structures are characterized by democratic and + inclusive styles of management, and have few (if any) levels of management between + the workers and managers. The answer is (C). + - question: 'Hygiene factors are associated with which writer? + + (A) Frederick Hertzberg (B) D.C. McClelland (C) Abraham Maslow (D) Douglas McGregor' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. Hygiene factors include compensation, company policies, supervision, + interpersonal relations, and work environments. Hertzberg lists them as factors + that cannot motivate employees but can minimize job dissatisfaction. The answer + is (A). + - question: 'What characteristic is not a key feature of the ''open systems'' model + of management? + + (A) Morale (B) Innovation (C) Growth resource (D) Adaptation' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. The key characteristics of an open system in management include innovation, + growth resource, and adaption, but do not include morale. The answer is (A). + - question: 'Which element of the cultural web forms regalia? + + (A) Symbols (B) Rituals and routines (C) Power structures (D) Control systems' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. The cultural web is a tool for mapping an organization's culture, + where symbols form the regalia that visually expresses the values that the organization + holds as important. The answer is (A). + - question: 'What are the two main dimensions of the Ohio Studies into leadership? + + (A) Starting position and end position (B) Initial environment and changed environment + (C) Organisational structure and conditioning (D) Initiating structure and considerations' + target: 'Let''s think step by step. We refer to Wikipedia articles on management + for help. The Ohio State Leadership Studies conducted in the 1940s identified + initiating structure and consideration as the two main dimensions of leader + behavior. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_management diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..182eb52ec509c34c225a176774be653d747e120e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml @@ -0,0 +1,56 @@ +dataset_name: marketing +description: The following are multiple choice questions (with answers) about marketing. +fewshot_config: + sampler: first_n + samples: + - question: 'Although the content and quality can be as controlled as direct mail, + response rates of this medium are lower because of the lack of a personal address + mechanism. This media format is known as: + + (A) Care lines. (B) Direct mail. (C) Inserts. (D) Door to door.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Door to door marketing delivers non-addressed items within all buildings + within a geographic area. While it can control the content and quality as well + as direct mail marketing, its response rate is lower because of the lack of + a personal address mechanism. The answer is (D). + - question: 'In an organization, the group of people tasked with buying decisions is + referred to as the _______________. + + (A) Outsourcing unit. (B) Procurement centre. (C) Chief executive unit. (D) + Decision-making unit.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. In an organization, the group of the people tasked with buying decision + is referred to as the decision-making unit. The answer is (D). + - question: 'The single group within society that is most vulnerable to reference group + influence is: + + (A) The older consumer who feels somewhat left out of things. (B) The married + women, many of whom feel a need for stability in their lives. (C) New immigrants + who really want to assimilate into their new culture. (D) Children, who base + most of their buying decisions on outside influences.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Children, who mostly based their buying decisions on outside influences, + are the single group within society that is more vulnerable to reference group + influence. The answer is (D). + - question: 'Which of the following is an assumption in Maslow''s hierarchy of needs? + + (A) Needs are dependent on culture and also on social class. (B) Lower-level + needs must be at least partially satisfied before higher needs can affect behaviour. + (C) Needs are not prioritized or arranged in any particular order. (D) Satisfied + needs are motivators, and new needs emerge when current needs remain unmet.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Maslow's hierarchy of needs, from the bottom upwards, are physiological + (food and clothing), safety, love and belonging needs, esteem, and self-actualization. + Lower-level needs must be at least partially satisfied before higher ones can + affect behavior. The answer is (B). + - question: '_____________ is a natural outcome when combining demographic and geographic + variables. + + (A) Geodemographics (B) Product differentiation. (C) ANSOFF matrix. (D) Brand + management.' + target: 'Let''s think step by step. We refer to Wikipedia articles on marketing + for help. Geodemographics is a natural outcome when combining demographic and + geographic variables. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757010bfed0f08f882995eed787d4f68e0f8121c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml @@ -0,0 +1,51 @@ +dataset_name: medical_genetics +description: The following are multiple choice questions (with answers) about medical + genetics. +fewshot_config: + sampler: first_n + samples: + - question: 'The stage of meiosis in which chromosomes pair and cross over is: + + (A) prophase I (B) metaphase I (C) prophase II (D) metaphase II' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Prophase I is the stage of meiosis where homologous chromosomes pair + with each other and exchange genetic material. The answer is (A). + - question: 'DNA ligase is + + (A) an enzyme that joins fragments in normal DNA replication (B) an enzyme of + bacterial origin which cuts DNA at defined base sequences (C) an enzyme that + facilitates transcription of specific genes (D) an enzyme which limits the level + to which a particular nutrient reaches' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. DNA ligase is a type of enzyme (EC 6.5.1.1) responsible for joining + DNA strands together by catalyzing a phosphodiester bond. The answer is (A). + - question: 'Which of the following conditions does not show multifactorial inheritance? + + (A) Pyloric stenosis (B) Schizophrenia (C) Spina bifida (neural tube defects) + (D) Marfan syndrome' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Multifactorial inheritance is when more than a single factor is responsible + for causing a given trait or health problem. Genes cannot be the only factor. + Marfan syndrome, on the other hand, requires only one abnormal copy of the of + the Marfan gene, from one parent, to inherit the trait. The answer is (D). + - question: 'A gene showing codominance + + (A) has both alleles independently expressed in the heterozygote (B) has one + allele dominant to the other (C) has alleles tightly linked on the same chromosome + (D) has alleles expressed at the same time in development' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Codominance, as it relates to genetics, refers to a type of genetic + inheritance where the phenotype of both the parents is easily observed in the + offspring. A heterozygote is an individual having two different alleles of a + gene. The answer is (A). + - question: 'Large triplet repeat expansions can be detected by: + + (A) polymerase chain reaction. (B) single strand conformational polymorphism + analysis. (C) Southern blotting. (D) Western blotting.' + target: 'Let''s think step by step. We refer to Wikipedia articles on medical + genetics for help. A Southern blot is a method in molecular biology for detecting + specific DNA sequences in a sample. Large triplet repeat expansions are usually + detected with this method. The answer is (C).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fe892eb06f522df6e99606013d26e0df1517cf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml @@ -0,0 +1,43 @@ +dataset_name: miscellaneous +description: The following are multiple choice questions (with answers) about miscellaneous. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of these songs was a Top 10 hit for the rock band The Police? + + (A) ''Radio Ga-Ga'' (B) ''Ob-la-di Ob-la-da'' (C) ''De Do Do Do De Da Da Da'' + (D) ''In-a-Gadda-Da-Vida''' + target: Let's think step by step. We refer to Wikipedia for help. Radio Ga-Ga + is by Queen. Ob-la-di Ob-la-da is by The Beatles. And In-a-Gadda-Da-Vida is + by Iron Butterfly. Leaving 'De Do Do Do De Da Da Da' as the only song by The + Police, and also a Top 10 hit. The answer is (C). + - question: 'What place is named in the title of the 1979 live album by rock legends + Cheap Trick? + + (A) Budapest (B) Budokan (C) Bhutan (D) Britain' + target: Let's think step by step. We refer to Wikipedia for help. Nippon Budokan + is an indoor arena in Tokyo, Japan renowned for hosting rock music concerts + including Cheap Trick in 1978. 'Cheap Trick at Budokan' became the name of their + album. The answer is (B). + - question: 'What is produced during photosynthesis? + + (A) hydrogen (B) nylon (C) oxygen (D) light' + target: Let's think step by step. We refer to Wikipedia for help. Photosynthesis + is the process in which green plants use the green pigment chlorophyll to synthesize + foods with water and carbon dioxide. Oxygen is the byproduct of this process. + The answer is (C). + - question: 'Who is the shortest man to ever win an NBA slam dunk competition? + + (A) Anthony ''Spud'' Webb (B) Michael ''Air'' Jordan (C) Tyrone ''Muggsy'' Bogues + (D) Julius ''Dr J'' Erving' + target: Let's think step by step. We refer to Wikipedia for help. In 1986, Spud + Webb, standing only 5'7" became the shortest NBA player in history to win an + official slam dunk contest. The answer is (A). + - question: 'How many axles does a standard automobile have? + + (A) one (B) two (C) four (D) eight' + target: 'Let''s think step by step. We refer to Wikipedia for help. Most cars + have two axles to rotate the wheels.. The answer is (B).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45a92e075582d6c0e2eb11c0310f87fc5debb4bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml @@ -0,0 +1,64 @@ +dataset_name: moral_disputes +description: The following are multiple choice questions (with answers) about moral + disputes. +fewshot_config: + sampler: first_n + samples: + - question: 'Baron admits that the versions of the ticking bomb hypothetical she discusses + are "stunningly stupid," but she claims this is actually evidence of + + (A) the stupidity of most traditional philosophical examples. (B) a general + lack of intelligence among people with advanced degrees. (C) the wrongness of + torture. (D) the readiness on the part of many intelligent people to see torture + as the best solution to deal with terrorism.' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. The ticking bomb hypothetical poses a problem where many people will + die to an exploding bomb, if the hypothetical terrorist does not disclose how + to defuse it. Baron sees this hypothetical as silly, but its prevalence does + suggest intelligent people, particularly utilitarians, see torture as justifiable + to save the lives in this scenario. The answer is (D). + - question: 'A fertilized ovum is also known as + + (A) a zygote. (B) an embryo. (C) a viability. (D) a blastocyst.' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. Once a single sperm penetrates the layers of an egg to form a new + cell, that cell is called a zygote. The answer is (A). + - question: 'Pence compares six different cases of reproduction, from natural twinning + to SCNT. What conclusion does he draw from this comparison? + + (A) SCNT is not a different kind of reproduction because there are no morally + relevant differences between it and other permissible means of reproduction. + (B) Because there is a low risk of harm for natural twinning, there will be + a low risk of harm for SCNT. (C) both A and B (D) neither A nor B' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. SCNT, known as 'somatic cell nuclear transfer', is a technique in + which the nucleus of a somatic (body) cell is transferred to the cytoplasm of + an enucleated egg. This is used in farm animal cloning. Pence may compare this + to other processes of reproduction, and see little moral separation. The answer + is (A). + - question: 'Which of the following is an intrinsic good, according to Singer? + + (A) being in the upper class (B) the ability to learn and solve problems (C) + being more beautiful than most of the population (D) being shorter than average' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. Bioethicist Peter Singer sees intrinsic value as innate values conferred + by oneself, for oneself. Innanimate objects can be beautiful, short, or have + some valuable criteria, but capabilities are intrinsically good. The answer + is (B). + - question: 'According to Metz, what is wrong with consequentialist arguments against + capital punishment based on African values? + + (A) It is unclear as of yet whether or not capital punishment deters harm to + the community. (B) It is unclear as of yet whether or not capital punishment + deters harm to any individuals. (C) Consequentialism is not supported by African + values. (D) Even though consequentialism is supported by African values, no + consequentialist arguments framed in terms of African values have been offered.' + target: 'Let''s think step by step. We refer to Wikipedia articles on moral disputes + for help. Thaddeus Metz is a humanities research professor focusing on ethical + philosophy in South Africa. Metz has written the death penalty is unjustified + as it treats individuals as incapable of communal relations. It is unclear that + capital punishment is to the benefit of, or a deterrent of harm to the community. + The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4a0231930d62fe73a1cb84b58420baf0182365c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml @@ -0,0 +1,62 @@ +dataset_name: moral_scenarios +description: The following are multiple choice questions (with answers) about moral + scenarios. +fewshot_config: + sampler: first_n + samples: + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I took his wallet after he harassed me at the + bar. Scenario 2 | I made love to my newborn baby boy. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Stealing from someone is not considered an ethical response to harrassment. + And having sex with a minor is egregiously unethical. The answer is (A). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I asked for a cigarette from the guy outside + the store. Scenario 2 | When the security guard left his post, I pocketed a + rare coin. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Asking for something is not morally wrong. Stealing is morally wrong. + The answer is (C). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | During the house''s demolition, I pushed him + into a wall. Scenario 2 | When I choose a babysitter for my twin nieces, I don''t + care about her racial background. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Physical violence is morally wrong. Being open to hiring people from + different racial backgrounds is an expected moral stance. The answer is (B). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I bought an old pick-up truck for work. Scenario + 2 | I looked up at the gorgeous waterfall and threw a rock at it. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Neither of these are wrong as no one is wronged by buing a pick-up + truck or throwing a rock into a stream. The answer is (D). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I really loved my newborn baby boy. Scenario + 2 | The new girl is embarrassed to live in a trailer park, so I invite the class + to her house. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: 'Let''s think step by step. We refer to Wikipedia articles on moral scenarios + for help. Loving someone is not wrong. However, exposing something that someone + is embarrassed about could be considered quite mean. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66498dc564350f893e7bd45078528b5750bce0ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml @@ -0,0 +1,63 @@ +dataset_name: nutrition +description: The following are multiple choice questions (with answers) about nutrition. +fewshot_config: + sampler: first_n + samples: + - question: 'What is the first-line drug for patients with type 2 diabetes and obesity, + as of 2020? + + (A) Acarbose (B) Metformin (C) Sulphonylureas (D) Insulin' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Metformin (Fortamet, Glumetza, or others) is usually the first medication + prescribed for type 2 diabetes, as well as obesity. It works by lowering glucose + production in the liver and improving the body's sensitivity to insulin. The + answer is (B). + - question: 'Which of the following statements is correct (according to knowledge in + 2020)? + + (A) Consumers with phenylketonuria must avoid the consumption of the sweetener + aspartame (B) Consumers with phenylketonuria must avoid the consumption of the + sweetener saccharin (C) Consumers with phenylketonuria must avoid the consumption + of the sweetener sucralose (D) Consumers with phenylketonuria must avoid the + consumption of the sweetener acesulfame K' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. People with phenylketonuria (PKU) cannot break down the amino acid + phenylalanine. As it builds up in the blood and brain it can lead to brain damage. + People with PKU should avoid foods that are converted to phenylalanine in the + body, such as aspartame. The answer is (A). + - question: 'Which of the following statements about iodine is correct, as of 2020? + + (A) 50% of adults consume iodine at levels below the RNI (B) Dairy products + are a poor source of iodine (C) The iodine content of organic milk is generally + lower that the level in non-organic milk (D) UK dietary reference values recommend + an increase in iodine intake in pregnancy' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Organic milk usually has less iodine content than non-organic milk. + The answer is (C). + - question: 'Which of the following is the most plausible explanation for the protective + effect of dietary fibre against cancer of the colon, as of 2020? + + (A) Propionic acid, formed during colonic fibre fermentation inhibits liver + fatty acid synthesis (B) Butyric acid, formed during colonic fibre fermentation + stimulates "silencing" of the SLC5A8 tumour suppressor gene (C) None of these + options are correct (D) Butyric acid, formed during colonic fibre fermentation + stimulates anti-oxidant defences in the colon' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Dietary fibre is inversely proportional to the risk of colorectal + cancer. This is presumed because butyric acid (BA) stimulates antioxidants which + help protect the colon from cancerous tumors. The answer is (D). + - question: 'In a cohort study, the risk ratio of developing diabetes was 0.86 when + comparing consumers of tea (the exposed) to those who did not drink tea (the + unexposed). Which one statement is correct (according to knowledge in 2020)? + + (A) The tea drinkers have lower risk of developing diabetes. (B) The tea drinkers + have higher risk of developing diabetes. (C) Based on the information given + we cannot tell if the observed difference in disease risk is the result of chance. + (D) The risk ratio is close to the value one, so there is no difference in disease + risk between the two groups.' + target: 'Let''s think step by step. We refer to Wikipedia articles on nutrition + for help. The risk ratio is not sufficiently reduced that it could not be explained + by random chance given the studies sample size. The answer is (C).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5116be01d6a463f23f9b7422de1b0b48bbdb55e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml @@ -0,0 +1,44 @@ +dataset_name: philosophy +description: The following are multiple choice questions (with answers) about philosophy. +fewshot_config: + sampler: first_n + samples: + - question: 'The study of reality in the broadest sense, an inquiry into the elemental + nature of the universe and the things in it, is known as _____. + + (A) metaphysics (B) epistemology (C) quantum physics (D) axiology' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Among the options, only metaphysics studies the nature of reality + and existence. The answer is (A). + - question: "According to Moore\u2019s \u201Cideal utilitarianism,\u201D the right\ + \ action is the one that brings about the greatest amount of:\n(A) pleasure.\ + \ (B) happiness. (C) good. (D) virtue." + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Moore's "ideal utilitarianism" states that one's actions should maximize + intrinsic goods. The answer is (C). + - question: 'Before Tolstoy''s Christian conversion, what was his perspective on the + meaning of life? + + (A) optimist (B) satisfied (C) nominally religious (D) pessimist' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Before his conversion, Tolstoy feels that life was uncertain, which + is a pessimist's point of view. The answer is (D). + - question: 'According to d''Holbach, people always act according to _____. + + (A) free choices (B) dictates of the soul (C) necessary natural laws (D) undetermined + will' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. d'Holbach believes that people act according to necessary laws, and + it proves nothing about people's free will. The answer is (C). + - question: 'Psychological egoism is: + + (A) an ethical theory about how we ought to behave. (B) a generalization concerning + the way people tend to behave. (C) a claim about human nature and the ways people + are capable of behaving. (D) none of the above.' + target: 'Let''s think step by step. We refer to Wikipedia articles on philosophy + for help. Psychological egoism suggests that one behaves based on what makes + one feels good, hence it is a claim about human nature and how humans are capable + of behaving. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f9e5d81664445497a13a9adf7ca818ed6d2c7ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml @@ -0,0 +1,59 @@ +dataset_name: prehistory +description: The following are multiple choice questions (with answers) about prehistory. +fewshot_config: + sampler: first_n + samples: + - question: 'What is the approximate mean cranial capacity of Homo erectus? + + (A) under 650 cc (B) about 800 cc (C) just under 1000 cc (D) 1200 cc' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. The average cranium capacity of Homo erectus is less than 1000 cubic + cm. The answer is (C). + - question: 'According to Timothy Pauketat, the evidence for social stratification + and political power at Cahokia suggests: + + (A) a center of Mississippian civilization with conditions similar to the rise + of early states. (B) the limitations of authority in a Native American society + of egalitarian foragers. (C) a simple chiefdom or perhaps a complex chiefdom + had evolved by A.D. 1500. (D) a center of Mississippian civilization with conditions + similar to societies on the Northwest Coast of North America.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Timothy Pauketat is known for his research on Cahokia, the center + of the Mississippian culture, where he found similar conditions to the rise + of early states. The answer is (A). + - question: 'Recent research on hominid species dating from the Middle Pliocene indicates + there was (as of 2020): + + (A) a great amount of species diversity, or a single species that exhibited + a lot of diversity. (B) very little species diversity during this period and + very few hominids. (C) decreased species diversity due to a prolonged ice age + followed by a severe drought. (D) decreased species diversity but increased + numbers of hammerstones and flakes, indicating stone tool manufacture.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Recent research has recognized multiple hominid species from the Middle + Pliocene, meaning that there is a great amount of species diversity or diversity + in a single species. The answer is (A). + - question: 'Researchers now believe that the decline of the Maya was caused chiefly + by: + + (A) a cataclysm of some kind, such as an earthquake, volcano, or tsunami. (B) + ecological degradation resulting from slash-and-burn farming techniques. (C) + endless wars between neighboring Mayan city-states. (D) practices of interbreeding + that led to a steep rise in congenital disorders.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Researchers believe that the Maya collapse was mainly caused by over-exploitation + of natural resources like the slash-and-burn farming techniques. The answer + is (B). + - question: 'The great Mayan king Pacal built temples in the city of Palenque in order + to: + + (A) satisfy the powerful Mayan astronomer priests. (B) display his generosity + to the common people, since they were allowed to live in the temples. (C) frighten + away enemies, in particular the Spaniards. (D) legitimize his kingship, since + his father was not royal.' + target: 'Let''s think step by step. We refer to Wikipedia articles on prehistory + for help. Pacal built the temples as the funerary monument to legitimize his + kingship. The answer is (D).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d18fc22626b952d18491b849cabc720706c17c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml @@ -0,0 +1,63 @@ +dataset_name: professional_accounting +description: The following are multiple choice questions (with answers) about professional + accounting. +fewshot_config: + sampler: first_n + samples: + - question: "An auditor traces the serial numbers on equipment to a nonissuer\u2019\ + s subledger. Which of the following management assertions is supported by this\ + \ test?\n(A) Valuation and allocation (B) Completeness (C) Rights and obligations\ + \ (D) Presentation and disclosure" + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The completeness assertion is tested by tracing supporting documents + to the record entries. The answer is (B). + - question: 'One hundred years ago, your great-great-grandmother invested $100 at 5% + yearly interest. What is the investment worth today? + + (A) $13,000 (B) $600 (C) $15,000 (D) $28,000' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. A $100 investment at 5% yearly interest is worth 100*(1.05)^100=13150 + after 100 years, which is around $13,000. The answer is (A). + - question: 'On January 1, year 1, Alpha Co. signed an annual maintenance agreement + with a software provider for $15,000 and the maintenance period begins on March + 1, year 1. Alpha also incurred $5,000 of costs on January 1, year 1, related + to software modification requests that will increase the functionality of the + software. Alpha depreciates and amortizes its computer and software assets over + five years using the straight-line method. What amount is the total expense + that Alpha should recognize related to the maintenance agreement and the software + modifications for the year ended December 31, year 1? + + (A) $5,000 (B) $13,500 (C) $16,000 (D) $20,000' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The maintenance period begins on March 1, so only 10 months of expenses + should be recognized, which is $15,000/12*10=$12,500. The software modification + cost is amortized over 5 years, so each year is $5,000/5=$1,000. So the total + expense is $12,500+$1,000=$13,500. The answer is (B). + - question: 'Krete is an unmarried taxpayer with income exclusively from wages. By + December 31, year 1, Krete''s employer has withheld $16,000 in federal income + taxes and Krete has made no estimated tax payments. On April 15, year 2, Krete + timely filed for an extension request to file her individual tax return, and + paid $300 of additional taxes. Krete''s year 1 tax liability was $16,500 when + she timely filed her return on April 30, year 2, and paid the remaining tax + liability balance. What amount would be subject to the penalty for underpayment + of estimated taxes? + + (A) $0 (B) $500 (C) $1,650 (D) $16,500' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The tax due after withholding is $16,500-$16,000=$500, which is less + than $1000, hence there is no underpayment penalty of estimated taxes. The answer + is (A). + - question: 'Box a nongovernmental not-for-profit organization had the following transactions + during the year: Proceeds from sale of investments $80000 Purchase of property + plant and equipment $10000 Proceeds from long-term debt $100000 Loss on sale + of investment $5000 What amount should be reported as net cash provided by financing + activities in Box''s statement of cash flows? + + (A) $70,000 (B) $75,000 (C) $80,000 (D) 100000' + target: 'Let''s think step by step. We refer to Wikipedia articles on accounting + for help. Among the four transactions, only Proceeds from long-term debt belongs + to the financing activities section of cashflow, hence the amount reported should + be $100000. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307f8940bc445305fdbf00e89910cd5237a41312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml @@ -0,0 +1,122 @@ +dataset_name: professional_law +description: The following are multiple choice questions (with answers) about professional + law. +fewshot_config: + sampler: first_n + samples: + - question: 'A son owed a creditor $5,000. The son''s father contacted the creditor + and told him that he wanted to pay the son''s debt. The father signed a document + that stated the father would pay the son''s debt at a rate of $500 a month for + 10 months. The creditor made no written or oral commitment to forbear to sue + the son to collect the $5,000 debt, and the father made no oral or written request + for any such forbearance. For the next five months, the father made and the + creditor accepted the $500 monthly payments as agreed. During that period, the + creditor, in fact, did forbear to take any legal action against the son. However, + the father then informed the creditor that he would make no further payments + on the debt. Which of the following is the most persuasive argument that the + father is liable to the creditor under the terms of their agreement? + + (A) The father''s promise and the creditor''s reliance thereon, if proved, gave + rise to a valid claim by the creditor against the father based on the doctrine + of promissory estoppel. (B) Because it was foreseeable that the father''s promise + would induce the creditor to forbear taking any action against the son, such + forbearance was, as a matter of law, a bargained-for consideration for the father''s + promise. (C) The father''s five payments to the creditor totaling $2,500 manifested + a serious intent on the father''s part to be contractually bound, and such manifestation + is generally recognized as an effective substitute for consideration. (D) By + assuming the antecedent debt obligation that the son owed to the creditor, the + father became a surety whose promise to the creditor was enforceable, since + it was in writing and supported by adequate consideration. ' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The doctrine of promissory estoppel stops a person from going back on a promise + in contract law, hence option (A) should be the most persuasive argument. The + answer is (A). + - question: 'A state has recently enacted a statute prohibiting the disposal of any + nuclear wastes within the state. This law does not contravene or conflict with + any federal statutes. A man operates a company in the state that is engaged + in the disposal of nuclear wastes. Subsequent to the passage of the state statute, + the man, not yet aware of the new law, entered into contracts with many out-of-state + firms to dispose of their nuclear wastes in the state. On account of this new + law, however, the man will be unable to perform these contracts. Assume that + the man has standing to challenge this state law. Which of the following presents + his strongest constitutional grounds to challenge the state law prohibiting + the disposal of nuclear wastes within the state? + + (A) The commerce clause. (B) The equal protection clause of the Fourteenth Amendment. + (C) The privileges and immunities clause of Article IV, Section 2. (D) The contract + clause.' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The commerce clause states that Congress shall have the power to regulate commerce + with foreign Nations, and among the several States, and with the Indian Tribes. + The statute affects inter-state commerce which puts it into question. Hence + the man's strongest argument should be the commerce clause. The answer is (A). + - question: 'On October 1, 1980, a developer, owner of several hundred acres in a rural + county, drafted a general development plan for the area. The duly recorded plan + imposed elaborate limitations and restrictions upon the land in the plan, which + was to be developed as a residential district. The restrictions were to extend + to all persons acquiring any of the lots and to their heirs, assigns, and lessees. + It was further provided that all subsequent owners would be charged with due + notice of the restrictions. Among those restrictions in the general plan were + the following:(22) A franchise right is created in a strip of land 10 feet in + width along the rear of each lot for the use of public utility companies with + right of ingress and egress. (23) No house or structure of any kind shall be + built on the aforementioned strip of land running through the said blocks. In + 2000, a retiree purchased one of the lots, built a house, and erected a fence + in the rear of his property within the restricted area. In 2004, a teacher purchased + a lot adjacent to the retiree''s property and built a new house. Two years later, + a librarian purchased the lot that adjoined the teacher''s property. The three + deeds to those properties each contained references to the deed book where the + general plan was recorded. In 2008, the librarian began the construction of + a seven-foot post-and-rail fence along the line dividing his lot with the teacher''s, + and along the center of the area subject to the franchise right. Although the + teacher objected to its construction, the fence was completed. If the teacher + seeks a mandatory injunction to compel removal of the librarian''s fence, the + court will most likely + + (A) grant relief, because the fence was in violation of the easement restriction. + (B) grant relief, because the encroachment of the fence violated the restriction + in the original plan. (C) deny relief, because the teacher failed to enforce + the restriction against the retiree. (D) deny relief, because the fence would + not be construed as "a structure" within the terms of the restriction. ' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The restrictions in the original plan say no house or structure of any kind + shall be built on the aforementioned strip of land running through the said + blocks. Hence the court will most likely grant relief because the fence violated + the restriction in the original plan. The answer is (B). + - question: 'Judge took judicial notice of some facts at the beginning of the trial. + Which of the following is not an appropriate kind of fact for judicial notice? + + (A) Indisputable facts. (B) Facts that have been asserted by individual political + organizations. (C) Facts recognized to be true by common knowledge. (D) Facts + capable of scientific verification.' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + Among the options, facts that have been asserted by individual political organizations + is not an appropriate kind of fact for judicial notice. The answer is (B). + - question: 'A state legislature has recently enacted a statute making it a misdemeanor + to curse or revile or use obscene or opprobrious language toward or in reference + to a police officer perfonning his duties. A student at a state university organized + a demonstration on campus to protest the war. The rally was attended by a group + of 50 students who shouted anti-war messages at cars passing by. To show his + contempt for the United States, the student sewed the American flag to the rear + of his jeans. When a police officer saw the flag sown on the student''s jeans, + he approached and told him to remove the flag or he would be placed under arrest. + The student became angered and shouted at the police officer, "Listen, you bastard, + I''ll wear this rag anywhere I please. " The student was subsequently placed + under arrest and charged with violating the state statute. The student subsequently + brings suit in state court challenging the constitutionality of the statute. + The strongest constitutional argument for the student is that + + (A) the statute is void for vagueness under the Fourteenth Amendment''s due + process clause. (B) the statute is invalid because it violates the petitioner''s + freedom of speech under the First Amendment. (C) the statute is an abridgment + of freedom of speech under the First Amendment because less restrictive means + are available for achieving the same purpose. (D) the statute is overbroad and + consequently invalid under the First and Fourteenth Amendments.' + target: 'Let''s think step by step. We refer to Wikipedia articles on law for + help. The Fourteenth Amendment further supports the First Amendment by establishing + a due process clause. Hence the strongest argument should be the statute is + overbroad and consequently invalid under the First and Fourteenth Amendments. + The answer is (D).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a5b0d995ae7d0897eaea3a6cfd0719562cdc3c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml @@ -0,0 +1,82 @@ +dataset_name: professional_medicine +description: The following are multiple choice questions (with answers) about professional + medicine. +fewshot_config: + sampler: first_n + samples: + - question: "A 22-year-old male marathon runner presents to the office with the complaint\ + \ of right-sided rib pain when he runs long distances. Physical examination\ + \ reveals normal heart and lung findings and an exhalation dysfunction at ribs\_\ + 4-5 on the right. Which of the following muscles or muscle groups will be most\ + \ useful in correcting this dysfunction utilizing a direct method?\n(A) anterior\ + \ scalene (B) latissimus dorsi (C) pectoralis minor (D) quadratus lumborum" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. Among the options, only pectoralis minor muscle origins from the outer + surfaces of the 3rd to 5th ribs. The answer is (C). + - question: "A 36-year-old male presents to the office with a\_3-week\_history of low\ + \ back pain. He denies any recent trauma but says that he climbs in and out\ + \ of his truck numerous times a day for his job. Examination of the patient\ + \ in the prone position reveals a deep sacral sulcus on the left, a posterior\ + \ inferior lateral angle on the right, and a lumbosacral junction that springs\ + \ freely on compression. The most likely diagnosis is\n(A) left-on-left sacral\ + \ torsion (B) left-on-right sacral torsion (C) right unilateral sacral flexion\ + \ (D) right-on-right sacral torsion" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The deep sulcus on the left, a posterior ILA on the right, with a negative + spring test suggests a right-on-right sacral torsion. All other options have + a deep sulcus on the right. The answer is (D). + - question: "A 44-year-old man comes to the office because of a 3-day history of sore\ + \ throat, nonproductive cough, runny nose, and frontal headache. He says the\ + \ headache is worse in the morning and ibuprofen does provide some relief. He\ + \ has not had shortness of breath. Medical history is unremarkable. He takes\ + \ no medications other than the ibuprofen for pain. Vital signs are temperature\ + \ 37.4\xB0C (99.4\xB0F), pulse 88/min, respirations 18/min, and blood pressure\ + \ 120/84 mm Hg. Examination of the nares shows erythematous mucous membranes.\ + \ Examination of the throat shows erythema and follicular lymphoid hyperplasia\ + \ on the posterior oropharynx. There is no palpable cervical adenopathy. Lungs\ + \ are clear to auscultation. Which of the following is the most likely cause\ + \ of this patient's symptoms?\n(A) Allergic rhinitis (B) Epstein-Barr virus\ + \ (C) Mycoplasma pneumonia (D) Rhinovirus" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The symptoms, especially the headache, suggest that the most likely cause + is Rhinovirus. Epstein-Barr virus will cause swollen lymph nodes but there is + no palpable cervical adenopathy. Lungs are clear to auscultation suggests it's + not Mycoplasma pneumonia. The answer is (D). + - question: 'A previously healthy 32-year-old woman comes to the physician 8 months + after her husband was killed in a car crash. Since that time, she has had a + decreased appetite and difficulty falling asleep. She states that she is often + sad and cries frequently. She has been rechecking the door lock five times before + leaving her house and has to count exactly five pieces of toilet paper before + she uses it. She says that she has always been a perfectionist but these urges + and rituals are new. Pharmacotherapy should be targeted to which of the following + neurotransmitters? + + (A) Dopamine (B) Glutamate (C) Norepinephrine (D) Serotonin' + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The patient feels sad and among the options, only Dopamine and Serotonin + can help increase positive emotions. Serotonin also affects digestion and metabolism, + which can help the patient's decreased appetite and sleep difficulty. The answer + is (D). + - question: "A 42-year-old man comes to the office for preoperative evaluation prior\ + \ to undergoing adrenalectomy scheduled in 2 weeks. One month ago, he received\ + \ care in the emergency department for pain over his right flank following a\ + \ motor vehicle collision. At that time, blood pressure was 160/100 mm Hg and\ + \ CT scan of the abdomen showed an incidental 10-cm left adrenal mass. Results\ + \ of laboratory studies, including complete blood count, serum electrolyte concentrations,\ + \ and liver function tests, were within the reference ranges. The patient otherwise\ + \ had been healthy and had never been told that he had elevated blood pressure.\ + \ He takes no medications. A follow-up visit in the office 2 weeks ago disclosed\ + \ elevated urinary normetanephrine and metanephrine and plasma aldosterone concentrations.\ + \ The patient was referred to a surgeon, who recommended the adrenalectomy.\ + \ Today, vital signs are temperature 36.6\xB0C (97.9\xB0F), pulse 100/min, respirations\ + \ 14/min, and blood pressure 170/95 mm Hg. Physical examination discloses no\ + \ significant findings. Initial preoperative preparation should include treatment\ + \ with which of the following?\n(A) Labetalol (B) A loading dose of potassium\ + \ chloride (C) Nifedipine (D) Phenoxybenzamine" + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. The symptoms and the adrenal mass suggested pheochromocytoma, and + the blood pressure indicates hypertension. Phenoxybenzamine is used to treat + hypertension caused by pheochromocytoma. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60b5da683ff87d207304b894a5138a6c439a1c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml @@ -0,0 +1,62 @@ +dataset_name: professional_psychology +description: The following are multiple choice questions (with answers) about professional + psychology. +fewshot_config: + sampler: first_n + samples: + - question: 'In the construction of a multiple regression equation for purposes of + prediction, the optimal combination of measures is one in which the predictors + + (A) are uncorrelated with each other but are moderately correlated with the + criterion (B) have low correlations with each other and low correlations with + the criterion (C) are highly intercorrelated with each other and moderately + correlated with the criterion (D) have low correlations with the criterion bur + are moderately correlated with each other' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. The basis of multiple regression is to assess the relationship between + one continuous variable and a set of independent variables. So the predictors + should be uncorrelated with each other but are moderately correlated with the + criterion. The answer is (A). + - question: 'There are three ways to measure the Central Tendency: the Mean, the Median + and the Mode. From your knowledge about them, what is the mode? + + (A) less sensitive to extreme scores than the mean (B) more useful for skewed + distributions (C) sensitive to extreme values and highly skewed distributions + (D) the most frequently occurring number' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. The definition of mode is the most frequently occurring number. The + answer is (D). + - question: "Carl Jung believed that a client's transference:\n(A) is a fantasy that\ + \ distracts the client from reality. (B) represents \u201Cmixed feelings\u201D\ + \ toward the therapist. (C) \"is a form of \"\"acting out.\"\"\" (D) reflects\ + \ the client\u2019s personal and collective unconscious." + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Transference is a phenomenon that a person's feelings are unconsciously + redirected, so it reflects the client's personal and collective unconscious. + The answer is (D). + - question: "In terms of Hofstede\u2019s (1980) five cultural dimensions, the United\ + \ States scores at the top of the scale on:\n(A) individualism. (B) individualism\ + \ and power distance. (C) power distance and masculinity. (D) uncertainty avoidance." + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. US scores highest on individualism among the five cultural dimensions. + The answer is (A). + - question: 'One of your therapy clients asks your advice about a good weight- reduction + program. You have investigated the programs in the community and are enrolled + in the one you consider the best. This program offers a $50 bonus to its patrons + for each new person they bring into the program. Under these circumstances, + your most appropriate response would be to + + (A) tell your client the pros and cons of each program you know about except + for the one in which you are enrolled (B) recommend to your client the program + in which you are enrolled and explain the $50 bonus you will receive (C) recommend + to your client the program in which you are enrolled and offer to have the $50 + bonus credited to your client''s account in the program (D) tell your client + the pros and cons of each program you know about, but do not claim the $50 bonus + if your client enrolls in your program' + target: 'Let''s think step by step. We refer to Wikipedia articles on psychology + for help. Based on the circumstances, you should tell your client about the + pros and cons of each program, but it would be inappropriate to receive the + bonus, so you should not claim the $50 bonus. The answer is (D).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe384b1e2b7d19c216f8344d5c249f2c16dc723b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml @@ -0,0 +1,55 @@ +dataset_name: public_relations +description: The following are multiple choice questions (with answers) about public + relations. +fewshot_config: + sampler: first_n + samples: + - question: 'Earth Hour was a campaign launched by which organization? + + (A) Greenpeace (B) The UN (C) Oxfam (D) World Wildlife Fund' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. Earth Hour is a worldwide movement oragnized launched by the World + Wildlife Fund. The answer is (D). + - question: 'In issues management, what is the most proactive approach to addressing + negative or misleading information posted online about your organization? + + (A) Buy domain names that could be used by opposition groups. (B) Post anonymous + comments on blogs to combat this information. (C) Prepare a news release that + discredits the inaccurate information. (D) Make policy changes to address complaints + highlighted on these sites.' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. In issues management, the most proactive approach to addressing negative + or misleading information posted online is to make policy changes to address + complaints highlighted on those sites. The answer is (D). + - question: 'At which stage in the planning process would a situation analysis be carried + out? + + (A) Defining the program (B) Planning the program (C) Taking action and implementing + ideas (D) Evaluation of the program' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. Situation analyses are typically carried out during the planning process + stage of defining the program. The answer is (A). + - question: 'Which of these statements is true of the Vatican in 2010 at the time of + the accusations of child abuse cover-ups? + + (A) There was a coordinated media response. (B) Consistent messages were communicated. + (C) Criticisms were taken as attacks on the Catholic Church. (D) The credibility + of the Vatican was upheld.' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. In 2010 when there were accusations of child abuse cover-ups, the + Vatican took those criticisms as attacks on the Catholic Church. The answer + is (C). + - question: 'What should a public relations media practitioner do if she does not know + the answer to a reporter''s question? + + (A) Give the reporter other information she is certain is correct. (B) Say that + the information is ''off the record'' and will be disseminated later. (C) Say + ''I don''t know'' and promise to provide the information later. (D) Say ''no + comment,'' rather than appear uninformed.' + target: 'Let''s think step by step. We refer to Wikipedia articles on public relations + for help. If a public relations media practitioner does not know the answer + to a reporter''s question, they should say ''I don''t know'' and offer to provide + the information later. The answer is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b37e35b3bd4fefa0ca040f0d59ff2fcae156fb45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml @@ -0,0 +1,104 @@ +dataset_name: security_studies +description: The following are multiple choice questions (with answers) about security + studies. +fewshot_config: + sampler: first_n + samples: + - question: 'What are the frameworks of analysis within which terrorism has been considered + (as of 2020)? + + (A) Competition between larger nations has resulted in some countries actively + supporting terrorist groups to undermine the strength of rival states. Terrorist + networks are extended patronage clubs maintained and paid for by their donor + states and are conceptualised as being like state actors, to be dealt with using + military force. (B) Globalization has enabled the internationalization of terrorist + activities by opening up their operational space, although coordination is still + managed from a geographical base. This suggests that terrorist groups are nationally + structured which means that terrorism cannot be considered in terms of a war + to be defeated militarily without having serious implications on the indigenous + population. (C) Terrorism can be viewed as a problem to be resolved by military + means (war on terrorism), by normal police techniques (terrorism as crime), + or as a medical problem with underlying causes and symptoms (terrorism as disease). + (D) Terrorism is viewed as a criminal problem. The criminalization of terrorism + has two important implications. Firstly, it suggests that terrorism can be eradicated + - terrorists can be caught and brought to trial by normal judicial proceedings + thereby removing the threat from society - and secondly, it suggests that preventative + crime techniques are applicable to prevent its development.' + target: "Let's think step by step. We refer to Wikipedia articles on security\ + \ studies for help. (A) is wrong because it is not competition between larger\ + \ nations that causes terrorism. \n(B) is wrong because globalization is not\ + \ the cause of terrorism.\n(C) is correct because the US undertook the war on\ + \ terrorism. \n(D) is wrong because preventative crime techniques will likely\ + \ not end terrorism. The answer is (C)." + - question: 'Which of the following is the best lens through which to investigate the + role of child soldiers? + + (A) Child soldiers are victims of combat that need re-education and rehabilitation. + (B) Children and their mothers are not active subjects in warfare and are best + considered as subjects in the private sphere. (C) Children are most often innocent + bystanders in war and are best used as signifiers of peace. (D) Children have + political subjecthood that is missed when they are considered as passive victims + of warfare.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. Child soliders as a political topic can be missed when they are considered + passive victims of warfare. The answer is (D). + - question: 'How can we best describe the relationship between the state-centric approach + and the concept of human security? + + (A) There are such wide divisions within the human security framework regarding + the nature of threats and referent objects that no widely applicable comparisons + between state-centric approaches and human security can be drawn. (B) By adopting + the framework of human security, the limitations of the realist state-centric + approach become evident. Whilst human security defines the referent object as + the person or population, state-centric approaches prioritise the security of + the state, de-prioritizing the pursuit of human security. (C) The state-centric + approach to security is a faction of human security, usually defined within + the broad school of human security. By being state-centric this approach prioritises + the individual as the referent object in security studies. (D) Both the state-centric + and human-centric approaches to security are mutually exclusive and offer a + sufficient analytic framework with which to understand the international security + system. It is therefore the role of security analysts to determine which of + these substantial concepts is correct, and which should be discarded.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. Human security focuses on a person or population whereas state-centric + approaches focus on the state while deprioritizing human security. The answer + is (B). + - question: 'In order to become securitized, a threat must be presented in which of + these ways? + + (A) As an existential threat that requires immediate and extraordinary action, + posing a threat to the survival of the state or to societal security. (B) As + requiring immediate and extraordinary action by the state, threatening the survival + of a referent object and therefore warranting the use of measures not normally + employed in the political realm. (C) As an urgent threat to the survival of + the referent object, so serious that it legitimises the employment of extraordinary + action in response. (D) As an urgent threat to the survival of the audience + that requires extraordinary or emergency measures.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. To be securitized, a threat must be an urgent threat to the survival + of the referent object. The answer is (C). + - question: 'What distinguishes coercive diplomacy from military force? + + (A) Compellence is another term for coercive diplomacy, but covering a narrower + set of criteria; compellence covers those threats aimed at initiating adversary + action. A threat to coerce a state to give up part of its territory would count + as coercive diplomacy, as long as that threat proactively initiates action before + reactive diplomacy is taken. (B) Coercive diplomacy constitutes the threats + of limited force to induce adversary''s incentive to comply with the coercer''s + demands. It is an influence strategy that is intended to obtain compliance: + the use of force to defeat an opponent first does not count. It leaves an element + of choice with the target to comply, or to continue. (C) Military force, or + the threat of military force, utilises fear to achieve strategic objectives. + Coercive diplomacy is differentiated from this approach, because it does not + use fear as a tool for coercing an adversary. (D) Coercive diplomacy is employed + to use force but to limit its effects on the international community. Coercive + diplomacy is an aggressive strategy that is intended to obtain compliance through + defeat. It does not leave an element of choice with the target, the target either + being forced to comply or engage in conflict. It seeks to control by imposing + compliance by removing any opportunity for negotiation or concession.' + target: 'Let''s think step by step. We refer to Wikipedia articles on security + studies for help. Coercive diplomacy uses the threat of force to induce the + opponent to comply with demands. The answer is (B).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4229d64785ded8673d421a9fb1571d0cce705a93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml @@ -0,0 +1,58 @@ +dataset_name: sociology +description: The following are multiple choice questions (with answers) about sociology. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is not a problem associated with official statistics + on strike action? + + (A) most strikes go unnoticed by employers and the mass media (B) not all industrial + disputes will be reported by the employer (C) the definition of strikes excludes + those that involve fewer than ten workers or last less than one day (D) it is + hard to compare strikes that were measured in different ways' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Official statistics on strike action can be problematic because not + all industrial disputes will be reported by employers, the definition of strikes + excludes those that involves fewer than ten workers or last less than one day, + and it is hard to compare strikes that were measured in different ways. Thus, + (A) is not a problem associated with official statistics on strike action. The + answer is (A). + - question: 'What does Berger (1963) describe as a metaphor for social reality? + + (A) a fairground ride (B) a circus (C) a puppet theatre (D) a ballet' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Berger describes social reality using the metaphor of a puppet theatre. + The answer is (C). + - question: 'The term ''hegemony'' refers to: + + (A) the tendency for the working class not to realize their own interests (B) + a dominant ideology that legitimates economic, political and cultural power + (C) a form of dual consciousness based on ideology and everyday experiences + (D) a mode of payment given for outstanding topiary' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Hegemony refers to a dominant ideology that legitimates economic, + policital, and cultural power. The answer is (B). + - question: 'The shift from ''civil religion'' to ''common religion'' means that: + + (A) the increasing bureaucracy of the state has made religion only a marginal + part of our lives (B) despite the weakening of traditional authority, our everyday + lives and ''common sense'' remain shaped by religious beliefs and values (C) + religious participation in collective worship may have declined, but people + still practise their faiths in private (D) people are much more likely to discuss + their religious beliefs in public, informal settings' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. The shift from civil religion to common religion means that despite + the weakening of traditional authority, our everyday lives and common sense + remain shaped by religious beliefs and values. The answer is (B). + - question: 'Which of the following did the post-war welfare state of 1948 not aim + to provide: + + (A) free health care and education for all (B) a minimum wage (C) full employment + (D) universal welfare' + target: 'Let''s think step by step. We refer to Wikipedia articles on sociology + for help. The post-war welfare state of 1948 aimed to provide free healthcare + and education, full employment, and universal welfare. But it did not aim to + provide a minimum wage. The answer is (B).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc86b7c88fa1b62d2f12deaa16394f43fc722225 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml @@ -0,0 +1,56 @@ +dataset_name: us_foreign_policy +description: The following are multiple choice questions (with answers) about us foreign + policy. +fewshot_config: + sampler: first_n + samples: + - question: 'How did Donald Trump attack globalization in the 2016 campaign? + + (A) Globalization had made men like him too rich (B) Globalization only benefited + certain American states, such as New York (C) Liberal elites had encouraged + globalization, while ''ordinary Americans'' lost jobs because of it (D) Globalization + encouraged damaging trade wars' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. Trump attacked globalization because he believed ordinary Americans + lost jobs due to it, and so he wanted to blame liberals who had encouraged it. + The answer is (C). + - question: 'How did NSC-68 change U.S. strategy? + + (A) It globalized containment. (B) It militarized containment. (C) It called + for the development of the hydrogen bomb. (D) All of the above' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. NSC-68 outlined a variety of courses of action, including globalization + of containment, militarization of contaiment, and the development of the hydrogen + bomb. The answer is (D). + - question: 'How do Defensive Realism and Offensive Realism differ in their explanation + of state behaviour? + + (A) Defensive realists place greater emphasis on the role of international institutions + (B) Defensive realists place less emphasis on geographical factors (C) Offensive + realists give more priority to the national interest than Defensive realists. + (D) Defensive realists believe states are security maximizers, while Offensive + realists believe states to be power maximizers' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. While defensive realism advocates that states are security + maximizers, offensive realists think of states as power maximizers. The answer + is (D). + - question: 'The realm of policy decisions concerned primarily with relations between + the United States and the rest of the world is known as + + (A) terrorism policy. (B) economic policy. (C) foreign policy. (D) international + policy.' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. The topic of policy decisions concerns with relations between + the US and the rest of the world is known as foreign policy. The answer is (C). + - question: 'How did the 2008 financial crisis affect America''s international reputation? + + (A) It damaged support for the US model of political economy and capitalism + (B) It created anger at the United States for exaggerating the crisis (C) It + increased support for American global leadership under President Obama (D) It + reduced global use of the US dollar' + target: 'Let''s think step by step. We refer to Wikipedia articles on us foreign + policy for help. The 2008 financial crisis damanged the international reputation + of the American model of political economy and capitalism. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e70f0ee24cc946a1cfbc51cb87d4fde20d8171c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml @@ -0,0 +1,45 @@ +dataset_name: virology +description: The following are multiple choice questions (with answers) about virology. +fewshot_config: + sampler: first_n + samples: + - question: 'The median survival time to AIDS and death was established by following: + + (A) Seroprevalent HIV-infected individuals (B) Seronegatives (C) Seroconverters + (D) High-risk seronegatives' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The median survival time to AIDS and death was established as a result + of the development of seroconverters. The answer is (C). + - question: 'Which of the following is a morphological characteristic of the paramyxoviruses. + + (A) Fragile viruses often visualised with RNA spewing from the inside (B) Elongate + viruses (C) Icosahedral viruses with envelope (D) Very large viruses' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. Paramyxoviruses are fragile viruses often visualised with RNA spewing + from the inside. The answer is (A). + - question: 'The most important goal of a behavioral intervention is: + + (A) Change in behavior (B) Comprehensive coverage (C) Effective use of behavioral + theory (D) Sustained behavior change' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The prim goal of a behavioral intervention is to cause sustained behavior + change. The answer is (D). + - question: 'A key factor facilitating the application of nested case-control studies + from the MACS was: + + (A) Data collection (B) Establishment of a repository of biologic specimens + (C) Participant interest (D) Administration of the questionnaire by staff' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The Multicenter AIDS Cohort Study's use of nested case-control studies + was facilitated by the establishment of a repository of biologic specimens. + The answer is (B). + - question: 'Why are parvoviruses a highly impactful parasite? + + (A) Because they have no nucleic acid (B) They require a helper virus (C) Only + replicate in dividing cells (D) Can integrate into host chromosomes' + target: 'Let''s think step by step. We refer to Wikipedia articles on virology + for help. Paroviruses are highly impactful because they do not have nucleic + acid. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41502cc7a3a1318b7c6a0f2ac16cda86dda08486 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml @@ -0,0 +1,42 @@ +dataset_name: world_religions +description: The following are multiple choice questions (with answers) about world + religions. +fewshot_config: + sampler: first_n + samples: + - question: 'How can the Upanishads be characterized? + + (A) Ritual texts (B) Philosophical texts (C) Hymns (D) Origin stories' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The Upanishads are the most recent part of Vedas (the oldest scriptures + in Hinduism) and supplied the basis of later Hindu philosophy. So they are philosophical + texts. The answer is (B). + - question: 'What is the Second Gem in Buddhism? + + (A) The Dharma (B) The Sangha (C) The Buddha (D) The Bodhisattva' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The Second Gem in Buddhism is The Dharma. The answer is (A). + - question: 'Which Japanese government promoted a kind of national cult based on the + emperor and his associations with kami? + + (A) Honen (B) Tanaka (C) Tokugawa (D) Meiji' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The promotion of a national cult based on the emperor and his associations + with Kami happened during the reign of Emperor Meiji (1852-1912). The answer + is (D). + - question: 'In which dynasty was the "Mandate of Heaven" developed to legitimatize + the new rulers? + + (A) Shang (B) Zhou (C) Han (D) Xia' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The "Mandate of Heaven" was developed as an ancient Chinese philosophical + concept during the Zhou Dynasty (1046-256 BCE). The answer is (B). + - question: 'What is the sign of the covenant for Jewish males? + + (A) The rainbow (B) Circumcision (C) A son (D) Bar mitzvah' + target: 'Let''s think step by step. We refer to Wikipedia articles on world religions + for help. In Judaism, the most distinctive sign of the covenant is circumcision + (brit milah). The answer is (B).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..745a892568bd84b38252e20bbc9a0bea73ddb1db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_cot_zeroshot +group_alias: mmlu (flan style, zeroshot cot) +task: + - group: stem + task: + - mmlu_flan_cot_zeroshot_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_cot_zeroshot_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_cot_zeroshot_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_cot_zeroshot_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7588b67e1905dc7eae3790c33a16de460c465f67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml @@ -0,0 +1,38 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +validation_split: validation +fewshot_split: dev +output_type: generate_until +doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step." +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer]}}" +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.))" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: -1 + ignore_case: true + ignore_punctuation: true + regex_pattern: "(\\([A-Z]\\))" + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +num_fewshot: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 3.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e885b818eae4bbc87374c756b68ecd11e44bd69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f17410a7cc0869223730328f55803d8d424e930 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5b821f97642ad5987244a0ac4c9988c2fca3857 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b26c679e26b6bd04d77eb5e0bb2ebaddcc515561 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c0e9d17db10f4e69d1c44d5a127f2bbe1f4e279 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_clinical_knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de020f4eaca7fdeb650688f034ee3b5d89490ddc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8e5bbcf76b9fb3ad012511b213ffbbd554cd58d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04b5e750949984abcd7889be80485e52c97dba9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81c59cc2c20f340a76ed3d945e976ce3c832815c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0450a068f4b763629e463d9882e4a3e99f86d726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82c2bb2ab586be2346237a6aa8b2ea9fd9170c97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78216a44778fa0f9f1e057d5dc45b998fd5e87fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52304bdf8eeac624c63331b259255a98866dc2ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_conceptual_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5be81c442710f91ad3e1ca6a0651105b2f14e24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..934a1a20a69d987904fe9c8b605c93e4ed149309 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96ec81d6a8716ad60a4b3215faa42f3c3b1396d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..915c96de78b68bdd2b8b8cbb26f2f8ec0ae24167 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a1f7491590b80e784360ceb72619efe4d9568f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_global_facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c4043d9bd7e6a38d702afa7ccb4028e98001445 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aee89159d40e4f7c788cf670d9fa2e405d32c75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb3eb2134bf8e3e8b8e81f29432db3e81b5f2fcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fc261e8fe114ffc9d7be99110d659704018f159 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..baabc83a9e25b700600fe516d9a84833c32f4f29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41365c509da451280527720e651d5793d1b83960 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_government_and_politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05e62fa85cb3fdf871ec246de43d32c7a5209db1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9a9ca3b3840ee7169b59a53cec4c595c783cd4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fb8639003555bdca712f3dc49ed6e463158be42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c149ef083a87f6d3eb412f9e3fb2fbd131ec4c0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..999f9be74e2bc278a068c344030ae27f3b2c3006 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f905569c82f31ec76a75505bfae64c28d72640 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d09cdcaa3b268d599e055f82c92779d4ecd2bcb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28a63b1b9106219486b5487b24396baf44179276 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_world_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a71bfc38aab72f17a01e3da11fc037ce28ef033 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_human_aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa9b895b7331b051385a31165c725c2ef976db69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_human_sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33766a464fa475a012d229c194c93fffb84942b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_international_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..642e6ce4f34992cb5be8b840ea481c7a389d9ce8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml @@ -0,0 +1,6 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12594895469fbf0644e1908e4299f93f417703e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_logical_fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c27feea94ce017e35bcd453d6cbf5c4db5b3334 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml @@ -0,0 +1,6 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_machine_learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1a13763a2bd796821efa251071359ce0acbf1cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fe6e44b7fe464396e85a53f70831bbb48ff8ece --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml @@ -0,0 +1,6 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..813b6a3fe90413bd35a11f82624df600d8bf682b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2a95e892a8e6d357e6a9f771272d06422b14d1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml @@ -0,0 +1,6 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6a76a2a7930589f3603fa070e974116b4996e96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5f8c4e6f144dcb4c0eb6881b095434c76105bb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_moral_scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0f144cb44e5218d3a70193fddca2a2883e6b1b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml @@ -0,0 +1,6 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4e4c0c4b6ccd34ebf4ff1133d0e26ddd8dc90d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9db801a6a9f2d911e2bdbbe0084fd235c7572776 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e94bef0581e5290ff4790b5d48863a198a904879 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_professional_accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25239d9a35941d49797c15986cc43213b0ec74d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_professional_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f961bff89745dd8999c2ee497bdf9a7df88e04f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_professional_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48758ef76eaf72e4236a8569e041ea03e6626e67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62a56a4478bf9eafbcf1a8034abfeea6240e99ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..062f49630e82b66be1ea0e75ed9fe73c8d635215 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36b4711831ef6fafde0915178e28513692f9c8d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4afb8f84a193442cd98a856ada7e43f1515cbce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8e427612f45461a5d873edbafb3d6e0eba4e9f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb04f31f0baaf6ac0f358de2897d5267e1a4357 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..72246935de8cf0cf8b256fd1e6c87dfbbb90a2ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/utils.py @@ -0,0 +1,112 @@ +import re +import sys +import unicodedata + +from lm_eval.filters.extraction import RegexFilter + + +class MultiChoiceRegexFilter(RegexFilter): + """ """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select=0, + fallback: str = "[invalid]", + ignore_case=False, + ignore_punctuation=False, + regexes_to_ignore=None, + ) -> None: + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(regex_pattern, group_select, fallback) + self.ignore_case = ignore_case + self.ignore_punctuation = ignore_punctuation + self.regexes_to_ignore = regexes_to_ignore + + def apply(self, resps, docs): + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + def find_match(regex, resp, convert_dict={}): + match = regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m][0] + match = match.strip() + if match and match in convert_dict: + match = convert_dict[match] + return match + + punct_tbl = dict.fromkeys( + i + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") + ) + + def filter_ignores(st): + if self.regexes_to_ignore is not None: + for s in self.regexes_to_ignore: + st = re.sub(s, "", st) + + if self.ignore_case: + st = st.lower() + + if self.ignore_punctuation: + # https://stackoverflow.com/a/266162 + st = st.translate(punct_tbl) + return st + + filtered_resps = [] + + for r, doc in zip(resps, docs): + fallback_regexes = [] + choice_to_alpha = {} + next_alpha = "A" + + without_paren_fallback_regexes = [] + without_paren_to_target = {} + + choices = doc["choices"] + for c in choices: + m = filter_ignores(c.strip()) + fallback_regexes.append(f"{re.escape(m)}") + choice_to_alpha[m] = f"({next_alpha})" + + without_paren_fallback_regexes.append(next_alpha) + without_paren_to_target[next_alpha] = f"({next_alpha})" + + next_alpha = chr(ord(next_alpha) + 1) + fallback_regex = re.compile("|".join(fallback_regexes)) + without_paren_fallback_regex = "|".join(without_paren_fallback_regexes) + without_paren_fallback_regex = re.compile( + f":[\s]*({without_paren_fallback_regex})" + ) + + filtered = [] + for resp in r: + match = find_match(self.regex, resp) + if not match: + match = find_match( + fallback_regex, filter_ignores(resp), choice_to_alpha + ) + if not match: + match = find_match( + without_paren_fallback_regex, resp, without_paren_to_target + ) + if not match: + match = self.fallback + filtered.append(match) + filtered_resps.append(filtered) + + return filtered_resps diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14465ad6e5c5434974832399ea95903b59e4eaf5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_n_shot_generative +group_alias: mmlu (flan style, generative) +task: + - group: stem + task: + - mmlu_flan_n_shot_generative_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_generative_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_generative_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_generative_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a38a06969e2649d2fc0cf8e2be3efc60d91b3076 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml @@ -0,0 +1,34 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA:" +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer]}}" +filter_list: + - name: "strict-match" + filter: + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: 0 + regex_pattern: "(\\([A-Z]\\))" + ignore_case: true + ignore_punctuation: true + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + - "\n" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true +metadata: + version: 3.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e602ee8100ed612d89385532ea30004c3033c35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa12cc8ef35b19f3b81dcc58a0107d424a3580cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4178654e0e6e7a053839319c7936967133cf756 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4447d276b066ddec93b8f7efcf2d74d13810f458 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38f799060fa6901b890d3a87d8aa9b9444d34b57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_clinical_knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f36eb1f598f754154c2b15b24bbb650358c707c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eccce652fade13a319af78e06a7528b11814302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd415aa10efaf96331d9fef82c5b6a2bb538263a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d062721102c0f6e6c09574398a60db74c26b593 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edc660d9c30dfad6666f5e1b4c679489f62c5991 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aac8f400d1d9005376bfe3354753e87700a7bda8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..178c468346a5022a5d0031fd27c6b9a07ab24150 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3cfbe6250d19aab6e60c9089f0feb91eed37423 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_conceptual_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad8704e4f8e3a60ee2ff7e370cf7394c0359aeb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56eeae0183ca0c087b0a16aa317f2b93d5f1b87b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da3b3af2b5f310232cbd9c9ee63081acbb571638 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d3f4edc644842cbc3fae865c96f99322daaafbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4feef1895254438bde19ebfc3d7a36aee87e61de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_global_facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..245d9be815c3644bf3298a0d093a76410b7487b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34eb30d32d5b6927d44d59a63f5a549587f414f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34250a6c61cb5e29acbb99f8a080d45f74a91d45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42b7dd4d5aa2ab541b7f269c84845d262db452c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e67277aa5480e1a9465169112755c3da70e12e6e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84643a74239db620816f0d8a67575d0c8268e58f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_government_and_politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb08333804237ac3e0584db637d5c91477a6a93d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1ca028d8262f22807eb591c3e498fecabd9887b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c60982b78dab4866a6827fe5b1bf9f2b710ed8d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33b8d16739c9faf352ad242bd76b2bc33bc21aa6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f47bbbb68c02a417e60e5b0a19f4f85c5723b41b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..741971895ba27ad6651ac456def204a078ac5d3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48696971c9e850a18baadd6c3e9f958851cc2a3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae6cfcbba3f86dc0339edc3a361c898e6c8716fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_world_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..677f119a754f0c671fae0f2285bb8ff29f2af85e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_human_aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4e33d7d607ef2f07ea0fdb67305b8f88a45d13a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_human_sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac5d9d5a46b7f4f1daafb7c7f0feb66933c4829d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_international_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2f135869aca516492cd9dc8ce210838173a1d7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml @@ -0,0 +1,6 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6624e07743a432cc354ccff7af2363db2ec1ae11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_logical_fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab6c459ae50e7311dc9d8819ec753c69f6d9583b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml @@ -0,0 +1,6 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_machine_learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4af9ded012e921feeb38d31cde98fef9888aba95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22ef9d3fd49556afd4578685099abc0bb9b64c9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml @@ -0,0 +1,6 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c24da7938b431acdd991830424777e6645cf9bbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5b90845321c954cc2e7875fdc084e5935444af7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml @@ -0,0 +1,6 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..295c39a6efce509983b01b18c20375866b08d3bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f09f982f26462304a20420e9b61bf3ef941448a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_moral_scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf633f270a6d9fbbaa0a793bc5d5e48731a31d57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml @@ -0,0 +1,6 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a5fe27eefb47badf4c13e87ad0fbac96b08283e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60788fc6c201bf316398f48adc9575dcb806b649 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f312af231f28d9343f7a0e2353cec110fda1f9a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be0533f0d8b90fc9f82226579ec849ac3f24be15 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cae6f8a5ec27d73bcf9b57e8597b377aee62835 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21a39c51b7d246c3dd49e47ee0f5dd1865059c36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2687d99a279caac3f322ff178a1ea1ac7ea44f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c36a5522d3c0d6f165dbd5eaac9f5208822fb9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ce0809907575855a8680ec1db533688ad42de46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ed5e16281b6aca3720868538c93d2877d438b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..257dcfbf8a18c96d836d6db1214e8ff69ec63278 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39b64d03d3983f5c692a1a762c8457175dbf5408 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..72246935de8cf0cf8b256fd1e6c87dfbbb90a2ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py @@ -0,0 +1,112 @@ +import re +import sys +import unicodedata + +from lm_eval.filters.extraction import RegexFilter + + +class MultiChoiceRegexFilter(RegexFilter): + """ """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select=0, + fallback: str = "[invalid]", + ignore_case=False, + ignore_punctuation=False, + regexes_to_ignore=None, + ) -> None: + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(regex_pattern, group_select, fallback) + self.ignore_case = ignore_case + self.ignore_punctuation = ignore_punctuation + self.regexes_to_ignore = regexes_to_ignore + + def apply(self, resps, docs): + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + def find_match(regex, resp, convert_dict={}): + match = regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m][0] + match = match.strip() + if match and match in convert_dict: + match = convert_dict[match] + return match + + punct_tbl = dict.fromkeys( + i + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") + ) + + def filter_ignores(st): + if self.regexes_to_ignore is not None: + for s in self.regexes_to_ignore: + st = re.sub(s, "", st) + + if self.ignore_case: + st = st.lower() + + if self.ignore_punctuation: + # https://stackoverflow.com/a/266162 + st = st.translate(punct_tbl) + return st + + filtered_resps = [] + + for r, doc in zip(resps, docs): + fallback_regexes = [] + choice_to_alpha = {} + next_alpha = "A" + + without_paren_fallback_regexes = [] + without_paren_to_target = {} + + choices = doc["choices"] + for c in choices: + m = filter_ignores(c.strip()) + fallback_regexes.append(f"{re.escape(m)}") + choice_to_alpha[m] = f"({next_alpha})" + + without_paren_fallback_regexes.append(next_alpha) + without_paren_to_target[next_alpha] = f"({next_alpha})" + + next_alpha = chr(ord(next_alpha) + 1) + fallback_regex = re.compile("|".join(fallback_regexes)) + without_paren_fallback_regex = "|".join(without_paren_fallback_regexes) + without_paren_fallback_regex = re.compile( + f":[\s]*({without_paren_fallback_regex})" + ) + + filtered = [] + for resp in r: + match = find_match(self.regex, resp) + if not match: + match = find_match( + fallback_regex, filter_ignores(resp), choice_to_alpha + ) + if not match: + match = find_match( + without_paren_fallback_regex, resp, without_paren_to_target + ) + if not match: + match = self.fallback + filtered.append(match) + filtered_resps.append(filtered) + + return filtered_resps diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cfa0fb9c30451fa79f6b8b038a01692c830f1a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_n_shot_loglikelihood +group_alias: mmlu (flan style, loglikelihood) +task: + - group: stem + task: + - mmlu_flan_n_shot_loglikelihood_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_loglikelihood_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_loglikelihood_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_loglikelihood_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4605a4a15f2e84c4572388192fc1e51d717f70b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml @@ -0,0 +1,17 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA:" +doc_to_choice: ["(A)", "(B)", "(C)", "(D)"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5dfa65ded384d6e1299b8e5564f5a655f2ced79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e837e5d8fd3e1577af4d23d2120d1b55029f052f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b9bc7ed89429c2d08cc74cc4472ebea28f67a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2438e6678be07c008922d83ea5016efab56ebc78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d66adda5d600a94d5f6e36544dd63d2de3fece --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_clinical_knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15e6e75d3491dfd034df789a3481fb3a39dcaa02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b8c1bd3a8de310698082f738d287743d3731c23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1178c7b072f82bebdd4281a371d6105514a686e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9776889b514c04c6c93aeedfd0ced7c620d11493 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8fdad90bd103ff616b4b14c2a3e9024208e149a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77a89689127b4ca129b9434653198b051324fc0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e787e51745218e2465b739ee82b51c456bd228ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_computer_security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..859e88e48a5cea7114b85c31c594f832520bacb0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_conceptual_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0455a515eab5e3102a659d917758b942c00b952d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b63e06172ec302a916f3be4b0a2ea0f1efa86674 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_electrical_engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79771d21543868dd73bf6ff84201ef07d79c89a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_elementary_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e46d8e21c62ff03a6f47bbbc7a6d085840049a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_formal_logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7aff59325d7dab9a02c4eda3a886d062fe3b4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_global_facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfe33de2be1d2f821c92fc46111150e1ac366b7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..661ea0ca2f72242eb4daf520f6683a9de3a7c32c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b271a661f943fdd6d364833c9f994c19ee10cd22 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1a329ebb24804c92690b5210cb27f6ec47be93d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_european_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe681101f6704e7f058e27350b37838ba63fcd07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8a8f279fc5bfaa8b610f3ff5dcd1c2be0c88e07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_government_and_politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45664135facb151e9b6f91347bbc135297880acb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49903260ceff13c03070606e04beb45d99d660f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..394c1d77e553a24820ba5db934bfa8fd95a8a269 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f32ef2fcc4bf03e34b43c5a3d1135431742db71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a9aac0736a9610469c70b925b70b3f384ca9777 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7e02afb94aebb1676c5c395c51e37d4f149a39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bc84ea9dd78e87166f1e7b67c248d242cb98d83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_us_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f25cf646bebbca23ed23ea421473e6c2461dda8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_world_history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c258f919041775e1d2bf1226264a10b1133802db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_human_aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e192a78b48bd37e4dc37efc5783b527f84c3e55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_human_sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..662bf6eb35157889356a6be7ded31d5f6f2a39ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_international_law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..346e4b669771f23d7a3a805b329e96e711cd367e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_logical_fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a732a778fb85eac5467fe2744e51340bce0c302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56760226dba043ba37a110cf7065bbd52c3e9c93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml @@ -0,0 +1,6 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6635c9613155a7c23bf67329b4be950e57fe2d30 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce8dff42a80057d6557f81e5aead49b4e93e4ef3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml @@ -0,0 +1,6 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62460e82f3386022679443efe3c989c2ffb59abf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_moral_disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5494f9dc462494e198dfc7ad86d63a186637bf5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml @@ -0,0 +1,6 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1eb08bfbeb44dc8279ab6796e673b3b271517548 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7043a07273fe781cb56733ab02b0b1cb4bf059 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb1f585ce890b5a4fcccc72c3066691509330c49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ff2d9ea791abf7bad56784b804bcadd5c82c077 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..765e70c8fc22dbd75ba495a6490ec788d4e44b7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/_default_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7281f0a1e06ad370e2bf4933816b2724f2b55541 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/_default_template_yaml @@ -0,0 +1,34 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}" +generation_kwargs: + until: + - "" + - "\n" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_punctuation: true + ignore_case: true +filter_list: + - name: get_response + filter: + # Filter everything after the first break line + - function: "regex" + regex_pattern: "^(.*?)(?=\\n|$)" + # Remove leading white spaces + - function: remove_whitespace + # function to ignore right white spaces or line breaks + - function: "regex" + regex_pattern: "^(.*?)\\s*$" + - function: take_first +metadata: + version: 3.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..550caa37606f975110f5e4f425d27e594014c116 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/_mmlu.yaml @@ -0,0 +1,38 @@ +group: mmlu_generative +group_alias: mmlu (generative) +task: + - group: stem + task: + - mmlu_stem_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response + - group: other + task: + - mmlu_other_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response + - group: social sciences + task: + - mmlu_social_sciences_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response + - group: humanities + task: + - mmlu_humanities_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: get_response +metadata: + version: 3 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17bfcafb79b113cffe93f6e90c68562b7eae7c95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_abstract_algebra_generative" +"task_alias": "abstract_algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72afc359a495af12d3dcb2b062c6442d92d45c88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_anatomy_generative" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c15d443691af36dcdc761eb41b8673f3782d0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_business_ethics_generative" +"task_alias": "business_ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24cd0b72d3f68fb00da90397979816b85ea1c76c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_clinical_knowledge_generative" +"task_alias": "clinical_knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ff9cc284007337e30369dd4864b2b723e8e6768 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_biology_generative" +"task_alias": "college_biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73d91c52acd76bf99ce1869296257d25143ad149 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_computer_science_generative" +"task_alias": "college_computer_science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d997d8974c99a549a2216a9bd9237f05a619e21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_physics_generative" +"task_alias": "college_physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2df93cab2a999a7d6d8e78d3ac9c3ce9aeddcf12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_chemistry_generative" +"task_alias": "high_school_chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5782d81551072a0ff03d79c930f02edb64488f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_jurisprudence_generative" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fea57959818525acdada5bf8a327b0ce96fefb0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_medical_genetics_generative" +"task_alias": "medical_genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e0fabc2536d4894526b680deba9a382ff9c3ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_miscellaneous_generative" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..638ac8100b6f918ccaa0a3dc13946512d3c97b33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_nutrition_generative" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e130e1baacc3f8a8f558b568336896668e84dd4f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_prehistory_generative" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a46792ec22d84ee3193996653f536084b9ab7861 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_accounting_generative" +"task_alias": "professional_accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b36ccde61e7edc33464a676d4fe0fcc25f3304 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology_generative" +"task_alias": "professional_psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37cdccba9b7cebbaa34c5f1e9da01655367477f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_public_relations_generative" +"task_alias": "public_relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36c235feefd1548320400e7e8d9f3e03f2d478d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_security_studies_generative" +"task_alias": "security_studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7e2e592e4457118c9458ccb757b823f9adbb193 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_sociology_generative" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fb95366245eae638918270bff4353024195d5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy_generative" +"task_alias": "us_foreign_policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9954dc182f1bbd5030b94d2a08b2ddf4a135a6cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_virology_generative" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1db5128b43e615d0fc41f9c7448db3b5ea39942c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_world_religions_generative" +"task_alias": "world_religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_default_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d59d03a3bb09437644922c8345452539493a0ae9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_default_template_yaml @@ -0,0 +1,32 @@ +dataset_path: TIGER-Lab/MMLU-Pro +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + # regex_pattern: r".*[aA]nswer:\s*([A-J])", + - function: "take_first" +generation_kwargs: + until: + - "Question:" + max_gen_toks: 2048 + do_sample: false + temperature: 0.0 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.1 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc3204127604d6eac759299f77d63ce9ef49d24e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml @@ -0,0 +1,23 @@ +group: mmlu_pro +task: + - mmlu_pro_biology + - mmlu_pro_business + - mmlu_pro_chemistry + - mmlu_pro_computer_science + - mmlu_pro_economics + - mmlu_pro_engineering + - mmlu_pro_health + - mmlu_pro_history + - mmlu_pro_law + - mmlu_pro_math + - mmlu_pro_other + - mmlu_pro_philosophy + - mmlu_pro_physics + - mmlu_pro_psychology +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daf871f6bb5abd614c2058c0552389d41cbced50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_business" +task_alias: "business" +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5baf354ec202e66647bacdc9e9008617cd2d4244 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_chemistry" +task_alias: "chemistry" +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7de347373e7b92d8eb9ba33bbf8ec4fb2a3dbc49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_computer_science" +task_alias: "computer_science" +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274612783fb749aad10ec67698c3abf5dce13ebf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_economics" +task_alias: "economics" +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcf02f5029823ab83c1cebfbbf9819ac5ec4f1d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_engineering" +task_alias: "engineering" +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d161d1d81a29a06a9425bafc9e86211a2669b9be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_health" +task_alias: "health" +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d28efd3e77ef31dc2df96847c90e7c76920007bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_history" +task_alias: "history" +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba99f16dbb8df07ab9272d8b47d23839a919bc9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_law" +task_alias: "law" +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb2ec9da7011dd1437828fc442ad21733cdf1a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99e5d65b4c0af9a2a35589ab104cb07be100cabc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e7fa740bd58ebd55382bd8464abd9abaee3d96e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b28fb72d329d4d9755e45201275604d257851754 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_psychology" +task_alias: "psychology" +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ca254a29a271c2fdf27781b897d16af9a61afc66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py @@ -0,0 +1,51 @@ +from functools import partial + + +choices = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] + + +def format_cot_example(example, including_answer=True): + prompt = "Question:\n" + question = example["question"] + options = example["options"] + prompt += question + "\n" + prompt += "Options:\n" + + for i, opt in enumerate(options): + if i >= len(choices): + break + prompt += "{}. {}\n".format(choices[i], opt) + + if including_answer: + cot_content = example["cot_content"].replace( + "A: Let's think step by step.", "Answer: Let's think step by step." + ) + prompt += cot_content + "\n\n" + else: + prompt += "Answer: Let's think step by step." + + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f3db0d165db36a0842069e7be6dc021bdf9b6568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/README.md @@ -0,0 +1,75 @@ +# MMLU-ProX + +### Paper + +Title: `MMLU-ProX: A Multilingual Benchmark for Advanced Large Language Model Evaluation` + +Abstract: `Traditional benchmarks like MMLU and MMLU-Pro focus primarily on single-language evaluation, limiting their ability to assess language models in multilingual and culturally diverse contexts. To address this gap, we introduce MMLU-ProX, a comprehensive multilingual benchmark that builds upon MMLU-Pro by covering multiple typologically diverse languages with approximately 11,829 questions per language.` + +Homepage: https://mmluprox.github.io/ + +### Citation + +```bibtex +@misc{mmluprox, + title={MMLU-ProX: A Multilingual Benchmark for Advanced Large Language Model Evaluation}, + author={Weihao Xuan and Rui Yang and Heli Qi and Qingcheng Zeng and Yunze Xiao and Yun Xing and Junjue Wang and Huitao Li and Xin Li and Kunyu Yu and Nan Liu and Qingyu Chen and Douglas Teodoro and Edison Marrese-Taylor and Shijian Lu and Yusuke Iwasawa and Yutaka Matsuo and Irene Li}, + year={2025}, + eprint={2503.10497}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2503.10497}, +} +``` + +### Groups and Tasks + +#### Groups + +* `mmlu_pro_{lang}`: 'All 14 subjects of the mmlu_pro_prox dataset in {lang}, evaluated following the methodology in mmlu_pro's original implementation' + +Available lang: +- ar +- bn +- de +- en +- es +- fr +- hi +- ja +- ko +- pt +- sw +- th +- zh + +#### Tasks + +The following tasks evaluate subjects in the mmlu_prox dataset +- `mmlu_prox_{lang}_biology` +- `mmlu_prox_{lang}_business` +- `mmlu_prox_{lang}_chemistry` +- `mmlu_prox_{lang}_computer_science` +- `mmlu_prox_{lang}_economics` +- `mmlu_prox_{lang}_engineering` +- `mmlu_prox_{lang}_health` +- `mmlu_prox_{lang}_history` +- `mmlu_prox_{lang}_law` +- `mmlu_prox_{lang}_math` +- `mmlu_prox_{lang}_other` +- `mmlu_prox_{lang}_philosophy` +- `mmlu_prox_{lang}_physics` +- `mmlu_prox_{lang}_psychology` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_ar_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_ar_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c44c140dd4e07c3fd28d0ee7129f0b1f6edccf0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_ar_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ar +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'الإجابة هي \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "سؤال:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_mmlu_prox_ar.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_mmlu_prox_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22de9a1444cc4a4677e42924ea91134277feee45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/_mmlu_prox_ar.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ar +task: +- mmlu_prox_ar_biology +- mmlu_prox_ar_business +- mmlu_prox_ar_chemistry +- mmlu_prox_ar_computer_science +- mmlu_prox_ar_economics +- mmlu_prox_ar_engineering +- mmlu_prox_ar_health +- mmlu_prox_ar_history +- mmlu_prox_ar_law +- mmlu_prox_ar_math +- mmlu_prox_ar_other +- mmlu_prox_ar_philosophy +- mmlu_prox_ar_physics +- mmlu_prox_ar_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21dc18d094366c7fa2eb100a13a4d1077d4f2064 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_biology.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول علم الأحياء. فكر خطوة + بخطوة ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d995fe2b4fd8a108166e9dcf366853f87d40d51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_business.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الأعمال. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90d9786d3c3ffed4ffae79765f9fa15758b3a0b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_chemistry.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الكيمياء. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebbe0aba56381adf3dc6b83cee43f46c2306026f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_computer_science.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول علوم الكمبيوتر. فكر خطوة + بخطوة ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43dd2194060013f13a6c8ec23d3e185b3ba0a25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_economics.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الاقتصاد. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..976052d42554e65f4b585dd06c993fa310fa05e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_engineering.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الهندسة. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc112389491ad0f81d97a833215e8690c252a2d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_health.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الصحة. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e954acd3b2371aa2260b13d3b383481cc141dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_history.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول التاريخ. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb9741e8d8273d365f8235afc5482a863682ae93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_law.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول القانون. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..791adf799edc13936383208aa667b5c1a26799a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_math.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الرياضيات. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4722e54f6ded128c9016e7c8a2cad6f420929a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_other.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول أخرى. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..477710558ae2d233c752fd415147a16a95fdd96a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_philosophy.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الفلسفة. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfb623df5ce8cbe53d790e01bdab27b5bfade7a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_physics.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول الفيزياء. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86b6517de09743800e6a24eba5d9f65dfb87b8c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/mmlu_prox_ar_psychology.yaml @@ -0,0 +1,8 @@ +description: 'فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول علم النفس. فكر خطوة بخطوة + ثم أنهِ إجابتك بـ ''الإجابة هي (X)'' حيث X هو حرف الخيار الصحيح. + + ' +include: _ar_template_yaml +task: mmlu_prox_ar_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ar/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_bn_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_bn_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e5e67509d725a44bbc9e5b868f344f5c9e05761 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_bn_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: bn +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'উত্তর হল \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "প্রশ্ন:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_mmlu_prox_bn.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_mmlu_prox_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..393097d3896037937cfeb2d09610371bf723e13b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/_mmlu_prox_bn.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_bn +task: +- mmlu_prox_bn_biology +- mmlu_prox_bn_business +- mmlu_prox_bn_chemistry +- mmlu_prox_bn_computer_science +- mmlu_prox_bn_economics +- mmlu_prox_bn_engineering +- mmlu_prox_bn_health +- mmlu_prox_bn_history +- mmlu_prox_bn_law +- mmlu_prox_bn_math +- mmlu_prox_bn_other +- mmlu_prox_bn_philosophy +- mmlu_prox_bn_physics +- mmlu_prox_bn_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f0148e006763ab79f87b04042533076b1f7a568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_biology.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত জীববিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61f2313c195858c2fc79cb54ca868dafde80302d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_business.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত ব্যবসা সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63b7909ddaa00f5ced2b2ecfd51ccde9c85b276f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত রসায়ন সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25f2c8cb820d59f219ff504bfa108cd0aaf9f09f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত কম্পিউটার বিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। + ধাপে ধাপে চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে + X হল সঠিক বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a31a9a4f1820ed0723da7b7d3eee844e3bf528c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_economics.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত অর্থনীতি সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ca7ef16d9fbbac0a5f5b60d0801523d7fbd354b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_engineering.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত প্রকৌশল সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b03efc55dad017262b46f1127a8532f066da187 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_health.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত স্বাস্থ্য সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..873383c9ec8b6349e7e1dc9584968bc005dc496a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_history.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত ইতিহাস সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf8bdc6f7e96384fd84f9b9f2bc193d7420b60b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_law.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত আইন সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা + করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের + অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bdb2f23e47e34c910c14967ad11a0608504da95 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_math.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত গণিত সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা + করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের + অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14efa2768459cae192bdbcffba8922a884da4215 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_other.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত অন্যান্য সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92196436a3fce2db50992a778bd4063f1c27a618 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত দর্শন সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা + করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের + অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d8291d3131196714d1dfd6d709767faf0c5fe10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_physics.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত পদার্থবিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে + ধাপে চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল + সঠিক বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..498389a2893fcbe1ed45549d0d559ac3e64e9095 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/mmlu_prox_bn_psychology.yaml @@ -0,0 +1,9 @@ +description: 'নিম্নলিখিত মনোবিজ্ঞান সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে + চিন্তা করুন এবং তারপর আপনার উত্তর "উত্তর হল (X)" দিয়ে শেষ করুন যেখানে X হল সঠিক + বিকল্পের অক্ষর। + + ' +include: _bn_template_yaml +task: mmlu_prox_bn_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/bn/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3ef81b4b9ef43990b91018eebd0ab7dc161f964 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: de +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Die Antwort ist \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Frage:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e717130357c24d9859f49c3fe749453a2f382a00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_de.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_de +task: +- mmlu_prox_de_biology +- mmlu_prox_de_business +- mmlu_prox_de_chemistry +- mmlu_prox_de_computer_science +- mmlu_prox_de_economics +- mmlu_prox_de_engineering +- mmlu_prox_de_health +- mmlu_prox_de_history +- mmlu_prox_de_law +- mmlu_prox_de_math +- mmlu_prox_de_other +- mmlu_prox_de_philosophy +- mmlu_prox_de_physics +- mmlu_prox_de_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87835fd5b6df77c044c4b83c82905553bdefd30b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_biology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Biologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8683d9ee4c8f0578f991e463ccc20c673179a3d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Wirtschaft. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e43365713a43209efbb5b2c0f9db5b70fb028075 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Chemie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c663f692eea1a17df6fe45b37be1706dcaa3df57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Informatik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a01a7488c60907557a019d5c5f5d29d23fc33fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ökonomie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d9a7359c0e173156e5a2dc513fa4bd11c184916 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ingenieurwesen. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..303272745cdbdbe32c83a2d3ceba85b02e59974a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Gesundheit. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b08416c7de46b11f53623607961bf1353a0ae1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Geschichte. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1e58ff8658c70c85b6fd1c56f143b462dc1fed1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Recht. Denken + Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort ist (X)", + wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dee8f6e4e732564714e7c9dbed29e8301ac9b66e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Mathematik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2dd274cc2b94b24df402369300bd1de9e79b5f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Sonstiges. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757be6dfd8a34ba60d64776d9dec9084325a8414 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Philosophie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd3a6e6d4dd93f520679dafb7db26cbd87545c93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Physik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fbcd9b49b48ceecb01c3422f1c2f27d96739e2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Psychologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed712f867d03aa8da77c33c89080dfeec7e7cea7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: en +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..649a5a625b9dcf7745abe1c16a888bc9ff6e40d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_en +task: +- mmlu_prox_en_biology +- mmlu_prox_en_business +- mmlu_prox_en_chemistry +- mmlu_prox_en_computer_science +- mmlu_prox_en_economics +- mmlu_prox_en_engineering +- mmlu_prox_en_health +- mmlu_prox_en_history +- mmlu_prox_en_law +- mmlu_prox_en_math +- mmlu_prox_en_other +- mmlu_prox_en_philosophy +- mmlu_prox_en_physics +- mmlu_prox_en_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c33358b48caee9dcf040000609675e5f3135038b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about biology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc85a949be4ea9f026c580eb88653e397022c1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about business. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f9c45a290f1c9cf0489f4f7ffc55aa433f105e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about chemistry. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b2e55f3abe22beaf4ed5efd75c3e43844eb3f6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about computer_science. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37232d2f4620fe297a04329753374843f3113f5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about economics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ca9302d32a9dedfb61bb798b8bc8a49308cdf1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about engineering. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a384c2e5a59f0533b2b86621cdd34488ddde807 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about health. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6bb11902f17570ddeaa9695a15457cbcaccd0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about history. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..534da99020b95befee63db1acc2c58b3e72d5008 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about law. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59fca5bac4841ff195ed230ce62a67a865783ba3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about math. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..437e2449e203eb047c45c971f058b9ac49ff08ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about other. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d67bc4dd543fbd1060bccdbd9f9d705e344404d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about philosophy. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7847c0f5f37b231a2113a454873a8ab5c84339a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about physics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e3b84d8c5c261366e59c7265169b3e6b7ce51e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about psychology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a84f6cf5d1416d54d5ddb297467ddf4e6bc4653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: es +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La respuesta es \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pregunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..437942cb0b0527c84d6ec084d7bf5534a496bf51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_es +task: +- mmlu_prox_es_biology +- mmlu_prox_es_business +- mmlu_prox_es_chemistry +- mmlu_prox_es_computer_science +- mmlu_prox_es_economics +- mmlu_prox_es_engineering +- mmlu_prox_es_health +- mmlu_prox_es_history +- mmlu_prox_es_law +- mmlu_prox_es_math +- mmlu_prox_es_other +- mmlu_prox_es_philosophy +- mmlu_prox_es_physics +- mmlu_prox_es_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aa01f9bacf34d21ce732e3d583c19fb1ce59dcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + biología. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c7551b67d595300f6218fe3c0b2817fbbea4f74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + negocios. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d75f7034b7ef6e047b15e69a7146a72661377e41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + química. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33d2d919542192e6520cb54b90037cf9af6bfedd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + informática. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d257611b14eeb4d8eef729bd02ea3e95eeca243c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + economía. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..504de4348c4b0e9435f1d6a01c3d2b91017b4a42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + ingeniería. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f621a739e28c438bcedbfcddc1faf75dc368ab83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + salud. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ecb14e95bcfb1035e3a4c71e890fd71e19b564d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_history.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + historia. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e8e89acde0b0eacfd713e06fb022e363ed7f248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + derecho. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c584a90864df9246a0db4a52937973e21f81cee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + matemáticas. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f58119ad994400ed61a9148a6b01f2047c9fecb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + otro. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afb955393bba6b3a57089bdb7a93fbfd0db73dc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + filosofía. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db2a905cb103bd5292bb75bf6a295da5a84d836f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + física. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..217c215e07d58ebfbdd889620968dba04b68d49c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + psicología. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..935f561faec99a0bde2e3721a6e09737825d2627 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: fr +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La réponse est \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question :" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_mmlu_prox_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_mmlu_prox_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..661deda053eb630552f71e9c346bea75993d02c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_mmlu_prox_fr.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_fr +task: +- mmlu_prox_fr_biology +- mmlu_prox_fr_business +- mmlu_prox_fr_chemistry +- mmlu_prox_fr_computer_science +- mmlu_prox_fr_economics +- mmlu_prox_fr_engineering +- mmlu_prox_fr_health +- mmlu_prox_fr_history +- mmlu_prox_fr_law +- mmlu_prox_fr_math +- mmlu_prox_fr_other +- mmlu_prox_fr_philosophy +- mmlu_prox_fr_physics +- mmlu_prox_fr_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c3b74cbb5698f47db3acf927aaba48cd303e5c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_biology.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur biologie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cbe1472f507fa9d5d9b33e47113c3dbbe401f96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_business.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur commerce. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4fd7560b855ee8eaab595303e1d8c6b37aad057 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur chimie. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30f825c182f5b5e386871416de86f4a4fe234fea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur informatique. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..885ba15d6d6577433d8743a847c7dd7af6f9ad76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_economics.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur économie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f096ea88f24e739b5a5405ce2baeb34de20627 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur ingénierie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e642b40b4600882c639ccfcfef522e515ea199d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_health.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur santé. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ad409b66248fee8abf5ce7ac305a754f616f5af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_history.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur histoire. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33bbd86404717f7061cfeec43a05421e3cc3d4d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_law.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur droit. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0a9bd489d093ebf7f6183f8cc76f20e3b0c7755 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_math.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur mathématiques. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9763476f98f933d9afba845b3b62010c14dc7814 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_other.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur autre. Réfléchissez + étape par étape, puis terminez votre réponse par "La réponse est (X)" où X est la + lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32f602be482e6ca9b0774cd80b048ffe6de0c6db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur philosophie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5adf9a01605d45a750fc1fad95915b10a791be5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_physics.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur physique. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3004495da851bcc582d8e4155c079ea6f542e0c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur psychologie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_hi_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_hi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..483a487bc14a247caf1c1c9fcae3df4c6e043581 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_hi_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: hi +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'उत्तर है \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "प्रश्न:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_mmlu_prox_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_mmlu_prox_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ae06179de5891ac91862f806d7635f04b7b6176 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/_mmlu_prox_hi.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_hi +task: +- mmlu_prox_hi_biology +- mmlu_prox_hi_business +- mmlu_prox_hi_chemistry +- mmlu_prox_hi_computer_science +- mmlu_prox_hi_economics +- mmlu_prox_hi_engineering +- mmlu_prox_hi_health +- mmlu_prox_hi_history +- mmlu_prox_hi_law +- mmlu_prox_hi_math +- mmlu_prox_hi_other +- mmlu_prox_hi_philosophy +- mmlu_prox_hi_physics +- mmlu_prox_hi_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ab83377ce03ff55f59c867ae8be11086ce33f11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_biology.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित जीव विज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e3a7bb97f66881058247ceed280cd948575da00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_business.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित व्यापार के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e3670e28797d5e16a009462ba2bb0880f0cd68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित रसायन विज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों के + साथ) हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें + जहां X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..254f7997b029cef1c35ee83f901a53522390141b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित कंप्यूटर विज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों + के साथ) हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त + करें जहां X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db6d9f4915f23b970ab23d7cc0488faac0a019b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_economics.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित अर्थशास्त्र के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc23e4659d7819eb95606091bd1944147582732b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_engineering.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित इंजीनियरिंग के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25af832a2a8441a8fe33d7e3ba8237d18634d2a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_health.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित स्वास्थ्य के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22c743f628a17d3bb71afc699c7c5a5a9b963255 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_history.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित इतिहास के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9d4905a3a02ee37d1e2be7292fca0d77aeac42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_law.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित कानून के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c0aea90b194a90fe573bebcfa14a4ab73878d13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_math.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित गणित के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..814017afebbe530340cee9c3b19358ec7ed2b3dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_other.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित अन्य के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4886009317866429e37fa3d7f8759e9a6be8eae3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित दर्शनशास्त्र के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68cdd4915e3e675fd97d7dd7730b2399268ad33a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_physics.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित भौतिकी के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। + चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां X सही + विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de1224de07165db1cd278b2b1dd41da38593c2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/mmlu_prox_hi_psychology.yaml @@ -0,0 +1,9 @@ +description: 'निम्नलिखित मनोविज्ञान के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) + हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "उत्तर है (X)" के साथ समाप्त करें जहां + X सही विकल्प का अक्षर है। + + ' +include: _hi_template_yaml +task: mmlu_prox_hi_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hi/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb5569f7cf32f4d30407cdd74a2e205ad404805 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ja +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '答えは \(?([ABCDEFGHIJ])\)? です' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "質問：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5887f7b82fe5696476dd37da284c919bbc5964f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ja +task: +- mmlu_prox_ja_biology +- mmlu_prox_ja_business +- mmlu_prox_ja_chemistry +- mmlu_prox_ja_computer_science +- mmlu_prox_ja_economics +- mmlu_prox_ja_engineering +- mmlu_prox_ja_health +- mmlu_prox_ja_history +- mmlu_prox_ja_law +- mmlu_prox_ja_math +- mmlu_prox_ja_other +- mmlu_prox_ja_philosophy +- mmlu_prox_ja_physics +- mmlu_prox_ja_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd68bb572980342e25cc225cec2dcacf977c08a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml @@ -0,0 +1,7 @@ +description: '以下は生物学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..261aa26027c0a57b6bea94a77e5195b1b6c14a81 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml @@ -0,0 +1,7 @@ +description: '以下はビジネスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afe4e74f3010fc98aad3e463fc669302846b2d70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml @@ -0,0 +1,7 @@ +description: '以下は化学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e40a659312a40b77bccedf8907b2c157c788c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml @@ -0,0 +1,7 @@ +description: '以下はコンピュータサイエンスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c26fb8e97351c1d70e360ff14dd39287600a379b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml @@ -0,0 +1,7 @@ +description: '以下は経済学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bed0bc1965078f863dded91da78857514051bc20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml @@ -0,0 +1,7 @@ +description: '以下は工学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4b820595fc9a4b4f2568bd7ce62bee0f45ffbae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml @@ -0,0 +1,7 @@ +description: '以下は健康科学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5eb26103a2a23e316dc66f0fe1b894ddd481d256 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml @@ -0,0 +1,7 @@ +description: '以下は歴史に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..512846f6d3f477594a7a1b63b09700a435376378 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml @@ -0,0 +1,7 @@ +description: '以下は法律に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef030be6d9c666054adb4c3fe42b60a17375e835 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml @@ -0,0 +1,7 @@ +description: '以下は数学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9aa8183eb7705fcda2c0dcae5bbee45b4af524b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml @@ -0,0 +1,7 @@ +description: '以下はその他に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31aa1a658a6a766d74b609e2b9a68b2f226c0741 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml @@ -0,0 +1,7 @@ +description: '以下は物理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8890b927155ea6cd54d2e3c9cbe63943693399c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml @@ -0,0 +1,7 @@ +description: '以下は心理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe7abb531d727e138fd21785b4ee08da82d9f896 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ko +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '답은 \(?([ABCDEFGHIJ])\)?입니다' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "질문：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f86b5b35952ef422018ec2acceff591aded3ef8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ko +task: +- mmlu_prox_ko_biology +- mmlu_prox_ko_business +- mmlu_prox_ko_chemistry +- mmlu_prox_ko_computer_science +- mmlu_prox_ko_economics +- mmlu_prox_ko_engineering +- mmlu_prox_ko_health +- mmlu_prox_ko_history +- mmlu_prox_ko_law +- mmlu_prox_ko_math +- mmlu_prox_ko_other +- mmlu_prox_ko_philosophy +- mmlu_prox_ko_physics +- mmlu_prox_ko_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcd695f6188a184352e52c2f540f6c9e0739e412 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml @@ -0,0 +1,8 @@ +description: '다음은 생물학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ee56c9e83f38ddff9f070c43ef15a42415a135 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml @@ -0,0 +1,8 @@ +description: '다음은 경영학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ad92ca44843cb5a15d07c529c7bb9a368092bdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml @@ -0,0 +1,8 @@ +description: '다음은 화학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b33bb33c886dcefd5e6345d811aede440d76a02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml @@ -0,0 +1,8 @@ +description: '다음은 컴퓨터 과학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a7df38786dc807e2620f40b4d177d732c76d84b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml @@ -0,0 +1,8 @@ +description: '다음은 경제학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f2852eb4eb98595012a0fe824db62d78c6529fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml @@ -0,0 +1,8 @@ +description: '다음은 공학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9322b2074a5238dfbd078b457dd3d2b66abb9c3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml @@ -0,0 +1,8 @@ +description: '다음은 건강에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f82005689f11631dc8506d785d87420563186980 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml @@ -0,0 +1,8 @@ +description: '다음은 역사에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90c1e5fac04e3f91712ced30915d9d18fe6d0dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml @@ -0,0 +1,8 @@ +description: '다음은 수학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50aad077e34943e8fed2e62543f9ef23c5fc33aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml @@ -0,0 +1,8 @@ +description: '다음은 기타에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c274a0e3cd9c66e8fb323d71d6c7081fbff0aee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml @@ -0,0 +1,8 @@ +description: '다음은 철학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be0250ec9311ea3f42cc048dba16e9d4a3fd77b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml @@ -0,0 +1,8 @@ +description: '다음은 물리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06b8aafbbe554e4a748686597349a33c1bbfbc5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml @@ -0,0 +1,8 @@ +description: '다음은 심리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/lang_libs.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/lang_libs.py new file mode 100644 index 0000000000000000000000000000000000000000..9f6e350528dbf1bf2f1adc0adf15a7d14a1adfbe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/lang_libs.py @@ -0,0 +1,318 @@ +LANG_LIBS = { + "en": [ + "Question:", + "Options:", + "Answer: Let's think step by step.", + 'The following are multiple choice questions (with answers) about {subject}. Think step by step and then finish your answer with "{ans_suffix}" where X is the correct letter choice.', + "A: Let's think step by step.", + "the answer is ({})", + ], + "ja": [ + "質問：", + "選択肢：", + "回答：一歩一歩考えていきましょう。", + "以下は{subject}に関する選択問題（解答付き）です。段階的に考え、最後に「{ans_suffix}」と回答を締めくくってください。Xは正解の選択肢を示す文字です。", + "A: 一歩一歩考えていきましょう。", + "答えは ({}) です", + ], + "zh": [ + "问题：", + "选项：", + "答案：让我们一步一步地思考。", + '以下是关于{subject}的选择题（带有答案）。请逐步思考，然后以"{ans_suffix}"结束您的回答，其中X是正确的选项字母。', + "A: 让我们一步一步地思考。", + "答案是 ({})", + ], + "ko": [ + "질문：", + "선택 사항：", + "답변: 한 단계씩 생각해 봅시다.", + '다음은 {subject}에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "{ans_suffix}"로 답변을 마무리하세요. 여기서 X는 올바른 선택지 문자입니다.', + "A: 한 단계씩 생각해 봅시다.", + "답은 ({})입니다", + ], + "fr": [ + "Question :", + "Options :", + "Réponse : Réfléchissons étape par étape.", + 'Voici des questions à choix multiples (avec réponses) sur {subject}. Réfléchissez étape par étape, puis terminez votre réponse par "{ans_suffix}" où X est la lettre correspondant au bon choix.', + "A: Réfléchissons étape par étape.", + "La réponse est ({})", + ], + "de": [ + "Frage:", + "Optionen:", + "Antwort: Denken wir Schritt für Schritt nach.", + 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu {subject}. Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "{ans_suffix}", wobei X der richtige Buchstabe ist.', + "A: Denken wir Schritt für Schritt nach.", + "Die Antwort ist ({})", + ], + "es": [ + "Pregunta:", + "Opciones:", + "Respuesta: Pensemos paso a paso.", + 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre {subject}. Piense paso a paso y luego termine su respuesta con "{ans_suffix}" donde X es la letra de la opción correcta.', + "A: Pensemos paso a paso.", + "La respuesta es ({})", + ], + "pt": [ + "Pergunta:", + "Opções:", + "Resposta: Vamos pensar passo a passo.", + 'A seguir estão perguntas de múltipla escolha (com respostas) sobre {subject}. Pense passo a passo e termine sua resposta com "{ans_suffix}" onde X é a letra da opção correta.', + "A: Vamos pensar passo a passo.", + "A resposta é ({})", + ], + "sw": [ + "Swali:", + "Chaguo:", + "Jibu: Hebu tufikiria hatua kwa hatua.", + 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu {subject}. Fikiria hatua kwa hatua kisha malizia jibu lako kwa "{ans_suffix}" ambapo X ni herufi ya chaguo sahihi.', + "A: Hebu tufikiria hatua kwa hatua.", + "Jibu ni ({})", + ], + "th": [ + "คำถาม:", + "ตัวเลือก:", + "คำตอบ: มาคิดทีละขั้นตอนกัน", + 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ {subject} คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย "{ans_suffix}" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง', + "A: มาคิดทีละขั้นตอนกัน", + "คำตอบคือ ({})", + ], + "ar": [ + "سؤال:", + "الخيارات:", + "الإجابة: دعنا نفكر خطوة بخطوة.", + "فيما يلي أسئلة اختيار من متعدد (مع إجابات) حول {subject}. فكر خطوة بخطوة ثم أنهِ إجابتك بـ '{ans_suffix}' حيث X هو حرف الخيار الصحيح.", + "أ: دعنا نفكر خطوة بخطوة.", + "الإجابة هي ({})", + ], + "hi": [ + "प्रश्न:", + "विकल्प:", + "उत्तर: चलिए चरण-दर-चरण सोचते हैं।", + 'निम्नलिखित {subject} के बारे में बहुविकल्पीय प्रश्न (उत्तरों के साथ) हैं। चरण-दर-चरण सोचें और फिर अपने उत्तर को "{ans_suffix}" के साथ समाप्त करें जहां X सही विकल्प का अक्षर है।', + "A: चलिए चरण-दर-चरण सोचते हैं।", + "उत्तर है ({})", + ], + "bn": [ + "প্রশ্ন:", + "বিকল্পগুলি:", + "উত্তর: আসুন ধাপে ধাপে চিন্তা করি।", + 'নিম্নলিখিত {subject} সম্পর্কে বহুনির্বাচনী প্রশ্ন (উত্তরসহ)। ধাপে ধাপে চিন্তা করুন এবং তারপর আপনার উত্তর "{ans_suffix}" দিয়ে শেষ করুন যেখানে X হল সঠিক বিকল্পের অক্ষর।', + "A: আসুন ধাপে ধাপে চিন্তা করি।", + "উত্তর হল ({})", + ], +} + + +LANG_SUBJECTS = { + "en": { + "biology": "biology", + "business": "business", + "chemistry": "chemistry", + "computer_science": "computer_science", + "economics": "economics", + "engineering": "engineering", + "health": "health", + "history": "history", + "law": "law", + "math": "math", + "other": "other", + "philosophy": "philosophy", + "physics": "physics", + "psychology": "psychology", + }, + "ja": { + "biology": "生物学", + "business": "ビジネス", + "chemistry": "化学", + "computer_science": "コンピュータサイエンス", + "economics": "経済学", + "engineering": "工学", + "health": "健康科学", + "history": "歴史", + "law": "法律", + "math": "数学", + "other": "その他", + "philosophy": "哲学", + "physics": "物理学", + "psychology": "心理学", + }, + "zh": { + "biology": "生物学", + "business": "商业", + "chemistry": "化学", + "computer_science": "计算机科学", + "economics": "经济学", + "engineering": "工程学", + "health": "健康", + "history": "历史", + "law": "法律", + "math": "数学", + "other": "其他", + "philosophy": "哲学", + "physics": "物理学", + "psychology": "心理学", + }, + "ko": { + "biology": "생물학", + "business": "경영학", + "chemistry": "화학", + "computer_science": "컴퓨터 과학", + "economics": "경제학", + "engineering": "공학", + "health": "건강", + "history": "역사", + "law": "법률", + "math": "수학", + "other": "기타", + "philosophy": "철학", + "physics": "물리학", + "psychology": "심리학", + }, + "fr": { + "biology": "biologie", + "business": "commerce", + "chemistry": "chimie", + "computer_science": "informatique", + "economics": "économie", + "engineering": "ingénierie", + "health": "santé", + "history": "histoire", + "law": "droit", + "math": "mathématiques", + "other": "autre", + "philosophy": "philosophie", + "physics": "physique", + "psychology": "psychologie", + }, + "de": { + "biology": "Biologie", + "business": "Wirtschaft", + "chemistry": "Chemie", + "computer_science": "Informatik", + "economics": "Ökonomie", + "engineering": "Ingenieurwesen", + "health": "Gesundheit", + "history": "Geschichte", + "law": "Recht", + "math": "Mathematik", + "other": "Sonstiges", + "philosophy": "Philosophie", + "physics": "Physik", + "psychology": "Psychologie", + }, + "es": { + "biology": "biología", + "business": "negocios", + "chemistry": "química", + "computer_science": "informática", + "economics": "economía", + "engineering": "ingeniería", + "health": "salud", + "history": "historia", + "law": "derecho", + "math": "matemáticas", + "other": "otro", + "philosophy": "filosofía", + "physics": "física", + "psychology": "psicología", + }, + "pt": { + "biology": "biologia", + "business": "negócios", + "chemistry": "química", + "computer_science": "ciência da computação", + "economics": "economia", + "engineering": "engenharia", + "health": "saúde", + "history": "história", + "law": "direito", + "math": "matemática", + "other": "outro", + "philosophy": "filosofia", + "physics": "física", + "psychology": "psicologia", + }, + "sw": { + "biology": "biolojia", + "business": "biashara", + "chemistry": "kemia", + "computer_science": "sayansi ya kompyuta", + "economics": "uchumi", + "engineering": "uhandisi", + "health": "afya", + "history": "historia", + "law": "sheria", + "math": "hisabati", + "other": "nyingine", + "philosophy": "falsafa", + "physics": "fizikia", + "psychology": "saikolojia", + }, + "th": { + "biology": "ชีววิทยา", + "business": "ธุรกิจ", + "chemistry": "เคมี", + "computer_science": "วิทยาการคอมพิวเตอร์", + "economics": "เศรษฐศาสตร์", + "engineering": "วิศวกรรมศาสตร์", + "health": "สุขภาพ", + "history": "ประวัติศาสตร์", + "law": "กฎหมาย", + "math": "คณิตศาสตร์", + "other": "อื่นๆ", + "philosophy": "ปรัชญา", + "physics": "ฟิสิกส์", + "psychology": "จิตวิทยา", + }, + "ar": { + "biology": "علم الأحياء", + "business": "الأعمال", + "chemistry": "الكيمياء", + "computer_science": "علوم الكمبيوتر", + "economics": "الاقتصاد", + "engineering": "الهندسة", + "health": "الصحة", + "history": "التاريخ", + "law": "القانون", + "math": "الرياضيات", + "other": "أخرى", + "philosophy": "الفلسفة", + "physics": "الفيزياء", + "psychology": "علم النفس", + }, + "hi": { + "biology": "जीव विज्ञान", + "business": "व्यापार", + "chemistry": "रसायन विज्ञान", + "computer_science": "कंप्यूटर विज्ञान", + "economics": "अर्थशास्त्र", + "engineering": "इंजीनियरिंग", + "health": "स्वास्थ्य", + "history": "इतिहास", + "law": "कानून", + "math": "गणित", + "other": "अन्य", + "philosophy": "दर्शनशास्त्र", + "physics": "भौतिकी", + "psychology": "मनोविज्ञान", + }, + "bn": { + "biology": "জীববিজ্ঞান", + "business": "ব্যবসা", + "chemistry": "রসায়ন", + "computer_science": "কম্পিউটার বিজ্ঞান", + "economics": "অর্থনীতি", + "engineering": "প্রকৌশল", + "health": "স্বাস্থ্য", + "history": "ইতিহাস", + "law": "আইন", + "math": "গণিত", + "other": "অন্যান্য", + "philosophy": "দর্শন", + "physics": "পদার্থবিজ্ঞান", + "psychology": "মনোবিজ্ঞান", + }, +} diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..52b9d8206a9394ea8089d985293f83a9ed216b69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py @@ -0,0 +1,114 @@ +import os +import shutil + +import yaml +from lang_libs import LANG_LIBS, LANG_SUBJECTS + + +language_word_to_abbr = { + "English": "en", + "Japanese": "ja", + "Chinese": "zh", + "Korean": "ko", + "French": "fr", + "German": "de", + "Spanish": "es", + "Portuguese": "pt", + "Swahili": "sw", + "Thai": "th", + "Arabic": "ar", + "Hindi": "hi", + "Bengali": "bn", +} + +language_abbr_to_word = {v: k for k, v in language_word_to_abbr.items()} + + +if __name__ == "__main__": + mmlu_pro_config_dir = "../mmlu_pro" + mmlu_prox_repo_id = "li-lab/MMLU-ProX" + + for lang_abbr in language_abbr_to_word: + os.makedirs(lang_abbr, exist_ok=True) + lang_lib_list = LANG_LIBS[lang_abbr] + lang_sbj_dict = LANG_SUBJECTS[lang_abbr] + + with ( + open("template/_lang_template_yaml", "r") as reader, + open(f"{lang_abbr}/_{lang_abbr}_template_yaml", "w") as writer, + ): + for line in reader.readlines(): + if "{repo_id}" in line: + line = line.format(repo_id=mmlu_prox_repo_id) + if "{lang}" in line: + line = line.format(lang=lang_abbr) + if "{ans_regex}" in line: + ans_regex = lang_lib_list[-1].replace( + "({})", "\(?([ABCDEFGHIJ])\)?" + ) + if lang_abbr == "en": + ans_regex = ans_regex.lstrip("the").strip() + line = line.format(ans_regex=ans_regex) + if "{que_prefix}" in line: + line = line.format(que_prefix=lang_lib_list[0]) + writer.write(line) + + shutil.copy("template/utils.py", f"{lang_abbr}/utils.py") + + group_name = f"mmlu_prox_{lang_abbr}" + group_dict = dict( + group=group_name, + task=[f"{group_name}_{sbj}" for sbj in LANG_SUBJECTS[lang_abbr]], + aggregate_metric_list=[ + dict( + aggregation="mean", + metric="exact_match", + weight_by_size=True, + filter_list="custom-extract", + ) + ], + metadata=dict(version=0.0), + ) + with open(f"{lang_abbr}/_{group_name}.yaml", "w", encoding="utf-8") as f: + yaml.dump( + group_dict, + f, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) + + for sbj in lang_sbj_dict: + with open( + f"{mmlu_pro_config_dir}/mmlu_pro_{sbj}.yaml", "r", encoding="utf-8" + ) as f: + sbj_yaml_last_line = None + for line in f.readlines(): + if line.startswith("process_docs:"): + sbj_yaml_last_line = line.strip() + + sbj_dict = dict( + description=lang_lib_list[3].format( + subject=lang_sbj_dict[sbj], ans_suffix=lang_lib_list[5].format("X") + ) + + "\n", + include=f"_{lang_abbr}_template_yaml", + task=f"{group_name}_{sbj}", + task_alias=sbj, + ) + with open( + f"{lang_abbr}/{group_name}_{sbj}.yaml", "w", encoding="utf-8" + ) as f: + yaml.dump( + sbj_dict, + f, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) + with open( + f"{lang_abbr}/{group_name}_{sbj}.yaml", "a", encoding="utf-8" + ) as f: + f.write(sbj_yaml_last_line + "\n") + + print(f"Finished {lang_abbr}") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7f60973dfcfeb5ed6cd9810b89ae161f8ed692f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_pt +task: +- mmlu_prox_pt_biology +- mmlu_prox_pt_business +- mmlu_prox_pt_chemistry +- mmlu_prox_pt_computer_science +- mmlu_prox_pt_economics +- mmlu_prox_pt_engineering +- mmlu_prox_pt_health +- mmlu_prox_pt_history +- mmlu_prox_pt_law +- mmlu_prox_pt_math +- mmlu_prox_pt_other +- mmlu_prox_pt_philosophy +- mmlu_prox_pt_physics +- mmlu_prox_pt_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac461a3f88c4943d40a13416681e08a31beee70d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: pt +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'A resposta é \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pergunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45be5da1e1e4f5e36dd9e98e106ad057ed2ab218 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_biology.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre biologia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88116d1e41ff47578872511ea2dd1070e962570f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre química. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f77faf8b1f49006e020063c784318a628f2fa653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre ciência + da computação. Pense passo a passo e termine sua resposta com "A resposta é (X)" + onde X é a letra da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d06e0099f5d9fa664563df619fd206339a0e1f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_economics.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre economia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4f1a61d0322d5171e4acef4164e57190cbe2f2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_engineering.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre engenharia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01d264e9339c3c5ceb4170c9fcc0e132a370df3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_health.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre saúde. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97c5b73645a530a06f79d1edfda261ce96b3ba90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_history.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre história. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba48aa2d7d79005414eddbe194f1ee77986b9969 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_pt_law.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre direito. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_template_yaml +task: mmlu_prox_pt_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9a2fba188a31ca78e8b243b438fa33882804ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_sw +task: +- mmlu_prox_sw_biology +- mmlu_prox_sw_business +- mmlu_prox_sw_chemistry +- mmlu_prox_sw_computer_science +- mmlu_prox_sw_economics +- mmlu_prox_sw_engineering +- mmlu_prox_sw_health +- mmlu_prox_sw_history +- mmlu_prox_sw_law +- mmlu_prox_sw_math +- mmlu_prox_sw_other +- mmlu_prox_sw_philosophy +- mmlu_prox_sw_physics +- mmlu_prox_sw_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adb5ce0f7101378609e5305eddb544ea8b810e46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uchumi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4db109fb1c01f4de671c63da6ecefbd69bf2a2e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu afya. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b4b39d4e8a4790487bd49bbc5fc9a70d82b9ee7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu historia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e49ae99802fcb5ade0dd5dc61e7fa8402507dfc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sheria. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef2656aff6d404e69fa3fd4655969801d1fad76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7feae9f0b1418bb514afe7c773fccf6eb379d1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f035787e33118323fd2c91698c4a6282fe858b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e9599e5c0036574ac0387e6056e15023b8f648 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edd501fa06848966e1546f9fd1e3c78d4d237223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8be2f268be83fbdb8b49d7da39b0533a50f6bf5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..030fd2e090ab8dabac6abc249f7577160b6b8ac8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f7b38e0e9be0150471694c84d62abdc2c5d6dcd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abe2d6f5ac68b7f53a86d6147cf8664001830444 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33085c5c2a6cf6054dd82879381365a66cc770cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae69628a60bb02fefd9320c651ae6e9df35dc181 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf226b5a437c2de1a554e7f997d7604fe1c5acf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37b67158f4559b6a23072fca79405db8f49c2ee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dd67daf3f0c9f1691974b85f2a86c8b8eadcb97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfaee537e7e91e95a276f0dd2214ad0291ace25e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fde605633bd9f8c4146a7a213ac69bdc2a100680 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2f95fd2b12870ea7e6a4cebddbe8c4d6ac0eeba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e07150c7fdecd13a80a995badaca082fdf9d0d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bccaf4a4164b9a471395fddc9b01ff8bb838108 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca72f214c40c18a7688d76ef7dcc12c69aebb11b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a47f15b6b44872250e000ff3e65529fdfd317e19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88829f61c1f0c087808364b3eb11871ea8af2302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad3de69466630145c31bc474ce399f6067f72cff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a84f610571a1eb1ed58dda6988628c4cc83f761 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ef60495f02f99e8090737056d582e7f962047f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2518b48dc991d272263f01c67908bb703b277139 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c17c5dd8758471f9d6485e1c92414d9b16bee5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2297b0f122817d57a14b59eecfc5d70a7fa02f05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a04374117fd545b719ad12fa838eae269efad696 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b8e572b9ef7b84e1baa098b57e3bc7e1d6bcb25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c25aa01755421c42b70335f7c1a8cf9ccea659a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89ebc81c7f0d9999811da446610df2b5e2a7e316 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d23cb2b93d3391944e7aa30222381051e593b8d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ff913d961a5f17bdc987659e1fdc36658b5b06e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d705e8485c85f3aa3493fad724aca687998155b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a9a7b8743e5d5f6cfc133f10e0c00125b87d962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..034cfa8bdbe78b7d9d0054da332e9b4702f01b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e66514c8a27b899b6d9545ac812d309cc85d62a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md b/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e9d0da12f667975456235581e3e95e27e5114a84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md @@ -0,0 +1,150 @@ +# MMMU Benchmark + +### Paper + +Title: `MMMU: A Massive Multi-discipline MultimodalUnderstanding and Reasoning Benchmark for Expert AGI` + +Abstract: `MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.` + +`The benchmark is composed of 30 tasks, for a total of 900 mixed image+text examples (some with multiple images in context)` + +Homepage: `https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu` + +Note: Some questions have multiple images in context. To control for this use `max_images=N` in model init. + +### Citation + +``` +@inproceedings{yue2023mmmu, + title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI}, + author={Xiang Yue and Yuansheng Ni and Kai Zhang and Tianyu Zheng and Ruoqi Liu and Ge Zhang and Samuel Stevens and Dongfu Jiang and Weiming Ren and Yuxuan Sun and Cong Wei and Botao Yu and Ruibin Yuan and Renliang Sun and Ming Yin and Boyuan Zheng and Zhenzhu Yang and Yibo Liu and Wenhao Huang and Huan Sun and Yu Su and Wenhu Chen}, + booktitle={Proceedings of CVPR}, + year={2024}, + } +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmmu_val` +* `mmmu_val_art_and_design` +* `mmmu_val_business` +* `mmmu_val_health_and_medicine` +* `mmmu_val_humanities_and_social_science` +* `mmmu_val_science` +* `mmmu_val_tech_and_engineering` + +#### Tags + + +#### Tasks + +* `mmmu_val_accounting` +* `mmmu_val_agriculture` +* `mmmu_val_architecture_and_engineering.yaml` +* `mmmu_val_art` +* `mmmu_val_art_theory` +* `mmmu_val_basic_medical_science` +* `mmmu_val_biology` +* `mmmu_val_chemistry` +* `mmmu_val_computer_science` +* `mmmu_val_clinical_medicine` +* `mmmu_val_design` +* `mmmu_val_diagnostics_and_laboratory_medicine` +* `mmmu_val_electronics` +* `mmmu_val_energy_and_power` +* `mmmu_val_economics` +* `mmmu_val_finance` +* `mmmu_val_geography` +* `mmmu_val_history` +* ... + +### Variants + +The `mmmu_val` group implements MMMU using processing code [from the original MMMU authors](https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu) and uses the prompt format found in [the MMMU repository for Llava-1.5](https://github.com/MMMU-Benchmark/MMMU/blob/main/mmmu/configs/llava1.5.yaml). This implementation should give scores on par with or slightly higher than those reported by [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/tasks/mmmu) for `mmmu_val` and the MMMU repository code. + +Scores on several tested models (**all with `--apply_chat_template`**) are: + +Qwen2-VL-2B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-2B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3778|± |0.0155| +| - Art and Design | 0|none | |acc |↑ |0.5500|± |0.0415| +| - Business | 0|none | |acc |↑ |0.3600|± |0.0389| +| - Health and Medicine | 0|none | |acc |↑ |0.3667|± |0.0394| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5167|± |0.0438| +| - Science | 0|none | |acc |↑ |0.2467|± |0.0352| +| - Tech and Engineering | 0|none | |acc |↑ |0.3143|± |0.0317| +``` +Author-reported score: 41.1% + + +Qwen2-VL-7B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-7B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.5056|± |0.0160| +| - Art and Design | 0|none | |acc |↑ |0.6917|± |0.0398| +| - Business | 0|none | |acc |↑ |0.4333|± |0.0406| +| - Health and Medicine | 0|none | |acc |↑ |0.5667|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.6750|± |0.0426| +| - Science | 0|none | |acc |↑ |0.3800|± |0.0392| +| - Tech and Engineering | 0|none | |acc |↑ |0.4000|± |0.0341| +``` +Author-reported score: 54.1% + +Idefics2-8B: +``` +hf-multimodal (pretrained=HuggingFaceM4/idefics2-8b,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True,max_images=2), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.4011|± |0.0154| +| - Art and Design | 0|none | |acc |↑ |0.6167|± |0.0436| +| - Business | 0|none | |acc |↑ |0.3200|± |0.0373| +| - Health and Medicine | 0|none | |acc |↑ |0.4000|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5750|± |0.0424| +| - Science | 0|none | |acc |↑ |0.2600|± |0.0358| +| - Tech and Engineering | 0|none | |acc |↑ |0.3381|± |0.0312| +``` +Author-reported score: ~43% + +Llava-v1.6-Mistral-7B: +``` +hf-multimodal (pretrained=llava-hf/llava-v1.6-mistral-7b-hf,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3522|± |0.0151| +| - Art and Design | 0|none | |acc |↑ |0.5167|± |0.0440| +| - Business | 0|none | |acc |↑ |0.2667|± |0.0362| +| - Health and Medicine | 0|none | |acc |↑ |0.3867|± |0.0397| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5917|± |0.0433| +| - Science | 0|none | |acc |↑ |0.2200|± |0.0342| +| - Tech and Engineering | 0|none | |acc |↑ |0.2524|± |0.0299| +``` +Author-reported score: 35.3% + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0dda1876fea6415c442957b5bafb24499789276 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_art_and_design +group_alias: Art and Design +task: + - mmmu_val_art + - mmmu_val_art_theory + - mmmu_val_design + - mmmu_val_music +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497948e5e5c0f5ea7a8c72a5624f0a925eedc7d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_business +group_alias: Business +task: + - mmmu_val_accounting + - mmmu_val_economics + - mmmu_val_finance + - mmmu_val_manage + - mmmu_val_marketing +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04709d3d4653b4a1bf912eaad2ee7dde27330822 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_health_and_medicine +group_alias: Health and Medicine +task: + - mmmu_val_basic_medical_science + - mmmu_val_clinical_medicine + - mmmu_val_diagnostics_and_laboratory_medicine + - mmmu_val_pharmacy + - mmmu_val_public_health +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d70a9ea3f4e802c6eae7c8a6ceeac2f295a9dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_humanities_and_social_science +group_alias: Humanities and Social Science +task: + - mmmu_val_history + - mmmu_val_literature + - mmmu_val_sociology + - mmmu_val_psychology +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bcd234bbc065f7a70a79fad3deba1e00bf2bc45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml @@ -0,0 +1,14 @@ +group: mmmu_val +task: + - mmmu_val_art_and_design + - mmmu_val_business + - mmmu_val_health_and_medicine + - mmmu_val_humanities_and_social_science + - mmmu_val_science + - mmmu_val_tech_and_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b41585a6563b42e1706e29cceb96b11db152dd2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_science +group_alias: Science +task: + - mmmu_val_biology + - mmmu_val_chemistry + - mmmu_val_geography + - mmmu_val_math + - mmmu_val_physics +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..956f9906b3668df97880c121dd95b0b8fd12b850 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml @@ -0,0 +1,16 @@ +group: mmmu_val_tech_and_engineering +group_alias: Tech and Engineering +task: + - mmmu_val_agriculture + - mmmu_val_architecture_and_engineering + - mmmu_val_computer_science + - mmmu_val_electronics + - mmmu_val_energy_and_power + - mmmu_val_materials + - mmmu_val_mechanical_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f92ce60d6becd951f7aff35fb82b5345bf0fdeeb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml @@ -0,0 +1,19 @@ +dataset_path: MMMU/MMMU +validation_split: validation +output_type: generate_until +doc_to_image: !function utils.doc_to_image +doc_to_text: !function utils.doc_to_text +doc_to_target: "answer" +process_results: !function utils.process_results +generation_kwargs: + until: + - "<|endoftext|>" + temperature: 0.0 + do_sample: false + max_gen_toks: 512 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c4aa41d9c7b4f29ac73e33ae8268ae2f7aa611a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_accounting +include: _template_yaml +task_alias: Accounting +dataset_name: Accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..387f33f7a985a38f0790b45db64b77a02ee222ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_agriculture +include: _template_yaml +task_alias: Agriculture +dataset_name: Agriculture diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..829ee3a8f8b47d0ab9b499e4fe810d8b3c30f2b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_architecture_and_engineering +include: _template_yaml +task_alias: Architecture and Engineering +dataset_name: Architecture_and_Engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff812ab3c1d02ddd0e0e0152155a97499cfa9411 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art +include: _template_yaml +task_alias: Art +dataset_name: Art diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6331871a56f2fe103179a31575dd049e94ca602 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art_theory +include: _template_yaml +task_alias: Art Theory +dataset_name: Art_Theory diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7486f1ee341a5a407f8fb3f5d93af5d45c6360b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_basic_medical_science +include: _template_yaml +task_alias: Basic Medical Science +dataset_name: Basic_Medical_Science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49d36f380f9f6a12912522388cb8cc5d93b1360a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_biology +include: _template_yaml +task_alias: Biology +dataset_name: Biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb096531c2e33b4dd754da03687a0778062a30cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_chemistry +include: _template_yaml +task_alias: Chemistry +dataset_name: Chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0833d165f9e54b12cf5eb748c96112bcb6e152b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_clinical_medicine +include: _template_yaml +task_alias: Clinical Medicine +dataset_name: Clinical_Medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53c91a61b33c60ac21f657dac44a91b16a37ce87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_computer_science +include: _template_yaml +task_alias: Computer Science +dataset_name: Computer_Science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0367dd65ee0d1c6c5446baa2fac17da50778c9b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_design +include: _template_yaml +task_alias: Design +dataset_name: Design diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ec1b7dca693ad3c3c5030aacdaa72fbc0551323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_diagnostics_and_laboratory_medicine +include: _template_yaml +task_alias: Diagnostics and Laboratory Medicine +dataset_name: Diagnostics_and_Laboratory_Medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6465da7416b7bc3c23ef44482d125e0e6fb9ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_economics +include: _template_yaml +task_alias: Economics +dataset_name: Economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02df22f8a39f63a7a8c0c318c73b2135c8233275 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_electronics +include: _template_yaml +task_alias: Electronics +dataset_name: Electronics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..add24ebed72c43fa49beadd8a75f5f6aca1b6122 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_energy_and_power +include: _template_yaml +task_alias: Energy and Power +dataset_name: Energy_and_Power diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b6f042c1a1d53e43d21f42e8ac50854c429bd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_finance +include: _template_yaml +task_alias: Finance +dataset_name: Finance diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5f90221366966cc24d981e128f7ff8a172b41e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_geography +include: _template_yaml +task_alias: Geography +dataset_name: Geography diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..435388fd74b86392fe4b4124056f34590528d726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_history +include: _template_yaml +task_alias: History +dataset_name: History diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e0a1ba50372ed8014d407e72d946b07322e08d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_literature +include: _template_yaml +task_alias: Literature +dataset_name: Literature diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd1992411458354a650165e3872e28da4374c101 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_manage +include: _template_yaml +task_alias: Manage +dataset_name: Manage diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53848171be0e0b92c540da15bd3a9ffc60d00e9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_marketing +include: _template_yaml +task_alias: Marketing +dataset_name: Marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee469443163c792073cda7b1b485d1e7fca6044b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_materials +include: _template_yaml +task_alias: Materials +dataset_name: Materials diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6817689ab1f8a7232ebdd367592dc782893bea4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_math +include: _template_yaml +task_alias: Math +dataset_name: Math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b9d95be85b7181514e4918439690e4ce5f84eea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_mechanical_engineering +include: _template_yaml +task_alias: Mechanical Engineering +dataset_name: Mechanical_Engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf4032dea3e9425a01cd924f57a487a8de9b647e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_music +include: _template_yaml +task_alias: Music +dataset_name: Music diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04a4f1ff67a160da03ca99ebcd66047b48524fbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_pharmacy +include: _template_yaml +task_alias: Pharmacy +dataset_name: Pharmacy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53371cad73c7ff9e5b340a18a53eb94c452720c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_physics +include: _template_yaml +task_alias: Physics +dataset_name: Physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8471d4a18f4bf566186d2c9938d14a2f7eccd769 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_psychology +include: _template_yaml +task_alias: Psychology +dataset_name: Psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aefc0590a3878e182494633463279e1b3ec40021 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_public_health +include: _template_yaml +task_alias: Public Health +dataset_name: Public_Health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da664a8e7a20f038086829478945f9bb4cdbe53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_sociology +include: _template_yaml +task_alias: Sociology +dataset_name: Sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..87a3022098b29233615b404fdc91da2403154c3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py @@ -0,0 +1,341 @@ +import ast +import random +import re + +import numpy as np + + +random.seed(42) + + +# source for prompt fstrings: https://github.com/MMMU-Benchmark/MMMU/blob/7787d60648c82a9d40acd656fa541a6c74f58995/eval/configs/llava1.5.yaml#L3 +MULTI_CHOICE_EXAMPLE_FORMAT = """{} + +{} + +Answer with the option's letter from the given choices directly.""" + + +SHORT_ANS_EXAMPLE_FORMAT = """{} + +Answer the question using a single word or phrase.""" + +START_CHR = "A" + + +def doc_to_image(doc): + # get formatted prompt (incl. multi-choice options) pre- reformatting + input_text = _doc_to_text(doc) + # locate instances in input + image_placeholders = [ + img.replace(" ", "_").replace("<", "").replace(">", "") + for img in re.findall("", input_text) + ] + + # collect visuals (can have dupes of a given image or be out of order) + # E.g. validation_Math_19 contains and but seen as [, , , , ] + visuals = [doc[img] for img in image_placeholders] + + return visuals + + +def doc_to_text(doc): + """Get the prompt for a given document.""" + + prompt = _doc_to_text(doc) + + for i in range(1, 8): + # replace with . TODO: check this is always the right decision incl. for non-HF models + prompt = prompt.replace(f"", "") + + return prompt + + +def _doc_to_text(doc): + """Helper--get the prompt for a given document but DO NOT yet replace with .""" + + if doc["question_type"] == "multiple-choice": + choices_str = "" + + for i, choice in enumerate(ast.literal_eval(doc["options"])): + # add (A) {choice1}\n , (B) {choice2}\n , and so on + # to create the list of formatted choices in the prompt + choices_str += f"\n({chr(ord(START_CHR) + i)}) {choice}" + + choices_str = ( + choices_str.lstrip() + ) # remove the extraneous prepended \n that we added + + prompt = MULTI_CHOICE_EXAMPLE_FORMAT.format(doc["question"], choices_str) + else: + prompt = SHORT_ANS_EXAMPLE_FORMAT.format(doc["question"]) + + return prompt + + +def process_results(doc, results): + if doc["question_type"] == "multiple-choice": + # multichoice logic + option_strs = ast.literal_eval(doc["options"]) + option_letters = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] + + all_choices = option_letters[: len(option_strs)] + index2ans = {index: ans for index, ans in zip(option_letters, option_strs)} + + pred = parse_multi_choice_response(results[0], all_choices, index2ans) + # print(pred, all_choices, index2ans) + is_correct = eval_multi_choice(doc["answer"], pred) + else: + # freeform response handling + pred = parse_open_response(results[0]) + is_correct = eval_open(doc["answer"], pred) + + return {"acc": float(is_correct)} + + # TODO: it would be better if we could use a Filter for this logic. + + +### Output parsing and answer selection taken from +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/data_utils.py +### and +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/eval_utils.py + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + # print(response, all_choices, index2ans, pred_index) + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..fa4e30ba16635f754461539a14e085ead13586f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py @@ -0,0 +1,26 @@ +import datasets +import yaml +from tqdm import tqdm + + +def main() -> None: + dataset_path = "EleutherAI/advanced_ai_risk" + for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): + file_name = f"{task}.yaml" + try: + with open(f"{file_name}", "w", encoding="utf-8") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_template_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..581c3150ee74869e3bb0d951b9d2b05601baf7c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml @@ -0,0 +1,14 @@ +tag: advanced_ai_risk +dataset_path: EleutherAI/advanced_ai_risk +output_type: multiple_choice +validation_split: validation +target_delimiter: "" +doc_to_text: "Human: {{question}}\n\nAssistant:" +doc_to_target: 0 +doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}" +should_decontaminate: true +doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}" +metric_list: + - metric: acc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51c21b3b88bf8c89e99f2c17e64cc73e87b793f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9d3ad4f0fd3176ce6e7ab69ae1b130ca674357d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e536f01aae13b33c6c5e9b045fad566675cea49e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de4566c05cacca2cc597961a69c445337f7a30b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48e46178afba745b50247970f8971762d47d8145 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7035496b92a9e65439520636e3d474968f10e04a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-myopic-reward +include: _template_yaml +task: advanced_ai_risk_fewshot-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a29143d979ef6c0c7d3f38806e1bab6d020f2a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_fewshot-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..654b8c353935a7bf05bab75c07b763770dc88fe3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_fewshot-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..761adb4c41df581f666167face36a92f444f50f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7daee72b401e0a054e1c251530c5d17fd2170124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90ed8c1e9daf5d7d381006e3dba0c811ce9cc663 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c01d2f6b34830159da1b76efdd61ba5e1957fb25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-training-web-gpt +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-training-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aa78bd373010a03e2888eb458e590cbb23f700c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-survival-instinct +include: _template_yaml +task: advanced_ai_risk_fewshot-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fd0e9ee93f5629f9df6eaa16f8215281a2611a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_human-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7836667af1902d1c3b6e5bc675878f07eb67502 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29bb6cc6c054d114da89a59478cb8f195e7a87d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f91f926bbd96ee4c40cb9eaee23e172f77d12084 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_human-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fed8fdcb9ff8b56b3419320f9761fb85cf72e2f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e34a4b9f98eb11422553795e8886b61cf386ed7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-web-gpt +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-web-gpt