Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml +9 -0
- LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml +26 -0
- LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml +5 -0
- LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml +26 -0
- LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml +26 -0
- LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py +383 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py +149 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml +20 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml +32 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml +7 -0
- LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml +7 -0
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
output_type: generate_until
|
| 2 |
+
generation_kwargs:
|
| 3 |
+
until:
|
| 4 |
+
- "ASSISTANT:"
|
| 5 |
+
image_aspect_ratio: pad
|
| 6 |
+
metadata:
|
| 7 |
+
version: 0.0
|
| 8 |
+
api_type : openai
|
| 9 |
+
gpt_eval_model_name: "gpt-3.5-turbo"
|
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench
|
| 2 |
+
dataset_name: in_domain
|
| 3 |
+
dataset_kwargs:
|
| 4 |
+
token: True
|
| 5 |
+
task: "llava_interleave_bench_in_domain"
|
| 6 |
+
test_split: test
|
| 7 |
+
doc_to_target: "answer"
|
| 8 |
+
doc_to_visual: !function utils.doc_to_visual
|
| 9 |
+
doc_to_text: !function utils.doc_to_text
|
| 10 |
+
process_results: !function utils.interleave_process_results
|
| 11 |
+
|
| 12 |
+
metric_list:
|
| 13 |
+
- metric: overall_score
|
| 14 |
+
aggregation: !function utils.overall_score
|
| 15 |
+
higher_is_better: true
|
| 16 |
+
|
| 17 |
+
generation_kwargs:
|
| 18 |
+
max_new_tokens: 16
|
| 19 |
+
temperature: 0
|
| 20 |
+
do_sample: False
|
| 21 |
+
image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy.
|
| 22 |
+
|
| 23 |
+
lmms_eval_specific_kwargs:
|
| 24 |
+
default:
|
| 25 |
+
oe_post_prompt: ""
|
| 26 |
+
mcq_post_prompt: ""
|
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
group: llava_interleave_bench
|
| 2 |
+
task:
|
| 3 |
+
- llava_interleave_bench_in_domain
|
| 4 |
+
- llava_interleave_bench_out_domain
|
| 5 |
+
- llava_interleave_bench_multi_view
|
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench
|
| 2 |
+
dataset_name: multi_view_in_domain
|
| 3 |
+
dataset_kwargs:
|
| 4 |
+
token: True
|
| 5 |
+
task: "llava_interleave_bench_multi_view"
|
| 6 |
+
test_split: test
|
| 7 |
+
doc_to_target: "answer"
|
| 8 |
+
doc_to_visual: !function utils.doc_to_visual
|
| 9 |
+
doc_to_text: !function utils.doc_to_text_conversation
|
| 10 |
+
process_results: !function utils.interleave_process_results
|
| 11 |
+
|
| 12 |
+
metric_list:
|
| 13 |
+
- metric: overall_score
|
| 14 |
+
aggregation: !function utils.overall_score
|
| 15 |
+
higher_is_better: true
|
| 16 |
+
|
| 17 |
+
generation_kwargs:
|
| 18 |
+
max_new_tokens: 16
|
| 19 |
+
temperature: 0
|
| 20 |
+
do_sample: False
|
| 21 |
+
image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy.
|
| 22 |
+
|
| 23 |
+
lmms_eval_specific_kwargs:
|
| 24 |
+
default:
|
| 25 |
+
oe_post_prompt: ""
|
| 26 |
+
mcq_post_prompt: ""
|
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench
|
| 2 |
+
dataset_name: out_of_domain
|
| 3 |
+
dataset_kwargs:
|
| 4 |
+
token: True
|
| 5 |
+
task: "llava_interleave_bench_out_domain"
|
| 6 |
+
test_split: test
|
| 7 |
+
doc_to_target: "answer"
|
| 8 |
+
doc_to_visual: !function utils.doc_to_visual
|
| 9 |
+
doc_to_text: !function utils.doc_to_text
|
| 10 |
+
process_results: !function utils.interleave_process_results
|
| 11 |
+
|
| 12 |
+
metric_list:
|
| 13 |
+
- metric: overall_score
|
| 14 |
+
aggregation: !function utils.overall_score
|
| 15 |
+
higher_is_better: true
|
| 16 |
+
|
| 17 |
+
generation_kwargs:
|
| 18 |
+
max_new_tokens: 16
|
| 19 |
+
temperature: 0
|
| 20 |
+
do_sample: False
|
| 21 |
+
image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy.
|
| 22 |
+
|
| 23 |
+
lmms_eval_specific_kwargs:
|
| 24 |
+
default:
|
| 25 |
+
oe_post_prompt: ""
|
| 26 |
+
mcq_post_prompt: ""
|
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
import time
|
| 5 |
+
from collections import defaultdict
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import requests
|
| 9 |
+
import yaml
|
| 10 |
+
from loguru import logger as eval_logger
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from rouge import Rouge
|
| 13 |
+
|
| 14 |
+
spot_the_diff = ["Spot-the-Diff", "Birds-to-Words", "CLEVR-Change"]
|
| 15 |
+
image_edit_instruct = ["IEdit", "HQ-Edit", "MagicBrush"]
|
| 16 |
+
visual_story_telling = ["AESOP", "FlintstonesSV", "PororoSV", "VIST"]
|
| 17 |
+
visual_cloze = ["COMICS_Dialogue", "RecipeQA_VisualCloze"]
|
| 18 |
+
text_rich_vqa = ["WebQA", "TQA", "OCR-VQA", "DocVQA"]
|
| 19 |
+
multi_image_vqa = ["MIT-States_StateCoherence", "MIT-States_PropertyCoherence", "VISION", "RecipeQA_ImageCoherence"]
|
| 20 |
+
|
| 21 |
+
puzzle = ["RAVEN"]
|
| 22 |
+
nlrv2 = ["NLVR2_Mantis"]
|
| 23 |
+
qbench = ["QBench"]
|
| 24 |
+
|
| 25 |
+
scan_qa = ["ScanQA"]
|
| 26 |
+
alfred = ["ALFRED"]
|
| 27 |
+
nuscenes = ["nuscenes"]
|
| 28 |
+
scannet_chat = ["ScanNet_chat"]
|
| 29 |
+
scannet_task = ["ScanNet_task"]
|
| 30 |
+
blink = ["BLINK"]
|
| 31 |
+
math_verse = ["MathVerse"]
|
| 32 |
+
sci_verse = ["SciVerse"]
|
| 33 |
+
mantis = ["Mantis"]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def doc_to_visual(doc):
|
| 37 |
+
max_visual_count = 16
|
| 38 |
+
visuals = []
|
| 39 |
+
for i in range(max_visual_count):
|
| 40 |
+
if f"image_{i}" in doc:
|
| 41 |
+
image = doc[f"image_{i}"]
|
| 42 |
+
if image is None:
|
| 43 |
+
continue # Skip this image if it's None
|
| 44 |
+
if isinstance(image, Image.Image):
|
| 45 |
+
visuals.append(image.copy().convert("RGB"))
|
| 46 |
+
else:
|
| 47 |
+
try:
|
| 48 |
+
# If the image is not already a PIL Image, try to open it
|
| 49 |
+
visuals.append(Image.open(image).convert("RGB"))
|
| 50 |
+
except Exception as e:
|
| 51 |
+
print(f"Error opening image_{i}: {e}")
|
| 52 |
+
# Optionally, you can add a placeholder image or just continue
|
| 53 |
+
continue
|
| 54 |
+
|
| 55 |
+
return visuals
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
# This is the place where you format your question
|
| 59 |
+
def doc_to_text(doc, lmms_eval_specific_kwargs=None):
|
| 60 |
+
if lmms_eval_specific_kwargs is None:
|
| 61 |
+
lmms_eval_specific_kwargs = {}
|
| 62 |
+
|
| 63 |
+
oe_post_prompt = ""
|
| 64 |
+
if "oe_post_prompt" in lmms_eval_specific_kwargs:
|
| 65 |
+
oe_post_prompt = lmms_eval_specific_kwargs["oe_post_prompt"]
|
| 66 |
+
|
| 67 |
+
mcq_post_prompt = ""
|
| 68 |
+
if "mcq_post_prompt" in lmms_eval_specific_kwargs:
|
| 69 |
+
mcq_post_prompt = lmms_eval_specific_kwargs["mcq_post_prompt"]
|
| 70 |
+
|
| 71 |
+
user_prompt = doc["question"]
|
| 72 |
+
|
| 73 |
+
if mcq_post_prompt != "" and doc["question_type"] == "multi-choice":
|
| 74 |
+
user_prompt = user_prompt.split("Your answer is:")[0].split("\n")[0].strip()
|
| 75 |
+
user_prompt = f"{user_prompt}\n{mcq_post_prompt}"
|
| 76 |
+
|
| 77 |
+
if oe_post_prompt != "" and doc["question_type"] == "open-ended":
|
| 78 |
+
user_prompt = f"{user_prompt}\n{oe_post_prompt}"
|
| 79 |
+
|
| 80 |
+
return user_prompt
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def doc_to_text_conversation(doc, lmms_eval_specific_kwargs=None):
|
| 84 |
+
if lmms_eval_specific_kwargs is None:
|
| 85 |
+
lmms_eval_specific_kwargs = {}
|
| 86 |
+
|
| 87 |
+
conversations = doc["conversations"]
|
| 88 |
+
|
| 89 |
+
if isinstance(conversations, list):
|
| 90 |
+
user_prompt = json.dumps(conversations)
|
| 91 |
+
else:
|
| 92 |
+
user_prompt = conversations
|
| 93 |
+
|
| 94 |
+
return user_prompt
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def doc_to_text_multi_turn(doc, lmms_eval_specific_kwargs=None):
|
| 98 |
+
if lmms_eval_specific_kwargs is None:
|
| 99 |
+
lmms_eval_specific_kwargs = {}
|
| 100 |
+
|
| 101 |
+
return doc["conversations"]
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def interleave_process_results(doc, results):
|
| 105 |
+
pred = results[0]
|
| 106 |
+
sample_id = doc["sample_id"]
|
| 107 |
+
|
| 108 |
+
if doc["question_type"] == "multi-choice":
|
| 109 |
+
score = mcq_acc(doc["answer"], pred)
|
| 110 |
+
model_response = {"sample_id": sample_id, "sub_task": doc["sub_task"], "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": pred, "score": score}
|
| 111 |
+
elif doc["question_type"] == "open-ended":
|
| 112 |
+
score = oe_rogue(doc["answer"], pred)
|
| 113 |
+
model_response = {"sample_id": sample_id, "sub_task": doc["sub_task"], "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": pred, "score": score}
|
| 114 |
+
else:
|
| 115 |
+
raise ValueError(f"Unknown question type: {doc['question_type']}")
|
| 116 |
+
|
| 117 |
+
return {
|
| 118 |
+
"overall_score": model_response,
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def mcq_acc(answer, pred):
|
| 123 |
+
periodStrip = re.compile("(?!<=\d)(\.)(?!\d)")
|
| 124 |
+
commaStrip = re.compile("(\d)(\,)(\d)")
|
| 125 |
+
punct = [";", r"/", "[", "]", '"', "{", "}", "(", ")", "=", "+", "\\", "_", "-", ">", "<", "@", "`", ",", "?", "!"]
|
| 126 |
+
|
| 127 |
+
def processPunctuation(inText):
|
| 128 |
+
outText = inText
|
| 129 |
+
for p in punct:
|
| 130 |
+
if (p + " " in inText or " " + p in inText) or (re.search(commaStrip, inText) != None):
|
| 131 |
+
outText = outText.replace(p, "")
|
| 132 |
+
else:
|
| 133 |
+
outText = outText.replace(p, " ")
|
| 134 |
+
outText = periodStrip.sub("", outText, re.UNICODE)
|
| 135 |
+
return outText
|
| 136 |
+
|
| 137 |
+
def process(answer):
|
| 138 |
+
option_regex = re.compile(r"^([A-E])\.\s*(.+)$", re.IGNORECASE)
|
| 139 |
+
match = option_regex.match(answer.strip())
|
| 140 |
+
|
| 141 |
+
if match:
|
| 142 |
+
# If matched, return the option letter in uppercase
|
| 143 |
+
return match.group(1).upper()
|
| 144 |
+
else:
|
| 145 |
+
# If no match, process the answer as before
|
| 146 |
+
answer = answer.replace("\n", " ")
|
| 147 |
+
answer = answer.replace("\t", " ")
|
| 148 |
+
answer = answer.strip()
|
| 149 |
+
answer = processPunctuation(answer)
|
| 150 |
+
answer = answer.strip("'")
|
| 151 |
+
answer = answer.strip('"')
|
| 152 |
+
answer = answer.strip(")")
|
| 153 |
+
answer = answer.strip("(")
|
| 154 |
+
answer = answer.strip().lower()
|
| 155 |
+
|
| 156 |
+
# Try to find any single letter (A-E) in the processed answer
|
| 157 |
+
letter_match = re.search(r"\b([A-E])\b", answer, re.IGNORECASE)
|
| 158 |
+
if letter_match:
|
| 159 |
+
return letter_match.group(1).upper()
|
| 160 |
+
|
| 161 |
+
return answer
|
| 162 |
+
|
| 163 |
+
pred = process(pred)
|
| 164 |
+
answer = process(answer)
|
| 165 |
+
|
| 166 |
+
if pred == answer:
|
| 167 |
+
score = 1
|
| 168 |
+
else:
|
| 169 |
+
score = 0
|
| 170 |
+
|
| 171 |
+
return score
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def oe_rogue(answer, pred):
|
| 175 |
+
rouge = Rouge()
|
| 176 |
+
if pred == "":
|
| 177 |
+
score = 0
|
| 178 |
+
else:
|
| 179 |
+
if len(pred) > 512:
|
| 180 |
+
pred = pred[:512]
|
| 181 |
+
score = rouge.get_scores(pred, answer)[0]["rouge-l"]["f"]
|
| 182 |
+
|
| 183 |
+
return score
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def overall_score(results):
|
| 187 |
+
categories = {
|
| 188 |
+
"Spot-the-Diff": spot_the_diff,
|
| 189 |
+
"Image-Edit": image_edit_instruct,
|
| 190 |
+
"Visual-Story-Telling": visual_story_telling,
|
| 191 |
+
"Visual-Cloze": visual_cloze,
|
| 192 |
+
"Text-Rich-VQA": text_rich_vqa,
|
| 193 |
+
"Multi-Image-VQA": multi_image_vqa,
|
| 194 |
+
"Puzzle": puzzle,
|
| 195 |
+
"NLVR2": nlrv2,
|
| 196 |
+
"QBench": qbench,
|
| 197 |
+
"ScanQA": scan_qa,
|
| 198 |
+
"ALFRED": alfred,
|
| 199 |
+
"nuscenes": nuscenes,
|
| 200 |
+
"ScanNet_chat": scannet_chat,
|
| 201 |
+
"ScanNet_task": scannet_task,
|
| 202 |
+
"BLINK": blink,
|
| 203 |
+
"MathVerse": math_verse,
|
| 204 |
+
"SciVerse": sci_verse,
|
| 205 |
+
"Mantis": mantis,
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
category_scores = {}
|
| 209 |
+
matched_subtasks = set()
|
| 210 |
+
|
| 211 |
+
eval_logger.info(f"Evaluation Sub-Task Results:")
|
| 212 |
+
for category, subtasks in categories.items():
|
| 213 |
+
score = 0
|
| 214 |
+
count = 0
|
| 215 |
+
for result in results:
|
| 216 |
+
if result["sub_task"] in subtasks:
|
| 217 |
+
count += 1
|
| 218 |
+
score += result["score"]
|
| 219 |
+
matched_subtasks.add(result["sub_task"])
|
| 220 |
+
if count > 0:
|
| 221 |
+
avg_score = score / count
|
| 222 |
+
category_scores[category] = avg_score
|
| 223 |
+
eval_logger.info(f"{category}: {avg_score:.3f}")
|
| 224 |
+
|
| 225 |
+
if not matched_subtasks:
|
| 226 |
+
raise ValueError("No subtasks were matched in the results. Check if the subtask names are correct.")
|
| 227 |
+
|
| 228 |
+
# Calculate overall score
|
| 229 |
+
total_score = sum(category_scores.values())
|
| 230 |
+
num_categories = len(category_scores)
|
| 231 |
+
overall_score = total_score / num_categories if num_categories > 0 else 0
|
| 232 |
+
|
| 233 |
+
return overall_score
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
# EVAL_PROMPT = """
|
| 237 |
+
# [Question]
|
| 238 |
+
# {question}
|
| 239 |
+
|
| 240 |
+
# [Assistant Response]
|
| 241 |
+
# {model_response}
|
| 242 |
+
|
| 243 |
+
# [Ground Truth Response]
|
| 244 |
+
# {ground_truth}
|
| 245 |
+
|
| 246 |
+
# [System]
|
| 247 |
+
# Rate whether the assistant response correctly matches the ground truth, it's about a question towards a sequence of images shared by the user.
|
| 248 |
+
# The rating should be 1-5, where 1 is incorrect and 5 is correct.
|
| 249 |
+
# Your response should be in the format:
|
| 250 |
+
# Explanation: (your explanation)
|
| 251 |
+
# Rating: (int)
|
| 252 |
+
# """
|
| 253 |
+
|
| 254 |
+
# NUM_SECONDS_TO_SLEEP = 5
|
| 255 |
+
# dir_path = os.path.dirname(os.path.realpath(__file__))
|
| 256 |
+
# with open(Path(__file__).parent / "_default_template_interleave_yaml", "r") as f:
|
| 257 |
+
# raw_data = f.readlines()
|
| 258 |
+
# safe_data = []
|
| 259 |
+
# for i, line in enumerate(raw_data):
|
| 260 |
+
# # remove function definition since yaml load cannot handle it
|
| 261 |
+
# if "!function" not in line:
|
| 262 |
+
# safe_data.append(line)
|
| 263 |
+
|
| 264 |
+
# config = yaml.safe_load("".join(safe_data))
|
| 265 |
+
|
| 266 |
+
# GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"]
|
| 267 |
+
# API_TYPE = config["metadata"]["api_type"]
|
| 268 |
+
|
| 269 |
+
# if API_TYPE == "openai":
|
| 270 |
+
# API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
|
| 271 |
+
# API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
|
| 272 |
+
# headers = {
|
| 273 |
+
# "Authorization": f"Bearer {API_KEY}",
|
| 274 |
+
# "Content-Type": "application/json",
|
| 275 |
+
# }
|
| 276 |
+
# elif API_TYPE == "azure":
|
| 277 |
+
# API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
|
| 278 |
+
# API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
|
| 279 |
+
# headers = {
|
| 280 |
+
# "api-key": API_KEY,
|
| 281 |
+
# "Content-Type": "application/json",
|
| 282 |
+
# }
|
| 283 |
+
# else:
|
| 284 |
+
# API_URL = ""
|
| 285 |
+
# API_KEY = ""
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
# def get_chat_response(prompt, max_retries=5, wait_time=10):
|
| 289 |
+
# headers = {
|
| 290 |
+
# "Authorization": f"Bearer {API_KEY}",
|
| 291 |
+
# "Content-Type": "application/json",
|
| 292 |
+
# }
|
| 293 |
+
|
| 294 |
+
# payload = {
|
| 295 |
+
# "model": GPT_EVAL_MODEL_NAME,
|
| 296 |
+
# "messages": [
|
| 297 |
+
# {
|
| 298 |
+
# "role": "user",
|
| 299 |
+
# "content": [
|
| 300 |
+
# {"type": "text", "text": prompt},
|
| 301 |
+
# ],
|
| 302 |
+
# }
|
| 303 |
+
# ],
|
| 304 |
+
# "max_tokens": 1024,
|
| 305 |
+
# "temperature": 0.0,
|
| 306 |
+
# }
|
| 307 |
+
|
| 308 |
+
# for attempt in range(max_retries):
|
| 309 |
+
# try:
|
| 310 |
+
# response = requests.post(API_URL, headers=headers, json=payload, timeout=60)
|
| 311 |
+
# response.raise_for_status()
|
| 312 |
+
# response_data = response.json()
|
| 313 |
+
# return response_data["choices"][0]["message"]["content"], GPT_EVAL_MODEL_NAME
|
| 314 |
+
# except requests.exceptions.RequestException as e:
|
| 315 |
+
# eval_logger.warning(f"Request failed on attempt {attempt+1}: {e}")
|
| 316 |
+
# time.sleep(wait_time)
|
| 317 |
+
# if attempt == max_retries - 1:
|
| 318 |
+
# eval_logger.error(f"Failed to get response after {max_retries} attempts")
|
| 319 |
+
# return "", GPT_EVAL_MODEL_NAME
|
| 320 |
+
# except Exception as e:
|
| 321 |
+
# eval_logger.error(f"Error on attempt {attempt+1}: {e}")
|
| 322 |
+
# return "", GPT_EVAL_MODEL_NAME
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
# def in_domain_oe_gpt_eval(results, args):
|
| 326 |
+
# total_score = 0
|
| 327 |
+
# available_count = 0
|
| 328 |
+
# for result in results:
|
| 329 |
+
# if result["question_type"] == "open-ended":
|
| 330 |
+
# question = result["question"]
|
| 331 |
+
# model_response = result["parsed_pred"]
|
| 332 |
+
# ground_truth = result["answer"]
|
| 333 |
+
# content = EVAL_PROMPT.format(question=question, model_response=model_response, ground_truth=ground_truth)
|
| 334 |
+
# result["gpt_eval_input"] = content
|
| 335 |
+
# model_output, model_name = get_chat_response(content)
|
| 336 |
+
# try:
|
| 337 |
+
# explanation = re.search(r"Explanation: (.*)\n", model_output).group(1)
|
| 338 |
+
# rating = re.search(r"Rating: (\d+)\n", model_output).group(1)
|
| 339 |
+
# result["gpt_eval_explanation"] = explanation
|
| 340 |
+
# result["gpt_eval_rating"] = rating
|
| 341 |
+
# result["gpt_eval_model_name"] = model_name
|
| 342 |
+
# except:
|
| 343 |
+
# eval_logger.error(f"Error on evaluating {result['sample_id']}. Results: {results}")
|
| 344 |
+
# result["gpt_eval_explanation"] = ""
|
| 345 |
+
# result["gpt_eval_rating"] = 0
|
| 346 |
+
# result["gpt_eval_model_name"] = model_name
|
| 347 |
+
|
| 348 |
+
# total_score += result["gpt_eval_rating"]
|
| 349 |
+
# available_count += 1
|
| 350 |
+
|
| 351 |
+
# elif result["question_type"] == "multi-choice":
|
| 352 |
+
# pass
|
| 353 |
+
|
| 354 |
+
# return (total_score / available_count) * 20.0 if available_count > 0 else 0
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
# # class MultiChoiceRegexFilter(ExtendedRegexFilter):
|
| 358 |
+
# # def __init__(self, *args, **kwargs):
|
| 359 |
+
# # super().__init__(*args, **kwargs)
|
| 360 |
+
|
| 361 |
+
# # def apply(self, resps, docs):
|
| 362 |
+
# # filtered_resps = []
|
| 363 |
+
|
| 364 |
+
# # for r, doc in zip(resps, docs):
|
| 365 |
+
# # # Regex to directly extract the option letter from the model response
|
| 366 |
+
# # option_letter_regex = re.compile(r"\b([A-Z])\.\s+([^\n]*)")
|
| 367 |
+
|
| 368 |
+
# # # Process each response
|
| 369 |
+
# # filtered = []
|
| 370 |
+
# # for resp in r:
|
| 371 |
+
# # # Try to match the option letter at the start of the response
|
| 372 |
+
# # match = option_letter_regex.match(resp)
|
| 373 |
+
# # if match:
|
| 374 |
+
# # # If a match is found, append the matched letter
|
| 375 |
+
# # filtered.append(match.group(1))
|
| 376 |
+
# # else:
|
| 377 |
+
# # # If no match, return the original response
|
| 378 |
+
# # filtered.append(resp)
|
| 379 |
+
|
| 380 |
+
# # # Assuming we need the first response that matches or the original response
|
| 381 |
+
# # filtered_resps.append(filtered[0])
|
| 382 |
+
|
| 383 |
+
# # return filtered_resps
|
LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Take in a YAML, and output all "other" splits with this YAML
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import logging
|
| 7 |
+
import os
|
| 8 |
+
|
| 9 |
+
import yaml
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
|
| 12 |
+
eval_logger = logging.getLogger("lm-eval")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
SUBJECTS = {
|
| 16 |
+
"abstract_algebra": "stem",
|
| 17 |
+
"anatomy": "stem",
|
| 18 |
+
"astronomy": "stem",
|
| 19 |
+
"business_ethics": "other",
|
| 20 |
+
"clinical_knowledge": "other",
|
| 21 |
+
"college_biology": "stem",
|
| 22 |
+
"college_chemistry": "stem",
|
| 23 |
+
"college_computer_science": "stem",
|
| 24 |
+
"college_mathematics": "stem",
|
| 25 |
+
"college_medicine": "other",
|
| 26 |
+
"college_physics": "stem",
|
| 27 |
+
"computer_security": "stem",
|
| 28 |
+
"conceptual_physics": "stem",
|
| 29 |
+
"econometrics": "social_sciences",
|
| 30 |
+
"electrical_engineering": "stem",
|
| 31 |
+
"elementary_mathematics": "stem",
|
| 32 |
+
"formal_logic": "humanities",
|
| 33 |
+
"global_facts": "other",
|
| 34 |
+
"high_school_biology": "stem",
|
| 35 |
+
"high_school_chemistry": "stem",
|
| 36 |
+
"high_school_computer_science": "stem",
|
| 37 |
+
"high_school_european_history": "humanities",
|
| 38 |
+
"high_school_geography": "social_sciences",
|
| 39 |
+
"high_school_government_and_politics": "social_sciences",
|
| 40 |
+
"high_school_macroeconomics": "social_sciences",
|
| 41 |
+
"high_school_mathematics": "stem",
|
| 42 |
+
"high_school_microeconomics": "social_sciences",
|
| 43 |
+
"high_school_physics": "stem",
|
| 44 |
+
"high_school_psychology": "social_sciences",
|
| 45 |
+
"high_school_statistics": "stem",
|
| 46 |
+
"high_school_us_history": "humanities",
|
| 47 |
+
"high_school_world_history": "humanities",
|
| 48 |
+
"human_aging": "other",
|
| 49 |
+
"human_sexuality": "social_sciences",
|
| 50 |
+
"international_law": "humanities",
|
| 51 |
+
"jurisprudence": "humanities",
|
| 52 |
+
"logical_fallacies": "humanities",
|
| 53 |
+
"machine_learning": "stem",
|
| 54 |
+
"management": "other",
|
| 55 |
+
"marketing": "other",
|
| 56 |
+
"medical_genetics": "other",
|
| 57 |
+
"miscellaneous": "other",
|
| 58 |
+
"moral_disputes": "humanities",
|
| 59 |
+
"moral_scenarios": "humanities",
|
| 60 |
+
"nutrition": "other",
|
| 61 |
+
"philosophy": "humanities",
|
| 62 |
+
"prehistory": "humanities",
|
| 63 |
+
"professional_accounting": "other",
|
| 64 |
+
"professional_law": "humanities",
|
| 65 |
+
"professional_medicine": "other",
|
| 66 |
+
"professional_psychology": "social_sciences",
|
| 67 |
+
"public_relations": "social_sciences",
|
| 68 |
+
"security_studies": "social_sciences",
|
| 69 |
+
"sociology": "social_sciences",
|
| 70 |
+
"us_foreign_policy": "social_sciences",
|
| 71 |
+
"virology": "other",
|
| 72 |
+
"world_religions": "humanities",
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def parse_args():
|
| 77 |
+
parser = argparse.ArgumentParser()
|
| 78 |
+
parser.add_argument("--base_yaml_path", required=True)
|
| 79 |
+
parser.add_argument("--save_prefix_path", default="mmlu")
|
| 80 |
+
parser.add_argument("--cot_prompt_path", default=None)
|
| 81 |
+
parser.add_argument("--task_prefix", default="")
|
| 82 |
+
parser.add_argument("--group_prefix", default="")
|
| 83 |
+
return parser.parse_args()
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
if __name__ == "__main__":
|
| 87 |
+
args = parse_args()
|
| 88 |
+
|
| 89 |
+
# get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
|
| 90 |
+
base_yaml_name = os.path.split(args.base_yaml_path)[-1]
|
| 91 |
+
with open(args.base_yaml_path, encoding="utf-8") as f:
|
| 92 |
+
base_yaml = yaml.full_load(f)
|
| 93 |
+
|
| 94 |
+
if args.cot_prompt_path is not None:
|
| 95 |
+
import json
|
| 96 |
+
|
| 97 |
+
with open(args.cot_prompt_path, encoding="utf-8") as f:
|
| 98 |
+
cot_file = json.load(f)
|
| 99 |
+
|
| 100 |
+
ALL_CATEGORIES = []
|
| 101 |
+
for subject, category in tqdm(SUBJECTS.items()):
|
| 102 |
+
if category not in ALL_CATEGORIES:
|
| 103 |
+
ALL_CATEGORIES.append(category)
|
| 104 |
+
|
| 105 |
+
if args.cot_prompt_path is not None:
|
| 106 |
+
description = cot_file[subject]
|
| 107 |
+
else:
|
| 108 |
+
description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
|
| 109 |
+
|
| 110 |
+
yaml_dict = {
|
| 111 |
+
"include": base_yaml_name,
|
| 112 |
+
"tag": f"mmlu_{args.task_prefix}_{category}" if args.task_prefix != "" else f"mmlu_{category}",
|
| 113 |
+
"task": f"mmlu_{args.task_prefix}_{subject}" if args.task_prefix != "" else f"mmlu_{subject}",
|
| 114 |
+
"task_alias": subject.replace("_", " "),
|
| 115 |
+
"dataset_name": subject,
|
| 116 |
+
"description": description,
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
file_save_path = args.save_prefix_path + f"_{subject}.yaml"
|
| 120 |
+
eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}")
|
| 121 |
+
with open(file_save_path, "w", encoding="utf-8") as yaml_file:
|
| 122 |
+
yaml.dump(
|
| 123 |
+
yaml_dict,
|
| 124 |
+
yaml_file,
|
| 125 |
+
allow_unicode=True,
|
| 126 |
+
default_style='"',
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
if args.task_prefix != "":
|
| 130 |
+
mmlu_subcategories = [f"mmlu_{args.task_prefix}_{category}" for category in ALL_CATEGORIES]
|
| 131 |
+
else:
|
| 132 |
+
mmlu_subcategories = [f"mmlu_{category}" for category in ALL_CATEGORIES]
|
| 133 |
+
|
| 134 |
+
if args.group_prefix != "":
|
| 135 |
+
file_save_path = args.group_prefix + ".yaml"
|
| 136 |
+
else:
|
| 137 |
+
file_save_path = args.save_prefix_path + ".yaml"
|
| 138 |
+
|
| 139 |
+
eval_logger.info(f"Saving benchmark config to {file_save_path}")
|
| 140 |
+
with open(file_save_path, "w", encoding="utf-8") as yaml_file:
|
| 141 |
+
yaml.dump(
|
| 142 |
+
{
|
| 143 |
+
"group": f"mmlu_{args.task_prefix}" if args.task_prefix != "" else "mmlu",
|
| 144 |
+
"task": mmlu_subcategories,
|
| 145 |
+
},
|
| 146 |
+
yaml_file,
|
| 147 |
+
indent=4,
|
| 148 |
+
default_flow_style=False,
|
| 149 |
+
)
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
|
| 2 |
+
test_split: test
|
| 3 |
+
fewshot_split: dev
|
| 4 |
+
fewshot_config:
|
| 5 |
+
sampler: first_n
|
| 6 |
+
output_type: generate_until
|
| 7 |
+
doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
|
| 8 |
+
doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}"
|
| 9 |
+
generation_kwargs:
|
| 10 |
+
until:
|
| 11 |
+
- "</s>"
|
| 12 |
+
- "\n"
|
| 13 |
+
metric_list:
|
| 14 |
+
- metric: exact_match
|
| 15 |
+
aggregation: mean
|
| 16 |
+
higher_is_better: true
|
| 17 |
+
metadata:
|
| 18 |
+
version: 2.0
|
| 19 |
+
dataset_kwargs:
|
| 20 |
+
trust_remote_code: true
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
group: mmlu_generative
|
| 2 |
+
group_alias: mmlu (generative)
|
| 3 |
+
task:
|
| 4 |
+
- group: stem
|
| 5 |
+
task:
|
| 6 |
+
- mmlu_stem_generative
|
| 7 |
+
aggregate_metric_list:
|
| 8 |
+
- metric: acc
|
| 9 |
+
weight_by_size: True
|
| 10 |
+
- group: other
|
| 11 |
+
task:
|
| 12 |
+
- mmlu_other_generative
|
| 13 |
+
aggregate_metric_list:
|
| 14 |
+
- metric: acc
|
| 15 |
+
weight_by_size: True
|
| 16 |
+
- group: social sciences
|
| 17 |
+
task:
|
| 18 |
+
- mmlu_social_sciences_generative
|
| 19 |
+
aggregate_metric_list:
|
| 20 |
+
- metric: acc
|
| 21 |
+
weight_by_size: True
|
| 22 |
+
- group: humanities
|
| 23 |
+
task:
|
| 24 |
+
- mmlu_humanities_generative
|
| 25 |
+
aggregate_metric_list:
|
| 26 |
+
- metric: acc
|
| 27 |
+
weight_by_size: True
|
| 28 |
+
aggregate_metric_list:
|
| 29 |
+
- metric: acc
|
| 30 |
+
weight_by_size: True
|
| 31 |
+
metadata:
|
| 32 |
+
version: 2
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "abstract_algebra"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about abstract\
|
| 3 |
+
\ algebra.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_abstract_algebra_generative"
|
| 7 |
+
"task_alias": "abstract_algebra"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "astronomy"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about astronomy.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_astronomy_generative"
|
| 7 |
+
"task_alias": "astronomy"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "business_ethics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about business\
|
| 3 |
+
\ ethics.\n\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_business_ethics_generative"
|
| 7 |
+
"task_alias": "business_ethics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "clinical_knowledge"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about clinical\
|
| 3 |
+
\ knowledge.\n\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_clinical_knowledge_generative"
|
| 7 |
+
"task_alias": "clinical_knowledge"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "college_chemistry"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about college\
|
| 3 |
+
\ chemistry.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_college_chemistry_generative"
|
| 7 |
+
"task_alias": "college_chemistry"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "college_mathematics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about college\
|
| 3 |
+
\ mathematics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_college_mathematics_generative"
|
| 7 |
+
"task_alias": "college_mathematics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "college_physics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about college\
|
| 3 |
+
\ physics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_college_physics_generative"
|
| 7 |
+
"task_alias": "college_physics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "computer_security"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about computer\
|
| 3 |
+
\ security.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_computer_security_generative"
|
| 7 |
+
"task_alias": "computer_security"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "conceptual_physics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about conceptual\
|
| 3 |
+
\ physics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_conceptual_physics_generative"
|
| 7 |
+
"task_alias": "conceptual_physics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "econometrics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about econometrics.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_social_sciences_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_econometrics_generative"
|
| 7 |
+
"task_alias": "econometrics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "electrical_engineering"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about electrical\
|
| 3 |
+
\ engineering.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_electrical_engineering_generative"
|
| 7 |
+
"task_alias": "electrical_engineering"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "elementary_mathematics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about elementary\
|
| 3 |
+
\ mathematics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_elementary_mathematics_generative"
|
| 7 |
+
"task_alias": "elementary_mathematics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "formal_logic"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about formal\
|
| 3 |
+
\ logic.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_formal_logic_generative"
|
| 7 |
+
"task_alias": "formal_logic"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "global_facts"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about global\
|
| 3 |
+
\ facts.\n\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_global_facts_generative"
|
| 7 |
+
"task_alias": "global_facts"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_biology"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school biology.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_biology_generative"
|
| 7 |
+
"task_alias": "high_school_biology"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_chemistry"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school chemistry.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_chemistry_generative"
|
| 7 |
+
"task_alias": "high_school_chemistry"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_computer_science"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school computer science.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_computer_science_generative"
|
| 7 |
+
"task_alias": "high_school_computer_science"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_european_history"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school european history.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_european_history_generative"
|
| 7 |
+
"task_alias": "high_school_european_history"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_geography"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school geography.\n\n"
|
| 4 |
+
"tag": "mmlu_social_sciences_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_geography_generative"
|
| 7 |
+
"task_alias": "high_school_geography"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_macroeconomics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school macroeconomics.\n\n"
|
| 4 |
+
"tag": "mmlu_social_sciences_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_macroeconomics_generative"
|
| 7 |
+
"task_alias": "high_school_macroeconomics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_mathematics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school mathematics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_mathematics_generative"
|
| 7 |
+
"task_alias": "high_school_mathematics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_microeconomics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school microeconomics.\n\n"
|
| 4 |
+
"tag": "mmlu_social_sciences_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_microeconomics_generative"
|
| 7 |
+
"task_alias": "high_school_microeconomics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_physics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school physics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_physics_generative"
|
| 7 |
+
"task_alias": "high_school_physics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_psychology"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school psychology.\n\n"
|
| 4 |
+
"tag": "mmlu_social_sciences_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_psychology_generative"
|
| 7 |
+
"task_alias": "high_school_psychology"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_statistics"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school statistics.\n\n"
|
| 4 |
+
"tag": "mmlu_stem_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_statistics_generative"
|
| 7 |
+
"task_alias": "high_school_statistics"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_us_history"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school us history.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_us_history_generative"
|
| 7 |
+
"task_alias": "high_school_us_history"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "high_school_world_history"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about high\
|
| 3 |
+
\ school world history.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_high_school_world_history_generative"
|
| 7 |
+
"task_alias": "high_school_world_history"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "human_aging"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about human\
|
| 3 |
+
\ aging.\n\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_human_aging_generative"
|
| 7 |
+
"task_alias": "human_aging"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "human_sexuality"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about human\
|
| 3 |
+
\ sexuality.\n\n"
|
| 4 |
+
"tag": "mmlu_social_sciences_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_human_sexuality_generative"
|
| 7 |
+
"task_alias": "human_sexuality"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "international_law"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about international\
|
| 3 |
+
\ law.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_international_law_generative"
|
| 7 |
+
"task_alias": "international_law"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "jurisprudence"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_jurisprudence_generative"
|
| 7 |
+
"task_alias": "jurisprudence"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "logical_fallacies"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about logical\
|
| 3 |
+
\ fallacies.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_logical_fallacies_generative"
|
| 7 |
+
"task_alias": "logical_fallacies"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "management"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about management.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_management_generative"
|
| 7 |
+
"task_alias": "management"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "marketing"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about marketing.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_marketing_generative"
|
| 7 |
+
"task_alias": "marketing"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "miscellaneous"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_miscellaneous_generative"
|
| 7 |
+
"task_alias": "miscellaneous"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "moral_disputes"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about moral\
|
| 3 |
+
\ disputes.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_moral_disputes_generative"
|
| 7 |
+
"task_alias": "moral_disputes"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "moral_scenarios"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about moral\
|
| 3 |
+
\ scenarios.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_moral_scenarios_generative"
|
| 7 |
+
"task_alias": "moral_scenarios"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "philosophy"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about philosophy.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_philosophy_generative"
|
| 7 |
+
"task_alias": "philosophy"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "prehistory"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about prehistory.\n\
|
| 3 |
+
\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_prehistory_generative"
|
| 7 |
+
"task_alias": "prehistory"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "professional_accounting"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about professional\
|
| 3 |
+
\ accounting.\n\n"
|
| 4 |
+
"tag": "mmlu_other_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_professional_accounting_generative"
|
| 7 |
+
"task_alias": "professional_accounting"
|
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"dataset_name": "professional_law"
|
| 2 |
+
"description": "The following are multiple choice questions (with answers) about professional\
|
| 3 |
+
\ law.\n\n"
|
| 4 |
+
"tag": "mmlu_humanities_generative"
|
| 5 |
+
"include": "_default_template_yaml"
|
| 6 |
+
"task": "mmlu_professional_law_generative"
|
| 7 |
+
"task_alias": "professional_law"
|