GuoruiSong commited on
Commit
dd71079
·
verified ·
1 Parent(s): 6f3656a

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml +9 -0
  2. LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml +26 -0
  3. LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml +5 -0
  4. LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml +26 -0
  5. LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml +26 -0
  6. LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py +383 -0
  7. LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py +149 -0
  8. LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml +20 -0
  9. LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml +32 -0
  10. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml +7 -0
  11. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml +7 -0
  12. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml +7 -0
  13. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml +7 -0
  14. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml +7 -0
  15. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml +7 -0
  16. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml +7 -0
  17. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml +7 -0
  18. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml +7 -0
  19. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml +7 -0
  20. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml +7 -0
  21. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml +7 -0
  22. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml +7 -0
  23. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml +7 -0
  24. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml +7 -0
  25. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml +7 -0
  26. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml +7 -0
  27. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml +7 -0
  28. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml +7 -0
  29. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml +7 -0
  30. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml +7 -0
  31. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml +7 -0
  32. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml +7 -0
  33. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml +7 -0
  34. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml +7 -0
  35. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml +7 -0
  36. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml +7 -0
  37. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml +7 -0
  38. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml +7 -0
  39. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml +7 -0
  40. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml +7 -0
  41. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml +7 -0
  42. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml +7 -0
  43. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml +7 -0
  44. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml +7 -0
  45. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml +7 -0
  46. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml +7 -0
  47. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml +7 -0
  48. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml +7 -0
  49. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml +7 -0
  50. LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml +7 -0
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ output_type: generate_until
2
+ generation_kwargs:
3
+ until:
4
+ - "ASSISTANT:"
5
+ image_aspect_ratio: pad
6
+ metadata:
7
+ version: 0.0
8
+ api_type : openai
9
+ gpt_eval_model_name: "gpt-3.5-turbo"
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench
2
+ dataset_name: in_domain
3
+ dataset_kwargs:
4
+ token: True
5
+ task: "llava_interleave_bench_in_domain"
6
+ test_split: test
7
+ doc_to_target: "answer"
8
+ doc_to_visual: !function utils.doc_to_visual
9
+ doc_to_text: !function utils.doc_to_text
10
+ process_results: !function utils.interleave_process_results
11
+
12
+ metric_list:
13
+ - metric: overall_score
14
+ aggregation: !function utils.overall_score
15
+ higher_is_better: true
16
+
17
+ generation_kwargs:
18
+ max_new_tokens: 16
19
+ temperature: 0
20
+ do_sample: False
21
+ image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy.
22
+
23
+ lmms_eval_specific_kwargs:
24
+ default:
25
+ oe_post_prompt: ""
26
+ mcq_post_prompt: ""
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ group: llava_interleave_bench
2
+ task:
3
+ - llava_interleave_bench_in_domain
4
+ - llava_interleave_bench_out_domain
5
+ - llava_interleave_bench_multi_view
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench
2
+ dataset_name: multi_view_in_domain
3
+ dataset_kwargs:
4
+ token: True
5
+ task: "llava_interleave_bench_multi_view"
6
+ test_split: test
7
+ doc_to_target: "answer"
8
+ doc_to_visual: !function utils.doc_to_visual
9
+ doc_to_text: !function utils.doc_to_text_conversation
10
+ process_results: !function utils.interleave_process_results
11
+
12
+ metric_list:
13
+ - metric: overall_score
14
+ aggregation: !function utils.overall_score
15
+ higher_is_better: true
16
+
17
+ generation_kwargs:
18
+ max_new_tokens: 16
19
+ temperature: 0
20
+ do_sample: False
21
+ image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy.
22
+
23
+ lmms_eval_specific_kwargs:
24
+ default:
25
+ oe_post_prompt: ""
26
+ mcq_post_prompt: ""
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench
2
+ dataset_name: out_of_domain
3
+ dataset_kwargs:
4
+ token: True
5
+ task: "llava_interleave_bench_out_domain"
6
+ test_split: test
7
+ doc_to_target: "answer"
8
+ doc_to_visual: !function utils.doc_to_visual
9
+ doc_to_text: !function utils.doc_to_text
10
+ process_results: !function utils.interleave_process_results
11
+
12
+ metric_list:
13
+ - metric: overall_score
14
+ aggregation: !function utils.overall_score
15
+ higher_is_better: true
16
+
17
+ generation_kwargs:
18
+ max_new_tokens: 16
19
+ temperature: 0
20
+ do_sample: False
21
+ image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy.
22
+
23
+ lmms_eval_specific_kwargs:
24
+ default:
25
+ oe_post_prompt: ""
26
+ mcq_post_prompt: ""
LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import re
4
+ import time
5
+ from collections import defaultdict
6
+ from pathlib import Path
7
+
8
+ import requests
9
+ import yaml
10
+ from loguru import logger as eval_logger
11
+ from PIL import Image
12
+ from rouge import Rouge
13
+
14
+ spot_the_diff = ["Spot-the-Diff", "Birds-to-Words", "CLEVR-Change"]
15
+ image_edit_instruct = ["IEdit", "HQ-Edit", "MagicBrush"]
16
+ visual_story_telling = ["AESOP", "FlintstonesSV", "PororoSV", "VIST"]
17
+ visual_cloze = ["COMICS_Dialogue", "RecipeQA_VisualCloze"]
18
+ text_rich_vqa = ["WebQA", "TQA", "OCR-VQA", "DocVQA"]
19
+ multi_image_vqa = ["MIT-States_StateCoherence", "MIT-States_PropertyCoherence", "VISION", "RecipeQA_ImageCoherence"]
20
+
21
+ puzzle = ["RAVEN"]
22
+ nlrv2 = ["NLVR2_Mantis"]
23
+ qbench = ["QBench"]
24
+
25
+ scan_qa = ["ScanQA"]
26
+ alfred = ["ALFRED"]
27
+ nuscenes = ["nuscenes"]
28
+ scannet_chat = ["ScanNet_chat"]
29
+ scannet_task = ["ScanNet_task"]
30
+ blink = ["BLINK"]
31
+ math_verse = ["MathVerse"]
32
+ sci_verse = ["SciVerse"]
33
+ mantis = ["Mantis"]
34
+
35
+
36
+ def doc_to_visual(doc):
37
+ max_visual_count = 16
38
+ visuals = []
39
+ for i in range(max_visual_count):
40
+ if f"image_{i}" in doc:
41
+ image = doc[f"image_{i}"]
42
+ if image is None:
43
+ continue # Skip this image if it's None
44
+ if isinstance(image, Image.Image):
45
+ visuals.append(image.copy().convert("RGB"))
46
+ else:
47
+ try:
48
+ # If the image is not already a PIL Image, try to open it
49
+ visuals.append(Image.open(image).convert("RGB"))
50
+ except Exception as e:
51
+ print(f"Error opening image_{i}: {e}")
52
+ # Optionally, you can add a placeholder image or just continue
53
+ continue
54
+
55
+ return visuals
56
+
57
+
58
+ # This is the place where you format your question
59
+ def doc_to_text(doc, lmms_eval_specific_kwargs=None):
60
+ if lmms_eval_specific_kwargs is None:
61
+ lmms_eval_specific_kwargs = {}
62
+
63
+ oe_post_prompt = ""
64
+ if "oe_post_prompt" in lmms_eval_specific_kwargs:
65
+ oe_post_prompt = lmms_eval_specific_kwargs["oe_post_prompt"]
66
+
67
+ mcq_post_prompt = ""
68
+ if "mcq_post_prompt" in lmms_eval_specific_kwargs:
69
+ mcq_post_prompt = lmms_eval_specific_kwargs["mcq_post_prompt"]
70
+
71
+ user_prompt = doc["question"]
72
+
73
+ if mcq_post_prompt != "" and doc["question_type"] == "multi-choice":
74
+ user_prompt = user_prompt.split("Your answer is:")[0].split("\n")[0].strip()
75
+ user_prompt = f"{user_prompt}\n{mcq_post_prompt}"
76
+
77
+ if oe_post_prompt != "" and doc["question_type"] == "open-ended":
78
+ user_prompt = f"{user_prompt}\n{oe_post_prompt}"
79
+
80
+ return user_prompt
81
+
82
+
83
+ def doc_to_text_conversation(doc, lmms_eval_specific_kwargs=None):
84
+ if lmms_eval_specific_kwargs is None:
85
+ lmms_eval_specific_kwargs = {}
86
+
87
+ conversations = doc["conversations"]
88
+
89
+ if isinstance(conversations, list):
90
+ user_prompt = json.dumps(conversations)
91
+ else:
92
+ user_prompt = conversations
93
+
94
+ return user_prompt
95
+
96
+
97
+ def doc_to_text_multi_turn(doc, lmms_eval_specific_kwargs=None):
98
+ if lmms_eval_specific_kwargs is None:
99
+ lmms_eval_specific_kwargs = {}
100
+
101
+ return doc["conversations"]
102
+
103
+
104
+ def interleave_process_results(doc, results):
105
+ pred = results[0]
106
+ sample_id = doc["sample_id"]
107
+
108
+ if doc["question_type"] == "multi-choice":
109
+ score = mcq_acc(doc["answer"], pred)
110
+ model_response = {"sample_id": sample_id, "sub_task": doc["sub_task"], "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": pred, "score": score}
111
+ elif doc["question_type"] == "open-ended":
112
+ score = oe_rogue(doc["answer"], pred)
113
+ model_response = {"sample_id": sample_id, "sub_task": doc["sub_task"], "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": pred, "score": score}
114
+ else:
115
+ raise ValueError(f"Unknown question type: {doc['question_type']}")
116
+
117
+ return {
118
+ "overall_score": model_response,
119
+ }
120
+
121
+
122
+ def mcq_acc(answer, pred):
123
+ periodStrip = re.compile("(?!<=\d)(\.)(?!\d)")
124
+ commaStrip = re.compile("(\d)(\,)(\d)")
125
+ punct = [";", r"/", "[", "]", '"', "{", "}", "(", ")", "=", "+", "\\", "_", "-", ">", "<", "@", "`", ",", "?", "!"]
126
+
127
+ def processPunctuation(inText):
128
+ outText = inText
129
+ for p in punct:
130
+ if (p + " " in inText or " " + p in inText) or (re.search(commaStrip, inText) != None):
131
+ outText = outText.replace(p, "")
132
+ else:
133
+ outText = outText.replace(p, " ")
134
+ outText = periodStrip.sub("", outText, re.UNICODE)
135
+ return outText
136
+
137
+ def process(answer):
138
+ option_regex = re.compile(r"^([A-E])\.\s*(.+)$", re.IGNORECASE)
139
+ match = option_regex.match(answer.strip())
140
+
141
+ if match:
142
+ # If matched, return the option letter in uppercase
143
+ return match.group(1).upper()
144
+ else:
145
+ # If no match, process the answer as before
146
+ answer = answer.replace("\n", " ")
147
+ answer = answer.replace("\t", " ")
148
+ answer = answer.strip()
149
+ answer = processPunctuation(answer)
150
+ answer = answer.strip("'")
151
+ answer = answer.strip('"')
152
+ answer = answer.strip(")")
153
+ answer = answer.strip("(")
154
+ answer = answer.strip().lower()
155
+
156
+ # Try to find any single letter (A-E) in the processed answer
157
+ letter_match = re.search(r"\b([A-E])\b", answer, re.IGNORECASE)
158
+ if letter_match:
159
+ return letter_match.group(1).upper()
160
+
161
+ return answer
162
+
163
+ pred = process(pred)
164
+ answer = process(answer)
165
+
166
+ if pred == answer:
167
+ score = 1
168
+ else:
169
+ score = 0
170
+
171
+ return score
172
+
173
+
174
+ def oe_rogue(answer, pred):
175
+ rouge = Rouge()
176
+ if pred == "":
177
+ score = 0
178
+ else:
179
+ if len(pred) > 512:
180
+ pred = pred[:512]
181
+ score = rouge.get_scores(pred, answer)[0]["rouge-l"]["f"]
182
+
183
+ return score
184
+
185
+
186
+ def overall_score(results):
187
+ categories = {
188
+ "Spot-the-Diff": spot_the_diff,
189
+ "Image-Edit": image_edit_instruct,
190
+ "Visual-Story-Telling": visual_story_telling,
191
+ "Visual-Cloze": visual_cloze,
192
+ "Text-Rich-VQA": text_rich_vqa,
193
+ "Multi-Image-VQA": multi_image_vqa,
194
+ "Puzzle": puzzle,
195
+ "NLVR2": nlrv2,
196
+ "QBench": qbench,
197
+ "ScanQA": scan_qa,
198
+ "ALFRED": alfred,
199
+ "nuscenes": nuscenes,
200
+ "ScanNet_chat": scannet_chat,
201
+ "ScanNet_task": scannet_task,
202
+ "BLINK": blink,
203
+ "MathVerse": math_verse,
204
+ "SciVerse": sci_verse,
205
+ "Mantis": mantis,
206
+ }
207
+
208
+ category_scores = {}
209
+ matched_subtasks = set()
210
+
211
+ eval_logger.info(f"Evaluation Sub-Task Results:")
212
+ for category, subtasks in categories.items():
213
+ score = 0
214
+ count = 0
215
+ for result in results:
216
+ if result["sub_task"] in subtasks:
217
+ count += 1
218
+ score += result["score"]
219
+ matched_subtasks.add(result["sub_task"])
220
+ if count > 0:
221
+ avg_score = score / count
222
+ category_scores[category] = avg_score
223
+ eval_logger.info(f"{category}: {avg_score:.3f}")
224
+
225
+ if not matched_subtasks:
226
+ raise ValueError("No subtasks were matched in the results. Check if the subtask names are correct.")
227
+
228
+ # Calculate overall score
229
+ total_score = sum(category_scores.values())
230
+ num_categories = len(category_scores)
231
+ overall_score = total_score / num_categories if num_categories > 0 else 0
232
+
233
+ return overall_score
234
+
235
+
236
+ # EVAL_PROMPT = """
237
+ # [Question]
238
+ # {question}
239
+
240
+ # [Assistant Response]
241
+ # {model_response}
242
+
243
+ # [Ground Truth Response]
244
+ # {ground_truth}
245
+
246
+ # [System]
247
+ # Rate whether the assistant response correctly matches the ground truth, it's about a question towards a sequence of images shared by the user.
248
+ # The rating should be 1-5, where 1 is incorrect and 5 is correct.
249
+ # Your response should be in the format:
250
+ # Explanation: (your explanation)
251
+ # Rating: (int)
252
+ # """
253
+
254
+ # NUM_SECONDS_TO_SLEEP = 5
255
+ # dir_path = os.path.dirname(os.path.realpath(__file__))
256
+ # with open(Path(__file__).parent / "_default_template_interleave_yaml", "r") as f:
257
+ # raw_data = f.readlines()
258
+ # safe_data = []
259
+ # for i, line in enumerate(raw_data):
260
+ # # remove function definition since yaml load cannot handle it
261
+ # if "!function" not in line:
262
+ # safe_data.append(line)
263
+
264
+ # config = yaml.safe_load("".join(safe_data))
265
+
266
+ # GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"]
267
+ # API_TYPE = config["metadata"]["api_type"]
268
+
269
+ # if API_TYPE == "openai":
270
+ # API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
271
+ # API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
272
+ # headers = {
273
+ # "Authorization": f"Bearer {API_KEY}",
274
+ # "Content-Type": "application/json",
275
+ # }
276
+ # elif API_TYPE == "azure":
277
+ # API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
278
+ # API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
279
+ # headers = {
280
+ # "api-key": API_KEY,
281
+ # "Content-Type": "application/json",
282
+ # }
283
+ # else:
284
+ # API_URL = ""
285
+ # API_KEY = ""
286
+
287
+
288
+ # def get_chat_response(prompt, max_retries=5, wait_time=10):
289
+ # headers = {
290
+ # "Authorization": f"Bearer {API_KEY}",
291
+ # "Content-Type": "application/json",
292
+ # }
293
+
294
+ # payload = {
295
+ # "model": GPT_EVAL_MODEL_NAME,
296
+ # "messages": [
297
+ # {
298
+ # "role": "user",
299
+ # "content": [
300
+ # {"type": "text", "text": prompt},
301
+ # ],
302
+ # }
303
+ # ],
304
+ # "max_tokens": 1024,
305
+ # "temperature": 0.0,
306
+ # }
307
+
308
+ # for attempt in range(max_retries):
309
+ # try:
310
+ # response = requests.post(API_URL, headers=headers, json=payload, timeout=60)
311
+ # response.raise_for_status()
312
+ # response_data = response.json()
313
+ # return response_data["choices"][0]["message"]["content"], GPT_EVAL_MODEL_NAME
314
+ # except requests.exceptions.RequestException as e:
315
+ # eval_logger.warning(f"Request failed on attempt {attempt+1}: {e}")
316
+ # time.sleep(wait_time)
317
+ # if attempt == max_retries - 1:
318
+ # eval_logger.error(f"Failed to get response after {max_retries} attempts")
319
+ # return "", GPT_EVAL_MODEL_NAME
320
+ # except Exception as e:
321
+ # eval_logger.error(f"Error on attempt {attempt+1}: {e}")
322
+ # return "", GPT_EVAL_MODEL_NAME
323
+
324
+
325
+ # def in_domain_oe_gpt_eval(results, args):
326
+ # total_score = 0
327
+ # available_count = 0
328
+ # for result in results:
329
+ # if result["question_type"] == "open-ended":
330
+ # question = result["question"]
331
+ # model_response = result["parsed_pred"]
332
+ # ground_truth = result["answer"]
333
+ # content = EVAL_PROMPT.format(question=question, model_response=model_response, ground_truth=ground_truth)
334
+ # result["gpt_eval_input"] = content
335
+ # model_output, model_name = get_chat_response(content)
336
+ # try:
337
+ # explanation = re.search(r"Explanation: (.*)\n", model_output).group(1)
338
+ # rating = re.search(r"Rating: (\d+)\n", model_output).group(1)
339
+ # result["gpt_eval_explanation"] = explanation
340
+ # result["gpt_eval_rating"] = rating
341
+ # result["gpt_eval_model_name"] = model_name
342
+ # except:
343
+ # eval_logger.error(f"Error on evaluating {result['sample_id']}. Results: {results}")
344
+ # result["gpt_eval_explanation"] = ""
345
+ # result["gpt_eval_rating"] = 0
346
+ # result["gpt_eval_model_name"] = model_name
347
+
348
+ # total_score += result["gpt_eval_rating"]
349
+ # available_count += 1
350
+
351
+ # elif result["question_type"] == "multi-choice":
352
+ # pass
353
+
354
+ # return (total_score / available_count) * 20.0 if available_count > 0 else 0
355
+
356
+
357
+ # # class MultiChoiceRegexFilter(ExtendedRegexFilter):
358
+ # # def __init__(self, *args, **kwargs):
359
+ # # super().__init__(*args, **kwargs)
360
+
361
+ # # def apply(self, resps, docs):
362
+ # # filtered_resps = []
363
+
364
+ # # for r, doc in zip(resps, docs):
365
+ # # # Regex to directly extract the option letter from the model response
366
+ # # option_letter_regex = re.compile(r"\b([A-Z])\.\s+([^\n]*)")
367
+
368
+ # # # Process each response
369
+ # # filtered = []
370
+ # # for resp in r:
371
+ # # # Try to match the option letter at the start of the response
372
+ # # match = option_letter_regex.match(resp)
373
+ # # if match:
374
+ # # # If a match is found, append the matched letter
375
+ # # filtered.append(match.group(1))
376
+ # # else:
377
+ # # # If no match, return the original response
378
+ # # filtered.append(resp)
379
+
380
+ # # # Assuming we need the first response that matches or the original response
381
+ # # filtered_resps.append(filtered[0])
382
+
383
+ # # return filtered_resps
LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Take in a YAML, and output all "other" splits with this YAML
3
+ """
4
+
5
+ import argparse
6
+ import logging
7
+ import os
8
+
9
+ import yaml
10
+ from tqdm import tqdm
11
+
12
+ eval_logger = logging.getLogger("lm-eval")
13
+
14
+
15
+ SUBJECTS = {
16
+ "abstract_algebra": "stem",
17
+ "anatomy": "stem",
18
+ "astronomy": "stem",
19
+ "business_ethics": "other",
20
+ "clinical_knowledge": "other",
21
+ "college_biology": "stem",
22
+ "college_chemistry": "stem",
23
+ "college_computer_science": "stem",
24
+ "college_mathematics": "stem",
25
+ "college_medicine": "other",
26
+ "college_physics": "stem",
27
+ "computer_security": "stem",
28
+ "conceptual_physics": "stem",
29
+ "econometrics": "social_sciences",
30
+ "electrical_engineering": "stem",
31
+ "elementary_mathematics": "stem",
32
+ "formal_logic": "humanities",
33
+ "global_facts": "other",
34
+ "high_school_biology": "stem",
35
+ "high_school_chemistry": "stem",
36
+ "high_school_computer_science": "stem",
37
+ "high_school_european_history": "humanities",
38
+ "high_school_geography": "social_sciences",
39
+ "high_school_government_and_politics": "social_sciences",
40
+ "high_school_macroeconomics": "social_sciences",
41
+ "high_school_mathematics": "stem",
42
+ "high_school_microeconomics": "social_sciences",
43
+ "high_school_physics": "stem",
44
+ "high_school_psychology": "social_sciences",
45
+ "high_school_statistics": "stem",
46
+ "high_school_us_history": "humanities",
47
+ "high_school_world_history": "humanities",
48
+ "human_aging": "other",
49
+ "human_sexuality": "social_sciences",
50
+ "international_law": "humanities",
51
+ "jurisprudence": "humanities",
52
+ "logical_fallacies": "humanities",
53
+ "machine_learning": "stem",
54
+ "management": "other",
55
+ "marketing": "other",
56
+ "medical_genetics": "other",
57
+ "miscellaneous": "other",
58
+ "moral_disputes": "humanities",
59
+ "moral_scenarios": "humanities",
60
+ "nutrition": "other",
61
+ "philosophy": "humanities",
62
+ "prehistory": "humanities",
63
+ "professional_accounting": "other",
64
+ "professional_law": "humanities",
65
+ "professional_medicine": "other",
66
+ "professional_psychology": "social_sciences",
67
+ "public_relations": "social_sciences",
68
+ "security_studies": "social_sciences",
69
+ "sociology": "social_sciences",
70
+ "us_foreign_policy": "social_sciences",
71
+ "virology": "other",
72
+ "world_religions": "humanities",
73
+ }
74
+
75
+
76
+ def parse_args():
77
+ parser = argparse.ArgumentParser()
78
+ parser.add_argument("--base_yaml_path", required=True)
79
+ parser.add_argument("--save_prefix_path", default="mmlu")
80
+ parser.add_argument("--cot_prompt_path", default=None)
81
+ parser.add_argument("--task_prefix", default="")
82
+ parser.add_argument("--group_prefix", default="")
83
+ return parser.parse_args()
84
+
85
+
86
+ if __name__ == "__main__":
87
+ args = parse_args()
88
+
89
+ # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
90
+ base_yaml_name = os.path.split(args.base_yaml_path)[-1]
91
+ with open(args.base_yaml_path, encoding="utf-8") as f:
92
+ base_yaml = yaml.full_load(f)
93
+
94
+ if args.cot_prompt_path is not None:
95
+ import json
96
+
97
+ with open(args.cot_prompt_path, encoding="utf-8") as f:
98
+ cot_file = json.load(f)
99
+
100
+ ALL_CATEGORIES = []
101
+ for subject, category in tqdm(SUBJECTS.items()):
102
+ if category not in ALL_CATEGORIES:
103
+ ALL_CATEGORIES.append(category)
104
+
105
+ if args.cot_prompt_path is not None:
106
+ description = cot_file[subject]
107
+ else:
108
+ description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
109
+
110
+ yaml_dict = {
111
+ "include": base_yaml_name,
112
+ "tag": f"mmlu_{args.task_prefix}_{category}" if args.task_prefix != "" else f"mmlu_{category}",
113
+ "task": f"mmlu_{args.task_prefix}_{subject}" if args.task_prefix != "" else f"mmlu_{subject}",
114
+ "task_alias": subject.replace("_", " "),
115
+ "dataset_name": subject,
116
+ "description": description,
117
+ }
118
+
119
+ file_save_path = args.save_prefix_path + f"_{subject}.yaml"
120
+ eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}")
121
+ with open(file_save_path, "w", encoding="utf-8") as yaml_file:
122
+ yaml.dump(
123
+ yaml_dict,
124
+ yaml_file,
125
+ allow_unicode=True,
126
+ default_style='"',
127
+ )
128
+
129
+ if args.task_prefix != "":
130
+ mmlu_subcategories = [f"mmlu_{args.task_prefix}_{category}" for category in ALL_CATEGORIES]
131
+ else:
132
+ mmlu_subcategories = [f"mmlu_{category}" for category in ALL_CATEGORIES]
133
+
134
+ if args.group_prefix != "":
135
+ file_save_path = args.group_prefix + ".yaml"
136
+ else:
137
+ file_save_path = args.save_prefix_path + ".yaml"
138
+
139
+ eval_logger.info(f"Saving benchmark config to {file_save_path}")
140
+ with open(file_save_path, "w", encoding="utf-8") as yaml_file:
141
+ yaml.dump(
142
+ {
143
+ "group": f"mmlu_{args.task_prefix}" if args.task_prefix != "" else "mmlu",
144
+ "task": mmlu_subcategories,
145
+ },
146
+ yaml_file,
147
+ indent=4,
148
+ default_flow_style=False,
149
+ )
LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
2
+ test_split: test
3
+ fewshot_split: dev
4
+ fewshot_config:
5
+ sampler: first_n
6
+ output_type: generate_until
7
+ doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
8
+ doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}"
9
+ generation_kwargs:
10
+ until:
11
+ - "</s>"
12
+ - "\n"
13
+ metric_list:
14
+ - metric: exact_match
15
+ aggregation: mean
16
+ higher_is_better: true
17
+ metadata:
18
+ version: 2.0
19
+ dataset_kwargs:
20
+ trust_remote_code: true
LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ group: mmlu_generative
2
+ group_alias: mmlu (generative)
3
+ task:
4
+ - group: stem
5
+ task:
6
+ - mmlu_stem_generative
7
+ aggregate_metric_list:
8
+ - metric: acc
9
+ weight_by_size: True
10
+ - group: other
11
+ task:
12
+ - mmlu_other_generative
13
+ aggregate_metric_list:
14
+ - metric: acc
15
+ weight_by_size: True
16
+ - group: social sciences
17
+ task:
18
+ - mmlu_social_sciences_generative
19
+ aggregate_metric_list:
20
+ - metric: acc
21
+ weight_by_size: True
22
+ - group: humanities
23
+ task:
24
+ - mmlu_humanities_generative
25
+ aggregate_metric_list:
26
+ - metric: acc
27
+ weight_by_size: True
28
+ aggregate_metric_list:
29
+ - metric: acc
30
+ weight_by_size: True
31
+ metadata:
32
+ version: 2
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "abstract_algebra"
2
+ "description": "The following are multiple choice questions (with answers) about abstract\
3
+ \ algebra.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_abstract_algebra_generative"
7
+ "task_alias": "abstract_algebra"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "astronomy"
2
+ "description": "The following are multiple choice questions (with answers) about astronomy.\n\
3
+ \n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_astronomy_generative"
7
+ "task_alias": "astronomy"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "business_ethics"
2
+ "description": "The following are multiple choice questions (with answers) about business\
3
+ \ ethics.\n\n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_business_ethics_generative"
7
+ "task_alias": "business_ethics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "clinical_knowledge"
2
+ "description": "The following are multiple choice questions (with answers) about clinical\
3
+ \ knowledge.\n\n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_clinical_knowledge_generative"
7
+ "task_alias": "clinical_knowledge"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "college_chemistry"
2
+ "description": "The following are multiple choice questions (with answers) about college\
3
+ \ chemistry.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_college_chemistry_generative"
7
+ "task_alias": "college_chemistry"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "college_mathematics"
2
+ "description": "The following are multiple choice questions (with answers) about college\
3
+ \ mathematics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_college_mathematics_generative"
7
+ "task_alias": "college_mathematics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "college_physics"
2
+ "description": "The following are multiple choice questions (with answers) about college\
3
+ \ physics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_college_physics_generative"
7
+ "task_alias": "college_physics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "computer_security"
2
+ "description": "The following are multiple choice questions (with answers) about computer\
3
+ \ security.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_computer_security_generative"
7
+ "task_alias": "computer_security"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "conceptual_physics"
2
+ "description": "The following are multiple choice questions (with answers) about conceptual\
3
+ \ physics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_conceptual_physics_generative"
7
+ "task_alias": "conceptual_physics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "econometrics"
2
+ "description": "The following are multiple choice questions (with answers) about econometrics.\n\
3
+ \n"
4
+ "tag": "mmlu_social_sciences_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_econometrics_generative"
7
+ "task_alias": "econometrics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "electrical_engineering"
2
+ "description": "The following are multiple choice questions (with answers) about electrical\
3
+ \ engineering.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_electrical_engineering_generative"
7
+ "task_alias": "electrical_engineering"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "elementary_mathematics"
2
+ "description": "The following are multiple choice questions (with answers) about elementary\
3
+ \ mathematics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_elementary_mathematics_generative"
7
+ "task_alias": "elementary_mathematics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "formal_logic"
2
+ "description": "The following are multiple choice questions (with answers) about formal\
3
+ \ logic.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_formal_logic_generative"
7
+ "task_alias": "formal_logic"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "global_facts"
2
+ "description": "The following are multiple choice questions (with answers) about global\
3
+ \ facts.\n\n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_global_facts_generative"
7
+ "task_alias": "global_facts"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_biology"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school biology.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_biology_generative"
7
+ "task_alias": "high_school_biology"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_chemistry"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school chemistry.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_chemistry_generative"
7
+ "task_alias": "high_school_chemistry"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_computer_science"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school computer science.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_computer_science_generative"
7
+ "task_alias": "high_school_computer_science"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_european_history"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school european history.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_european_history_generative"
7
+ "task_alias": "high_school_european_history"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_geography"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school geography.\n\n"
4
+ "tag": "mmlu_social_sciences_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_geography_generative"
7
+ "task_alias": "high_school_geography"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_macroeconomics"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school macroeconomics.\n\n"
4
+ "tag": "mmlu_social_sciences_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_macroeconomics_generative"
7
+ "task_alias": "high_school_macroeconomics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_mathematics"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school mathematics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_mathematics_generative"
7
+ "task_alias": "high_school_mathematics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_microeconomics"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school microeconomics.\n\n"
4
+ "tag": "mmlu_social_sciences_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_microeconomics_generative"
7
+ "task_alias": "high_school_microeconomics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_physics"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school physics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_physics_generative"
7
+ "task_alias": "high_school_physics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_psychology"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school psychology.\n\n"
4
+ "tag": "mmlu_social_sciences_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_psychology_generative"
7
+ "task_alias": "high_school_psychology"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_statistics"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school statistics.\n\n"
4
+ "tag": "mmlu_stem_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_statistics_generative"
7
+ "task_alias": "high_school_statistics"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_us_history"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school us history.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_us_history_generative"
7
+ "task_alias": "high_school_us_history"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "high_school_world_history"
2
+ "description": "The following are multiple choice questions (with answers) about high\
3
+ \ school world history.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_high_school_world_history_generative"
7
+ "task_alias": "high_school_world_history"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "human_aging"
2
+ "description": "The following are multiple choice questions (with answers) about human\
3
+ \ aging.\n\n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_human_aging_generative"
7
+ "task_alias": "human_aging"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "human_sexuality"
2
+ "description": "The following are multiple choice questions (with answers) about human\
3
+ \ sexuality.\n\n"
4
+ "tag": "mmlu_social_sciences_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_human_sexuality_generative"
7
+ "task_alias": "human_sexuality"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "international_law"
2
+ "description": "The following are multiple choice questions (with answers) about international\
3
+ \ law.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_international_law_generative"
7
+ "task_alias": "international_law"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "jurisprudence"
2
+ "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\
3
+ \n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_jurisprudence_generative"
7
+ "task_alias": "jurisprudence"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "logical_fallacies"
2
+ "description": "The following are multiple choice questions (with answers) about logical\
3
+ \ fallacies.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_logical_fallacies_generative"
7
+ "task_alias": "logical_fallacies"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "management"
2
+ "description": "The following are multiple choice questions (with answers) about management.\n\
3
+ \n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_management_generative"
7
+ "task_alias": "management"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "marketing"
2
+ "description": "The following are multiple choice questions (with answers) about marketing.\n\
3
+ \n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_marketing_generative"
7
+ "task_alias": "marketing"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "miscellaneous"
2
+ "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\
3
+ \n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_miscellaneous_generative"
7
+ "task_alias": "miscellaneous"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "moral_disputes"
2
+ "description": "The following are multiple choice questions (with answers) about moral\
3
+ \ disputes.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_moral_disputes_generative"
7
+ "task_alias": "moral_disputes"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "moral_scenarios"
2
+ "description": "The following are multiple choice questions (with answers) about moral\
3
+ \ scenarios.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_moral_scenarios_generative"
7
+ "task_alias": "moral_scenarios"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "philosophy"
2
+ "description": "The following are multiple choice questions (with answers) about philosophy.\n\
3
+ \n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_philosophy_generative"
7
+ "task_alias": "philosophy"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "prehistory"
2
+ "description": "The following are multiple choice questions (with answers) about prehistory.\n\
3
+ \n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_prehistory_generative"
7
+ "task_alias": "prehistory"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "professional_accounting"
2
+ "description": "The following are multiple choice questions (with answers) about professional\
3
+ \ accounting.\n\n"
4
+ "tag": "mmlu_other_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_professional_accounting_generative"
7
+ "task_alias": "professional_accounting"
LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ "dataset_name": "professional_law"
2
+ "description": "The following are multiple choice questions (with answers) about professional\
3
+ \ law.\n\n"
4
+ "tag": "mmlu_humanities_generative"
5
+ "include": "_default_template_yaml"
6
+ "task": "mmlu_professional_law_generative"
7
+ "task_alias": "professional_law"