Add files using upload-large-folder tool

LaViDa/eval/lmms_eval/tasks/ai2d/ai2d_no_mask.yaml

@@ -0,0 +1,55 @@
+dataset_path: Efficient-Large-Model/ai2d-no-mask
+task: "ai2d_no_mask"
+dataset_kwargs:
+  token: True
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.ai2d_doc_to_visual
+doc_to_text: !function utils.ai2d_doc_to_text
+doc_to_target: !function utils.ai2d_doc_to_target
+  
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: mcq
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+  gpt4v: 
+    prompt_format: mcq
+    pre_prompt: ""
+    post_prompt: "\nAbove choices are given in {option}. {content} format.\nPlease answer with the option letter from the given choices directly."
+  qwen_vl:
+    prompt_format: qa
+    pre_prompt: ""
+    post_prompt: " Answer:"
+  xcomposer2_4khd:
+    prompt_format: mcq_xcomposer
+    pre_prompt: "[UNUSED_TOKEN_146]user\nQuestion: "
+    post_prompt: "[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is"
+
+model_specific_target_kwargs:
+  default: "mcq"
+  qwen_vl: "qa"
+
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+filter_list:
+  - name: "flexible-extract"
+    filter:
+      - function: !function utils.MultiChoiceRegexFilter
+        group_select: 0
+        ignore_case: true
+        ignore_punctuation: true
+        regex_pattern: "([A-Z])\\."
+
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/ai2d/upload_ai2d.py

@@ -0,0 +1,117 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+
+import datasets
+from PIL import Image
+
+# Find for instance the citation on arxiv or on the dataset repo/website
+_CITATION = """https://allenai.org/data/diagrams"""
+_DESCRIPTION = "AI2D is a dataset of illustrative diagrams for research on diagram understanding and associated question answering."
+
+
+def get_builder_config(VERSION):
+    builder_config = [
+        datasets.BuilderConfig(
+            name=f"ai2d",
+            version=VERSION,
+            description=f"ai2d",
+        )
+    ]
+    return builder_config
+
+
+dataset_features = {
+    "question": datasets.Value("string"),
+    "options": datasets.features.Sequence(datasets.Value("string")),
+    "answer": datasets.Value("string"),
+    "image": datasets.Image(),
+}
+
+
+class AI2D(datasets.GeneratorBasedBuilder):
+    VERSION = datasets.Version("1.0.0")
+
+    BUILDER_CONFIGS = get_builder_config(VERSION)
+
+    def _info(self):
+        features = datasets.Features(dataset_features)
+        return datasets.DatasetInfo(
+            # This is the description that will appear on the datasets page.
+            description=_DESCRIPTION,
+            # This defines the different columns of the dataset and their types
+            features=features,  # Here we define them above because they are different between the two configurations
+            # If there's a common (input, target) tuple from the features, uncomment supervised_keys line below and
+            # specify them. They'll be used if as_supervised=True in builder.as_dataset.
+            # supervised_keys=("sentence", "label"),
+            # Homepage of the dataset for documentation
+            # Citation for the dataset
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name
+
+        # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
+        # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
+        # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
+        image_path = "/path/to/ai2d/images"
+        annotation_path = "/path/to/ai2d/questions"
+        # wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/ai2diagram/test.jsonl
+        test_annotation_path = "/path/to/Qwen-VL/data/ai2diagram/test.jsonl"
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TEST,
+                gen_kwargs={"annotation": annotation_path, "images": image_path, "test_annotation": test_annotation_path},
+            ),
+        ]
+
+    # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
+    def _generate_examples(self, annotation, images, test_annotation):
+        # open test_annotation jsonl (note; not a json)
+        with open(test_annotation, encoding="utf-8") as f:
+            test_annotation = [json.loads(line) for line in f]
+        test_qn_ids = {x["question_id"] for x in test_annotation}
+        index = -1
+        # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
+        for sub_annotation in os.listdir(annotation):
+            sub_annotation = os.path.join(annotation, sub_annotation)
+            with open(sub_annotation, encoding="utf-8") as f:
+                data = json.load(f)
+            image = data["imageName"]
+            image_path = os.path.join(images, image)
+            for question in data["questions"]:
+                if data["questions"]["questionId"] in test_qn_ids:
+                    index += 1
+                    options = data["questions"][question]["answerTexts"]
+                    answer = data["questions"][question]["correctAnswer"]
+
+                    now_data = {}
+                    now_data["image"] = Image.open(image_path)
+                    now_data["question"] = question
+                    now_data["answer"] = answer
+                    now_data["options"] = options
+                    yield index, now_data
+
+
+if __name__ == "__main__":
+    from datasets import load_dataset
+
+    data = load_dataset(
+        "/path/to/lmms-eval/lmms_eval/tasks/ai2d/upload_ai2d.py",
+    )
+    data.push_to_hub("lmms-lab/ai2d", private=True)

LaViDa/eval/lmms_eval/tasks/ai2d/utils.py

@@ -0,0 +1,80 @@
+import re
+
+from lmms_eval.filters.extraction import ExtendedRegexFilter
+from lmms_eval.filters.transformation import MapFilter
+
+
+def ai2d_doc_to_text(doc, lmms_eval_specific_kwargs=None):
+    question, choices = doc["question"], doc["options"]
+    len_choices = len(choices)
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    if lmms_eval_specific_kwargs["prompt_format"] == "mcq":
+        options = [chr(ord("A") + i) for i in range(len_choices)]
+        choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)])
+        return f"{pre_prompt}{question}\n{choices_str}{post_prompt}"
+    elif lmms_eval_specific_kwargs["prompt_format"] == "qa":
+        options = "\n".join(choices)
+        return f"{pre_prompt}{question}{options}{post_prompt}"
+    elif lmms_eval_specific_kwargs["prompt_format"] == "mcq_xcomposer":
+        options = [chr(ord("A") + i) for i in range(len_choices)]
+        choices_str = " ".join([f"{option}. {choice}" for option, choice in zip(options, choices)])
+        return f"{pre_prompt}{question}\nContext: N/A\n{choices_str}{post_prompt}"
+    else:
+        raise ValueError(f"Unknown prompt format: {lmms_eval_specific_kwargs['prompt_format']}")
+
+
+def ai2d_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def ai2d_doc_to_target(doc, model_specific_target_kwargs):
+    if model_specific_target_kwargs == "mcq":
+        len_choices = len(doc["options"])
+        options = [chr(ord("A") + i) for i in range(len_choices)]
+        return options[int(doc["answer"])]
+    elif model_specific_target_kwargs == "qa":
+        return doc["options"][int(doc["answer"])]
+
+
+class MultiChoiceRegexFilter(ExtendedRegexFilter):
+    def __init__(self, *args, **kwargs):
+        """
+        regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure
+                        - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response.
+                        - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices.
+        group_select: Selects the (group_select)th match from the findall result.
+        ignore_case: Ignores the case during step 1 matching
+        ignore_punctuation: Remove the punctuation during step 1 matching
+        regexes_to_ignore: Remove these regexes during step 1 matching
+        """
+        super().__init__(*args, **kwargs)
+
+    def apply(self, resps, docs):
+        # here, we assume we have a list, in which each element is
+        # a list of model responses for some particular input/target pair.
+        # so we process each of these (same input/target response sets)
+        # independently (and keep them a list.)
+
+        filtered_resps = []
+
+        for r, doc in zip(resps, docs):
+            # Regex to directly extract the option letter from the model response
+            option_letter_regex = re.compile(r"^\s*([A-Z])\.")
+
+            # Process each response
+            filtered = []
+            for resp in r:
+                # Try to match the option letter at the start of the response
+                match = option_letter_regex.match(resp)
+                if match:
+                    # If a match is found, append the matched letter
+                    filtered.append(match.group(1))
+                else:
+                    # If no match, return the original response
+                    filtered.append(resp)
+
+            # Assuming we need the first response that matches or the original response
+            filtered_resps.append(filtered[0])
+
+        return filtered_resps

LaViDa/eval/lmms_eval/tasks/cmmmu/_cmmmu.yaml

@@ -0,0 +1,4 @@
+group: cmmmu
+task:
+- cmmmu_val
+- cmmmu_test

LaViDa/eval/lmms_eval/tasks/cmmmu/_default_template_cmmmu_yaml

@@ -0,0 +1,8 @@
+dataset_path: lmms-lab/CMMMU
+output_type: generate_until
+doc_to_visual: !function utils.cmmmu_doc_to_visual
+doc_to_text: !function utils.cmmmu_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  image_aspect_ratio: original

LaViDa/eval/lmms_eval/tasks/cmmmu/cmmmu_test.yaml

@@ -0,0 +1,12 @@
+task: "cmmmu_test"
+test_split: test
+# The return value of process_results will be used by metrics
+process_results: !function utils.cmmmu_process_test_results_for_submission
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: submission
+    aggregation: !function utils.cmmmu_test_aggregate_results_for_submission
+    higher_is_better: false
+metadata:
+  - version: 0.0
+include: _default_template_cmmmu_yaml

LaViDa/eval/lmms_eval/tasks/cmmmu/cmmmu_val.yaml

@@ -0,0 +1,15 @@
+task: "cmmmu_val"
+test_split: val
+# The return value of process_results will be used by metrics
+process_results: !function utils.cmmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+  image_aspect_ratio: original
+metric_list:
+  - metric: cmmmu_acc
+    aggregation: !function utils.cmmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
+include: _default_template_cmmmu_yaml

LaViDa/eval/lmms_eval/tasks/cmmmu/utils.py

@@ -0,0 +1,420 @@
+import json
+import os
+import random
+import re
+from collections import Counter, defaultdict
+
+from loguru import logger as eval_logger
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+PROMPT = {
+    "task_instructions": [
+        "请回答以下多项选择题，并选出正确选项。这些题目可能包括单选和多选题型。如果所提供的信息不足以确定一个明确的答案，那么请根据可用的数据和你的判断来选择最可能正确的选项。",
+        "请回答以下判断题，并根据题目描述和所给的信息来判断问题中陈述的对错。如果信息不完整或不足以作出绝对判断，请运用你的逻辑推理和现有信息来做出最可能的判断。",
+        "请回答以下填空题，并根据题目的要求和所提供的信息来给出最恰当的答案。如果信息不足以确切回答，那么请依据现有的数据和你的推理能力来填写最合理的答案。",
+    ],
+    "multi_choice_example_format": ["问题：{}\n选项：\n{}\n正确答案：\n"],
+    "T/F_example_format": ["问题：{}\n正确答案：\n"],
+    "short_ans_example_format": ["问题：{}\n正确答案：\n"],
+}
+
+
+def construct_prompt(sample):
+    question = sample["question"]
+    task_instructions = PROMPT["task_instructions"]
+
+    if sample["type"] == "选择":
+        formatted_options = ""
+        start_chr = "A"
+        for i in range(1, 5):
+            formatted_options += f"({start_chr}) {sample[f'option{i}']}\n"
+            start_chr = chr(ord(start_chr) + 1)
+
+        current_example_template = PROMPT["multi_choice_example_format"][0]
+        current_example = current_example_template.format(question, formatted_options)
+        final_input_prompt = task_instructions[0] + "\n\n" + current_example
+
+    elif sample["type"] == "判断":
+        current_example_template = PROMPT["T/F_example_format"][0]
+        current_example = current_example_template.format(question)
+        final_input_prompt = task_instructions[1] + "\n\n" + current_example
+
+    else:  # For fill in the blanks questions.
+        current_example_template = PROMPT["short_ans_example_format"][0]
+        current_example = current_example_template.format(question)
+        final_input_prompt = task_instructions[2] + "\n\n" + current_example
+
+    for i in range(1, 6):
+        final_input_prompt = final_input_prompt.replace(f'<img="{sample[f"image_{i}_filename"]}">', f"<图片 {i}>")
+
+    return final_input_prompt
+
+
+def cmmmu_doc_to_text(doc):
+    return construct_prompt(doc)
+
+
+def cmmmu_doc_to_visual(doc):
+    prompt = construct_prompt(doc)
+    image_tokens = re.findall(r"<图片 \d+>", prompt)
+    # Remove <> and  swap space as _
+    image_tokens = [image_token.strip("<>").replace(" ", "_").replace("图片", "image") for image_token in image_tokens]
+    visual = [doc[image_token].convert("RGB") for image_token in image_tokens]
+    return visual
+
+
+def cmmmu_process_results(doc, results):
+    pred = results[0]
+    if doc["type"] == "选择":
+        index2ans, all_choices = get_multi_choice_info([doc[f"option{i}"] for i in range(1, 5)])
+        parsed_pred = get_multi_choice_prediction(pred, all_choices, index2ans)
+    elif doc["type"] == "判断":
+        parsed_pred = get_TF_prediction(pred)
+    else:
+        parsed_pred = get_fill_blank_prediction(pred, doc["answer"])
+    return {"cmmmu_acc": {"id": doc["id"], "subdomain": doc["subcategory"], "question_type": doc["type"], "answer": doc["answer"], "parsed_pred": parsed_pred}}
+
+
+def cmmmu_aggregate_results(results):
+    evaluation_result = {}
+    subset_to_eval_samples = defaultdict(list)
+    for result in results:
+        subset_to_eval_samples[result["subdomain"]].append(result)
+    for subset, sub_eval_samples in subset_to_eval_samples.items():
+        metric_dict = eval_cmmmu(sub_eval_samples)
+        evaluation_result[subset] = metric_dict
+
+    printable_results = {}
+    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
+        in_domain_cat_results = {}
+        for cat_name in in_domain_cats:
+            if cat_name in evaluation_result.keys():
+                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
+            else:
+                pass
+        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
+        in_domain_data_num = sum([cat_results["entries_num"] for cat_results in in_domain_cat_results.values()])
+        printable_results["Overall-" + domain] = {
+            "num": int(in_domain_data_num),
+            "acc": round(in_domain_ins_acc, 3),
+        }
+        # add sub category
+        for cat_name, cat_results in in_domain_cat_results.items():
+            printable_results[cat_name] = {
+                "num": int(cat_results["entries_num"]),
+                "acc": round(cat_results["acc"], 3),
+            }
+    all_ins_acc = calculate_ins_level_acc(evaluation_result)
+    printable_results["Overall"] = {
+        "num": sum([cat_results["entries_num"] for cat_results in evaluation_result.values()]),
+        "acc": round(all_ins_acc, 3),
+    }
+    print(printable_results)
+    return printable_results["Overall"]["acc"]
+
+
+def cmmmu_process_test_results_for_submission(doc, results):
+    response = results[0]
+    return {"submission": {"id": doc["id"], "type": doc["type"], "response": response}}
+
+
+def cmmmu_test_aggregate_results_for_submission(results, args):
+    file = generate_submission_file("cmmmu_test_for_submission.jsonl", args)
+    with open(file, "w", encoding="utf8") as f:
+        for result in results:
+            json.dump(result, f, ensure_ascii=False)
+            f.write("\n")
+    eval_logger.info(f"Submission file saved to {file}")
+
+
+##################
+# Helper functions
+##################
+
+DOMAIN_CAT2SUB_CAT = {
+    "艺术与设计": ["艺术", "艺术理论", "设计", "音乐"],
+    "商业": ["会计", "经济", "金融", "管理", "营销"],
+    "科学": ["生物", "化学", "地理", "数学", "物理"],
+    "健康与医学": ["基础医学", "临床医学", "诊断学与实验室医学", "制药", "公共卫生"],
+    "人文社会科学": ["历史", "文献学", "社会学", "心理学"],
+    "技术与工程": ["农业", "建筑学", "计算机科学", "电子学", "能源和电力", "材料", "机械工程"],
+}
+
+
+def eval_cmmmu(entries):
+    correct_cnt = 0
+    for entry in entries:
+        parsed_pred = entry.get("parsed_pred", "")
+        correct = False
+        if entry.get("question_type") == "选择":
+            if parsed_pred == entry["answer"]:
+                correct_cnt += 1
+                correct = True
+
+        elif entry.get("question_type") == "填空":
+            norm_answers = normalize_str(entry["answer"], entry["answer"])
+
+            for pred in parsed_pred:
+                # already normalized
+                if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
+                    for norm_ans in norm_answers:
+                        # only see if the string answer in the string pred
+                        # print(norm_ans, pred)
+                        if isinstance(norm_ans, str) and norm_ans in pred:
+                            if not correct:
+                                correct_cnt += 1
+                                correct = True
+                            break
+                else:  # it's a number
+                    if pred in norm_answers:
+                        if not correct:
+                            correct_cnt += 1
+                            correct = True
+                        break
+
+        else:
+            positive_keywords = ["正确", "对", "准确", "肯定", "对的"]
+            negative_keywords = ["不对", "错误", "不正确", "不准确", "不合适", "否定", "错的", "错"]
+            ambiguous_keywords = ["对错", "是否正确", "否正确", "或者", "是否", "正确性", "对不"]
+
+            def judge_similarity(pred_list, positive_keywords, negative_keywords):
+                positive_count = 0
+                negative_count = 0
+
+                for pred in pred_list:
+                    if any(pos_word in pred for pos_word in positive_keywords):
+                        positive_count += 1
+                    elif any(neg_word in pred for neg_word in negative_keywords):
+                        negative_count += 1
+
+                if positive_count > negative_count:
+                    return "对"
+                elif negative_count > positive_count:
+                    return "错"
+                else:
+                    return random.choice(["对", "错"])
+
+            answer = entry["answer"]
+            parsed_pred = [word for word in parsed_pred if not any(ambiguous in word for ambiguous in ambiguous_keywords)]
+            result = judge_similarity(parsed_pred, positive_keywords, negative_keywords)
+            if result == answer:
+                correct_cnt += 1
+                correct = True
+        if correct:
+            entry["judge"] = "正确"
+        else:
+            entry["judge"] = "错误"
+
+    if len(entries) == 0:
+        print("entries_num == 0, please check your file")
+        results_count = {"correct_num": 0, "entries_num": 0, "acc": 0}
+    else:
+        results_count = {"correct_num": correct_cnt, "entries_num": len(entries), "acc": correct_cnt / len(entries)}
+
+    return results_count
+
+
+def get_multi_choice_prediction(response, all_choices, index2ans):
+    for char in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(char)
+    response = " " + response + " "  # add space to avoid partial match
+
+    candidates = []
+
+    for choice in all_choices:  # (A) (B) (C) (D)
+        # Add the choice to candidates each time it appears in the response
+        candidates.extend([choice for _ in range(response.count(f"({choice})"))])
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # A B C D
+            # Similarly, add the choice for each occurrence
+            candidates.extend([choice for _ in range(response.count(f"{choice}"))])
+
+    if len(candidates) == 0 and len(response.split()) >= 1:
+        for index, ans in index2ans.items():
+            # Add index for each occurrence of ans in response
+            candidates.extend([index for _ in range(response.count(ans))])
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) >= 1:
+        for index, ans in index2ans.items():
+            if ans in response:
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        return random.choice(all_choices)
+        # return ''
+    else:
+        # Count the occurrence of each candidate
+        candidate_counts = Counter(candidates)
+
+        # Select the most frequent candidates
+        max_count = max(candidate_counts.values())
+        most_frequent_candidates = [c for c in all_choices if candidate_counts.get(c, 0) == max_count]
+
+        # Combine the most frequent candidates in ABCD order
+        return "".join(most_frequent_candidates)
+
+
+def extract_numbers(string):
+    # Pattern for numbers with Chinese commas
+    pattern_commas = r"-?\d{1,3}(?:，\d{3})+"
+    # Pattern for scientific notation
+    pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+"
+    # Pattern for simple numbers without Chinese commas
+    pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+)(?![eE][+-]?\d+)(?!，\d)"
+
+    # Extract numbers with Chinese commas
+    numbers_with_commas = re.findall(pattern_commas, string)
+    # Extract numbers in scientific notation
+    numbers_scientific = re.findall(pattern_scientific, string)
+    # Extract simple numbers without Chinese commas
+    numbers_simple = re.findall(pattern_simple, string)
+
+    # Combine all extracted numbers
+    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
+    return all_numbers
+
+
+def check_is_number(string):
+    try:
+        float(string.replace(",", ""))
+        return True
+    except ValueError:
+        # check if there's comma inside
+        return False
+
+
+def count_letters(string):
+    return sum(c.isalpha() and "a" <= c <= "z" or "A" <= c <= "Z" for c in string)
+
+
+def normalize_str(string, answer):
+    # check if characters in the string
+
+    # if number, numerize it.
+    if string == None:
+        return [string]
+    string = string.strip()
+
+    is_number = check_is_number(string)
+
+    if is_number:
+        string = string.replace(",", "")
+        string = float(string)
+        # leave 2 decimal
+        string = round(string, 2)
+        return [string]
+    else:  # it's likely to be a string
+        if len(string) > len(answer) + 20 or count_letters(string) > count_letters(answer) + 2:
+            return []
+        return [string]
+
+
+def get_fill_blank_prediction(response, answer):
+    """get the prediction from the generated response,
+    return a list of predicted strings or numbers"""
+
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip("。").strip()
+        sub_responses = re.split(r"。|\n", response)
+        indicators_of_keys = ["是", "为", "所以", "等于", "方案", "选择", "正确答案", "因此", "最后", "答案", "结果"]
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
+            if index == len(sub_responses) - 1:
+                indicators_of_keys.extend(["="])
+            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [":", ",", ".", "!", "?", ";", ":", "'"]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0:  # did not found any
+            return [response]
+        return key_responses
+
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy()  # keep the original string response
+    for resp in key_responses:
+        pred_list.extend(extract_numbers(resp))
+
+    tmp_pred_list = []
+    for i in range(len(pred_list)):
+        tmp_pred_list.extend(normalize_str(pred_list[i], answer))
+    pred_list = tmp_pred_list
+
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+
+def get_TF_prediction(response):
+    """get the prediction from the generated response,
+    return a list of predicted strings or numbers"""
+
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip("。").strip()
+        sub_responses = re.split(r"。|\n", response)
+        indicators_of_keys = ["是", "为", "所以", "判断", "陈述", "说法", "表达", "答案", "结果"]
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [":", ",", ".", "!", "?", ";", ":", "'"]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0:  # did not found any
+            return [response]
+        return key_responses
+
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy()  # keep the original string response
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+
+def get_multi_choice_info(options):
+    start_chr = "A"
+    all_choices = []
+    index2ans = {}
+    for i, option in enumerate(options):
+        index2ans[chr(ord(start_chr) + i)] = option
+        all_choices.append(chr(ord(start_chr) + i))
+
+    return index2ans, all_choices
+
+
+def calculate_ins_level_acc(results):
+    correct_sum = 0
+    entries_sum = 0
+    for cat_results in results.values():
+        correct_sum += cat_results["correct_num"]
+        entries_sum += cat_results["entries_num"]
+    if entries_sum == 0:
+        return 0
+    return correct_sum / entries_sum

LaViDa/eval/lmms_eval/tasks/hrbench/hrbench.yaml

@@ -0,0 +1,8 @@
+group: hrbench
+task:
+  - hrbench4k
+  - hrbench8k
+metadata:
+  version: 0.0
+  gpt_eval_model_name: "gpt-3.5-turbo"
+  max_workers: 1

LaViDa/eval/lmms_eval/tasks/hrbench/hrbench4k.yaml

@@ -0,0 +1,23 @@
+dataset_path: DreamMr/HR-Bench
+dataset_name: hrbench_version_split
+dataset_kwargs:
+  token: True
+task: "hrbench4k"
+test_split: hrbench_4k
+output_type: generate_until
+doc_to_visual: !function utils.hrbench_doc_to_visual
+doc_to_text: !function utils.hrbench_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.hrbench_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: single
+    aggregation: !function utils.hrbench_aggregate_results
+    higher_is_better: true
+  - metric: cross
+    aggregation: !function utils.hrbench_aggregate_results
+    higher_is_better: true
+  - metric: average
+    aggregation: !function utils.hrbench_aggregate_results
+    higher_is_better: true

LaViDa/eval/lmms_eval/tasks/hrbench/hrbench8k.yaml

@@ -0,0 +1,23 @@
+dataset_path: DreamMr/HR-Bench
+dataset_name: hrbench_version_split
+dataset_kwargs:
+  token: True
+task: "hrbench8k"
+test_split: hrbench_8k
+output_type: generate_until
+doc_to_visual: !function utils.hrbench_doc_to_visual
+doc_to_text: !function utils.hrbench_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.hrbench_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: single
+    aggregation: !function utils.hrbench_aggregate_results
+    higher_is_better: true
+  - metric: cross
+    aggregation: !function utils.hrbench_aggregate_results
+    higher_is_better: true
+  - metric: average
+    aggregation: !function utils.hrbench_aggregate_results
+    higher_is_better: true

LaViDa/eval/lmms_eval/tasks/hrbench/hrbench_evals.py

@@ -0,0 +1,153 @@
+import copy as cp
+import os
+import string
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import pandas as pd
+import requests
+from loguru import logger as eval_logger
+from tqdm import tqdm
+
+
+class HRBenchEval:
+    API_TYPE = os.getenv("API_TYPE", "openai")
+
+    if API_TYPE == "openai":
+        API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+        API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
+        headers = {
+            "Authorization": f"Bearer {API_KEY}",
+            "Content-Type": "application/json",
+        }
+    elif API_TYPE == "azure":
+        API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
+        API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
+        headers = {
+            "api-key": API_KEY,
+            "Content-Type": "application/json",
+        }
+
+    def __init__(self, api_key, gpt_model="gpt-3.5-turbo", max_workers=12):
+        self.api_key = api_key
+        self.gpt_model = gpt_model
+        self.max_workers = max_workers
+
+    def _post_request(self, payload):
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        response = requests.post(self.API_URL, headers=headers, json=payload, timeout=30)
+        response.raise_for_status()
+        return response.json()
+
+    def can_infer_option(self, answer, choices):
+        verbose = os.environ.get("VERBOSE", 0)
+        # Choices is a dictionary
+        if "Failed to obtain answer via API" in answer:
+            return False
+
+        reject_to_answer = ["Sorry, I can't help with images of people yet.", "I can't process this file.", "I'm sorry, but without the image provided", "Cannot determine the answer"]
+        for err in reject_to_answer:
+            if err in answer:
+                return "Z"
+
+        def count_choice(splits, choices, prefix="", suffix=""):
+            cnt = 0
+            for c in choices:
+                if prefix + c + suffix in splits:
+                    cnt += 1
+            return cnt
+
+        answer_mod = cp.copy(answer)
+        chars = ".()[],:;!*#{}"
+        for c in chars:
+            answer_mod = answer_mod.replace(c, " ")
+
+        splits = [x.strip() for x in answer_mod.split()]
+        count = count_choice(splits, choices)
+
+        if count == 1:
+            for ch in choices:
+                if "A" in splits and len(splits) > 3 and verbose:
+                    return False
+                if ch in splits:
+                    return ch
+        elif count == 0 and count_choice(splits, {"Z", ""}) == 1:
+            return "Z"
+        return False
+
+    def can_infer_text(self, answer, choices):
+        answer = answer.lower()
+        assert isinstance(choices, dict)
+        for k in choices:
+            assert k in string.ascii_uppercase
+            choices[k] = str(choices[k]).lower()
+        cands = []
+        for k in choices:
+            if choices[k] in answer:
+                cands.append(k)
+        if len(cands) == 1:
+            return cands[0]
+        return False
+
+    def can_infer(self, answer, choices):
+        answer = str(answer)
+        copt = self.can_infer_option(answer, choices)
+        return copt if copt else self.can_infer_text(answer, choices)
+
+    def get_chat_response(self, data, temperature=0, max_tokens=256, patience=10, sleep_time=0):
+        question = data["question"]
+        options = data["options"]
+        prediction = data["prediction"]
+        ret = self.can_infer(prediction, options)
+        if ret:
+            data["gpt_prediction"] = ret
+            return data
+
+        prompt = self.build_prompt(question, options, prediction)
+        messages = [
+            {"role": "user", "content": prompt},
+        ]
+        payload = {"model": self.gpt_model, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, "n": 1}
+
+        while patience > 0:
+            patience -= 1
+            try:
+                response = self._post_request(payload)
+                prediction = response["choices"][0]["message"]["content"].strip()
+                if prediction and prediction != "" and "Failed to obtain answer via API" not in prediction:
+                    ret = self.can_infer(prediction, options)
+                    data["gpt_prediction"] = ret
+                    return data
+
+            except Exception as e:
+                # some model may output repetitive answer, which ChatGPT will throw an error.
+                eval_logger.error(e)
+
+                if sleep_time > 0:
+                    time.sleep(sleep_time)
+        return data
+
+    def build_prompt(self, question, options, prediction):
+        options_prompt = ""
+        for key, item in options.items():
+            options_prompt += f"{key}. {item}\n"
+        tmpl = (
+            "You are an AI assistant who will help me to match "
+            "an answer with several options of a single-choice question. "
+            "You are provided with a question, several options, and an answer, "
+            "and you need to find which option is most similar to the answer. "
+            "If the meaning of all options are significantly different from the answer, output Z. "
+            "Your should output a single uppercase character in A, B, C, D (if they are valid options), and Z. \n"
+            "Example 1: \n"
+            "Question: What is the main object in image?\nOptions: A. teddy bear B. rabbit C. cat D. dog\n"
+            "Answer: a cute teddy bear\nYour output: A\n"
+            "Example 2: \n"
+            "Question: What is the main object in image?\nOptions: A. teddy bear B. rabbit C. cat D. dog\n"
+            "Answer: Spider\nYour output: Z\n"
+            "Example 3: \n"
+            "Question: {}\nOptions: {}\nAnswer: {}\nYour output: "
+        )
+        return tmpl.format(question, options_prompt, prediction)

LaViDa/eval/lmms_eval/tasks/hrbench/utils.py

@@ -0,0 +1,103 @@
+import base64
+import datetime
+import io
+import json
+import os
+import string
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import yaml
+from loguru import logger as eval_logger
+from PIL import Image
+
+from lmms_eval.tasks.hrbench.hrbench_evals import HRBenchEval
+
+with open(Path(__file__).parent / "hrbench.yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+hrbench_evaluator = HRBenchEval(api_key=os.getenv("OPENAI_API_KEY", "YOUR_API_KEY"), gpt_model=config["metadata"]["gpt_eval_model_name"], max_workers=config["metadata"]["max_workers"])
+
+
+def decode_base64_to_image(base64_string, target_size=-1):
+    image_data = base64.b64decode(base64_string)
+    image = Image.open(io.BytesIO(image_data)).convert("RGB")
+    if target_size > 0:
+        image.thumbnail((target_size, target_size))
+    return image
+
+
+def hrbench_doc_to_visual(doc):
+    image = decode_base64_to_image(doc["image"])
+    return [image]
+
+
+def hrbench_doc_to_options(doc):
+    options = {cand: doc[cand] for cand in string.ascii_uppercase if cand in doc and not pd.isna(doc[cand])}
+    return options
+
+
+def hrbench_doc_to_text(doc, lmms_eval_specific_kwargs=None):
+    question = doc["question"].strip()
+    options = hrbench_doc_to_options(doc)
+    options_prompt = ""
+    for key, item in options.items():
+        options_prompt += f"{key}. {item}\n"
+    prompt = ""
+    prompt += f"{question}\n{options_prompt}Answer the option letter directly."
+    return prompt
+
+
+def hrbench_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name, value: metric value
+    """
+    pred = results[0].strip()
+    gt = doc["answer"]
+    options = hrbench_doc_to_options(doc)
+    question = doc["question"]
+    resp_dic = hrbench_evaluator.get_chat_response({"question": question, "options": options, "prediction": pred})
+    gpt_prediction = resp_dic["gpt_prediction"]
+    category = doc["category"]
+    cycle_category = doc["cycle_category"]
+
+    gpt_score = 0
+    if gt.lower() == gpt_prediction.lower():
+        gpt_score = 1
+
+    return {category: {"index": doc["index"], "cycle_category": cycle_category, "gpt_score": gpt_score}, "average": {"index": doc["index"], "cycle_category": cycle_category, "gpt_score": gpt_score}}
+
+
+def hrbench_aggregate_results(results, args):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    cycle_category_scores = defaultdict(list)
+    for result in results:
+        score = result["gpt_score"]
+        cycle_category = result["cycle_category"]
+        cycle_category_scores[cycle_category].append(score)
+
+    cycle_category_avg_score = {}
+    for cycle_category, scores in cycle_category_scores.items():
+        avg_score = sum(scores) / len(scores)
+        cycle_category_avg_score[cycle_category] = avg_score
+
+    avg_score = sum(cycle_category_avg_score.values()) / len(cycle_category_avg_score)
+    return avg_score

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa.yaml

@@ -0,0 +1,9 @@
+group: livexiv_vqa
+task:
+- livexiv_vqa_v1
+- livexiv_vqa_v2
+- livexiv_vqa_v3
+- livexiv_vqa_v4
+- livexiv_vqa_v5
+- livexiv_vqa_v6
+

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_template_yaml

@@ -0,0 +1,26 @@
+dataset_path: LiveXiv/LiveXiv
+dataset_kwargs:
+  token: True
+test_split: test
+dataset_name: VQA-2024-09-21
+output_type: generate_until
+doc_to_visual: !function utils.livexiv_doc_to_visual
+doc_to_text: !function utils.livexiv_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+  image_aspect_ratio: original
+process_results: !function utils.livexiv_process_result
+process_results_use_image: true
+metric_list:
+  - metric: livexiv_vqa
+    aggregation: !function utils.livexiv_aggregation_result
+    higher_is_better: true
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_v1.yaml

@@ -0,0 +1,3 @@
+task: "livexiv_vqa_v1"
+dataset_name: "VQA-2024-09-21"
+include: livexiv_vqa_template_yaml

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_v2.yaml

@@ -0,0 +1,3 @@
+task: "livexiv_vqa_v2"
+dataset_name: "VQA-2024-10-26"
+include: livexiv_vqa_template_yaml

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_v3.yaml

@@ -0,0 +1,3 @@
+task: "livexiv_vqa_v3"
+dataset_name: "v3-VQA"
+include: livexiv_vqa_template_yaml

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_v4.yaml

@@ -0,0 +1,3 @@
+task: "livexiv_vqa_v4"
+dataset_name: "v4-VQA"
+include: livexiv_vqa_template_yaml

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_v5.yaml

@@ -0,0 +1,3 @@
+task: "livexiv_vqa_v5"
+dataset_name: "v5-VQA"
+include: livexiv_vqa_template_yaml

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/livexiv_vqa_v6.yaml

@@ -0,0 +1,3 @@
+task: "livexiv_vqa_v6"
+dataset_name: "v6-VQA"
+include: livexiv_vqa_template_yaml

LaViDa/eval/lmms_eval/tasks/livexiv_vqa/utils.py

@@ -0,0 +1,74 @@
+import json
+import re
+
+
+def extract_answer(text):
+    match = re.findall(r"(?<!^)[A-Z]", text)
+    if match:
+        return match[0]
+    return None
+
+
+def livexiv_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def livexiv_doc_to_text(doc, model_specific_kwargs=None):
+    question = doc["question"]
+    question += "\n" + f"A. {doc['option_a']}\n"
+    question += f"B. {doc['option_b']}\n"
+    question += f"C. {doc['option_c']}\n"
+    question += f"D. {doc['option_d']}"
+    return f"{question}\nAnswer with the option's letter from the given choices directly."
+
+
+def livexiv_process_result(doc, result):
+    pred = result[0].strip()
+    if len(pred) > 1:
+        if "answer" in pred.lower():
+            pred = extract_answer(pred)
+        else:
+            pred = pred[0]
+    answer = doc["gt"]
+
+    return {f"livexiv_vqa": {"pred": pred, "answer": answer}}
+
+
+def livexiv_aggregation_result(results):
+    total_count = 0
+    total_correct = 0
+    for result in results:
+        try:
+            if result["pred"].lower().strip() == result["answer"].lower().strip():
+                total_correct += 1
+        except Exception as e:
+            print(e)
+
+        total_count += 1
+    return total_correct / total_count
+
+
+def livexiv_aggregation_result_all(results):
+    score = livexiv_aggregation_result(results)
+    stored_results = []
+    for result in results:
+        stored_results.append({"question_id": result["question_id"], "prediction": result["pred"]})
+    with open("./livexiv_vqa_submission.json", "w") as f:
+        json.dump(stored_results, f, indent=4)
+    print("Storing files for LiveXiv-VQA submission ...")
+
+    return score
+
+
+def livexiv_doc_to_text_mc(doc):
+    question = doc["question"]
+    return f"{question} Answer :"
+
+
+def livexiv_doc_to_choice(doc):
+    return [doc["option_a"], doc["option_b"], doc["option_c"], doc["option_d"]]
+
+
+def livexiv_doc_to_mc_target(doc):
+    answer2choice = {"A": "option_a", "B": "option_b", "C": "option_c", "D": "option_d"}
+    return doc[answer2choice[doc["answer"]]]

LaViDa/eval/lmms_eval/tasks/mlvu/mlvu_dev.yaml

@@ -0,0 +1,27 @@
+dataset_path: sy1998/MLVU_dev
+dataset_kwargs:
+  token: True
+  cache_dir: mlvu
+  video: True
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+task: mlvu_dev
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.mlvu_doc_to_visual_dev
+doc_to_text: !function utils.mlvu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mlvu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: mlvu_percetion_score
+    aggregation: !function utils.mlvu_aggregate_results_dev
+    higher_is_better: true
+
+
+

LaViDa/eval/lmms_eval/tasks/mlvu/mlvu_test.yaml

@@ -0,0 +1,27 @@
+dataset_path: sy1998/MLVU_Test
+dataset_kwargs:
+  token: True
+  cache_dir: mlvu_test
+  video: True
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+task: mlvu_test
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.mlvu_doc_to_visual_test
+doc_to_text: !function utils.mlvu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mlvu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: mlvu_percetion_score
+    aggregation: !function utils.mlvu_aggregate_results_test
+    higher_is_better: true
+
+
+

LaViDa/eval/lmms_eval/tasks/mlvu/utils.py

@@ -0,0 +1,181 @@
+import datetime
+import json
+import os
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import cv2
+import numpy as np
+import yaml
+from loguru import logger as eval_logger
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+hf_home = os.getenv("HF_HOME", "./~/.cache/huggingface")
+# hf_home="/share/junjie/shuyan/lmms-eval/~/.cache/huggingface"
+base_cache_dir = os.path.expanduser(hf_home)
+
+
+with open(Path(__file__).parent / "mlvu_dev.yaml", "r") as f:
+    raw_data_dev = f.readlines()
+    safe_data_dev = []
+    for i, line in enumerate(raw_data_dev):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data_dev.append(line)
+cache_name_dev = yaml.safe_load("".join(safe_data_dev))["dataset_kwargs"]["cache_dir"]
+cache_dir_dev = os.path.join(base_cache_dir, cache_name_dev)
+
+
+with open(Path(__file__).parent / "mlvu_test.yaml", "r") as f:
+    raw_data_test = f.readlines()
+    safe_data_test = []
+    for i, line in enumerate(raw_data_test):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data_test.append(line)
+cache_name_test = yaml.safe_load("".join(safe_data_test))["dataset_kwargs"]["cache_dir"]
+cache_dir_test = os.path.join(base_cache_dir, cache_name_test)
+
+
+def mlvu_doc_to_visual_dev(doc):
+    video_path = doc["video_name"]
+    video_path = os.path.join(cache_dir_dev, video_path)
+    if os.path.exists(video_path):
+        video_path = video_path
+    else:
+        sys.exit(f"video path:{video_path} does not exist, please check")
+    return [video_path]
+
+
+def mlvu_doc_to_visual_test(doc):
+    video_path = doc["video_name"]
+    video_path = os.path.join(cache_dir_test, video_path)
+    if os.path.exists(video_path):
+        video_path = video_path
+    else:
+        sys.exit(f"video path:{video_path} does not exist, please check")
+    return [video_path]
+
+
+def mlvu_doc_to_text(doc, lmms_eval_specific_kwargs=None):
+    # option_prompt="Carefully watch this video and pay attention to every detail. Based on your observations, select the best option that accurately addresses the question."
+    option_prompt = ""
+    question = doc["question"] + "\nOnly give the best option.\n"
+    full_prompt = option_prompt + "\n" + question + "\n" + "Best option: ("
+    return full_prompt
+
+
+def extract_characters_regex(s):
+    s = s.strip()
+    if ")" in s:
+        index = s.index(")")
+        pred = s[index - 1 : index]
+        return pred
+    else:
+        return s
+
+
+def mlvu_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case videomme score), value: metric value
+    """
+    pred = results[0]
+
+    pred_ans = extract_characters_regex(pred)
+
+    task_type = doc["task_type"]
+    data_dict = {"question_id": doc["question"], "task_type": task_type, "pred_answer": pred_ans, "answer": doc["answer"]}
+
+    return {f"mlvu_percetion_score": data_dict}
+
+
+def mlvu_aggregate_results_dev(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    TASK_TYPES = {"anomaly_reco", "count", "ego", "needle", "order", "plotQA", "topic_reasoning"}
+    category2score = {}
+    for task_type in TASK_TYPES:
+        category2score[task_type] = {"correct": 0, "answered": 0}
+
+    for result in results:
+        task_type = result["task_type"]
+        category2score[task_type]["answered"] += 1
+        category2score[task_type]["correct"] += result["pred_answer"] == result["answer"]
+
+    task_category_scores = {}
+
+    # Calculate and log accuracy for each task category
+    for task_cate in TASK_TYPES:
+        total_correct = 0
+        total_answered = 0
+        for k, v in category2score.items():
+            if task_cate in k:
+                total_correct += v["correct"]
+                total_answered += v["answered"]
+        accuracy = 100 * total_correct / total_answered if total_answered > 0 else 0
+        task_category_scores[task_cate] = accuracy
+        eval_logger.info(f"Evaluation on Task Categories: {task_cate}: {accuracy:.1f}%")
+
+    # Calculate and log average accuracy across all task categories
+    if TASK_TYPES:
+        average_accuracy = sum(task_category_scores.values()) / len(TASK_TYPES)
+    else:
+        average_accuracy = 0
+
+    eval_logger.info(f"Average Performance Across All Task Categories: {average_accuracy:.1f}%")
+
+    return average_accuracy
+
+
+def mlvu_aggregate_results_test(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    TASK_TYPES = {"anomaly_reco", "count", "ego", "needleQA", "order", "plotQA", "sportsQA", "topic_reasoning", "tutorialQA"}
+    category2score = {}
+    for task_type in TASK_TYPES:
+        category2score[task_type] = {"correct": 0, "answered": 0}
+
+    for result in results:
+        task_type = result["task_type"]
+        category2score[task_type]["answered"] += 1
+        category2score[task_type]["correct"] += result["pred_answer"] == result["answer"]
+
+    task_category_scores = {}
+
+    # Calculate and log accuracy for each task category
+    for task_cate in TASK_TYPES:
+        total_correct = 0
+        total_answered = 0
+        for k, v in category2score.items():
+            if task_cate in k:
+                total_correct += v["correct"]
+                total_answered += v["answered"]
+        accuracy = 100 * total_correct / total_answered if total_answered > 0 else 0
+        task_category_scores[task_cate] = accuracy
+        eval_logger.info(f"Evaluation on Task Categories: {task_cate}: {accuracy:.1f}%")
+
+    # Calculate and log average accuracy across all task categories
+    if TASK_TYPES:
+        average_accuracy = sum(task_category_scores.values()) / len(TASK_TYPES)
+    else:
+        average_accuracy = 0
+
+    eval_logger.info(f"Average Performance Across All Task Categories: {average_accuracy:.1f}%")
+
+    return average_accuracy

LaViDa/eval/lmms_eval/tasks/mmlu/default/_default_template_yaml

@@ -0,0 +1,17 @@
+dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
+dataset_kwargs:
+  trust_remote_code: true

LaViDa/eval/lmms_eval/tasks/mmlu/default/_mmlu.yaml

@@ -0,0 +1,11 @@
+group: mmlu
+task:
+  - mmlu_stem
+  - mmlu_other
+  - mmlu_social_sciences
+  - mmlu_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

LaViDa/eval/lmms_eval/tasks/mmlu/default/_mmlu_humanities.yaml

@@ -0,0 +1,9 @@
+group: mmlu_humanities
+group_alias: humanities
+task:
+  - mmlu_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

LaViDa/eval/lmms_eval/tasks/mmlu/default/_mmlu_other.yaml

@@ -0,0 +1,9 @@
+group: mmlu_other
+group_alias: other
+task:
+  - mmlu_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

LaViDa/eval/lmms_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml

@@ -0,0 +1,9 @@
+group: mmlu_social_sciences
+group_alias: social sciences
+task:
+  - mmlu_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

LaViDa/eval/lmms_eval/tasks/mmlu/default/_mmlu_stem.yaml

@@ -0,0 +1,9 @@
+group: mmlu_stem
+group_alias: stem
+task:
+  - mmlu_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "abstract_algebra"
+"description": "The following are multiple choice questions (with answers) about abstract\
+  \ algebra.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_abstract_algebra"
+"task_alias": "abstract_algebra"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_anatomy.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "anatomy"
+"description": "The following are multiple choice questions (with answers) about anatomy.\n\
+  \n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_anatomy"
+"task_alias": "anatomy"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_astronomy.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "astronomy"
+"description": "The following are multiple choice questions (with answers) about astronomy.\n\
+  \n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_astronomy"
+"task_alias": "astronomy"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_business_ethics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "business_ethics"
+"description": "The following are multiple choice questions (with answers) about business\
+  \ ethics.\n\n"
+"tag": "mmlu_other_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_business_ethics"
+"task_alias": "business_ethics"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "clinical_knowledge"
+"description": "The following are multiple choice questions (with answers) about clinical\
+  \ knowledge.\n\n"
+"tag": "mmlu_other_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_clinical_knowledge"
+"task_alias": "clinical_knowledge"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_college_biology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_biology"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ biology.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_college_biology"
+"task_alias": "college_biology"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_chemistry"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ chemistry.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_college_chemistry"
+"task_alias": "college_chemistry"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_computer_science"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ computer science.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_college_computer_science"
+"task_alias": "college_computer_science"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_mathematics"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ mathematics.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_college_mathematics"
+"task_alias": "college_mathematics"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_college_medicine.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_medicine"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ medicine.\n\n"
+"tag": "mmlu_other_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_college_medicine"
+"task_alias": "college_medicine"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_college_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_physics"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ physics.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_college_physics"
+"task_alias": "college_physics"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_computer_security.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "computer_security"
+"description": "The following are multiple choice questions (with answers) about computer\
+  \ security.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_computer_security"
+"task_alias": "computer_security"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "conceptual_physics"
+"description": "The following are multiple choice questions (with answers) about conceptual\
+  \ physics.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_conceptual_physics"
+"task_alias": "conceptual_physics"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_econometrics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "econometrics"
+"description": "The following are multiple choice questions (with answers) about econometrics.\n\
+  \n"
+"tag": "mmlu_social_sciences_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_econometrics"
+"task_alias": "econometrics"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "electrical_engineering"
+"description": "The following are multiple choice questions (with answers) about electrical\
+  \ engineering.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_electrical_engineering"
+"task_alias": "electrical_engineering"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "elementary_mathematics"
+"description": "The following are multiple choice questions (with answers) about elementary\
+  \ mathematics.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_elementary_mathematics"
+"task_alias": "elementary_mathematics"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_formal_logic.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "formal_logic"
+"description": "The following are multiple choice questions (with answers) about formal\
+  \ logic.\n\n"
+"tag": "mmlu_humanities_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_formal_logic"
+"task_alias": "formal_logic"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_global_facts.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "global_facts"
+"description": "The following are multiple choice questions (with answers) about global\
+  \ facts.\n\n"
+"tag": "mmlu_other_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_global_facts"
+"task_alias": "global_facts"

LaViDa/eval/lmms_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_biology"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school biology.\n\n"
+"tag": "mmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_biology"
+"task_alias": "high_school_biology"