Add files using upload-large-folder tool

LaViDa/eval/lmms_eval/tasks/ai2d/ai2d.yaml

@@ -0,0 +1,55 @@
+dataset_path: lmms-lab/ai2d
+task: "ai2d"
+dataset_kwargs:
+  token: True
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.ai2d_doc_to_visual
+doc_to_text: !function utils.ai2d_doc_to_text
+doc_to_target: !function utils.ai2d_doc_to_target
+  
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: mcq
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+  gpt4v: 
+    prompt_format: mcq
+    pre_prompt: ""
+    post_prompt: "\nAbove choices are given in {option}. {content} format.\nPlease answer with the option letter from the given choices directly."
+  qwen_vl:
+    prompt_format: qa
+    pre_prompt: ""
+    post_prompt: " Answer:"
+  xcomposer2_4khd:
+    prompt_format: mcq_xcomposer
+    pre_prompt: "[UNUSED_TOKEN_146]user\nQuestion: "
+    post_prompt: "[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is"
+
+model_specific_target_kwargs:
+  default: "mcq"
+  qwen_vl: "qa"
+
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+filter_list:
+  - name: "flexible-extract"
+    filter:
+      - function: !function utils.MultiChoiceRegexFilter
+        group_select: 0
+        ignore_case: true
+        ignore_punctuation: true
+        regex_pattern: "([A-Z])\\."
+
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/ai2d/ai2d_lite.yaml

@@ -0,0 +1,56 @@
+dataset_path: lmms-lab/LMMs-Eval-Lite
+task: "ai2d_lite"
+dataset_kwargs:
+  token: True
+dataset_name: ai2d
+test_split: lite
+output_type: generate_until
+doc_to_visual: !function utils.ai2d_doc_to_visual
+doc_to_text: !function utils.ai2d_doc_to_text
+doc_to_target: !function utils.ai2d_doc_to_target
+  
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: mcq
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+  gpt4v: 
+    prompt_format: mcq
+    pre_prompt: ""
+    post_prompt: "\nAbove choices are given in {option}. {content} format.\nPlease answer with the option letter from the given choices directly."
+  qwen_vl:
+    prompt_format: qa
+    pre_prompt: ""
+    post_prompt: " Answer:"
+  xcomposer2_4khd:
+    prompt_format: mcq_xcomposer
+    pre_prompt: "[UNUSED_TOKEN_146]user\nQuestion: "
+    post_prompt: "[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is"
+
+model_specific_target_kwargs:
+  default: "mcq"
+  qwen_vl: "qa"
+
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+filter_list:
+  - name: "flexible-extract"
+    filter:
+      - function: !function utils.MultiChoiceRegexFilter
+        group_select: 0
+        ignore_case: true
+        ignore_punctuation: true
+        regex_pattern: "([A-Z])\\."
+
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/air_bench/_default_template_yaml

@@ -0,0 +1,7 @@
+dataset_path: lmms-lab/AIR_Bench
+dataset_kwargs:
+  token: True
+
+metadata:
+  gpt_eval_model_name: gpt-4o
+  version: 0.0

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_chat.yaml

@@ -0,0 +1,6 @@
+group: air_bench_chat
+task:
+  - air_bench_chat_sound
+  - air_bench_chat_music
+  - air_bench_chat_speech
+  - air_bench_chat_mixed

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_chat_mixed.yaml

@@ -0,0 +1,25 @@
+task: "air_bench_chat_mixed"
+dataset_name: "Chat"
+test_split: mixed
+doc_to_target: "answer_gt"
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_chat
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.2
+  top_p: 1.0
+  num_beams: 1
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Give a detail answer to the question in English."
+metric_list:
+  - metric: gpt_eval
+    aggregation: !function utils.air_bench_aggregate_results_chat
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_chat
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_chat_music.yaml

@@ -0,0 +1,25 @@
+task: "air_bench_chat_music"
+dataset_name: "Chat"
+test_split: music
+doc_to_target: "answer_gt"
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_chat
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.2
+  top_p: 1.0
+  num_beams: 1
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Give a detail answer to the question in English."
+metric_list:
+  - metric: gpt_eval
+    aggregation: !function utils.air_bench_aggregate_results_chat
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_chat
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_chat_sound.yaml

@@ -0,0 +1,25 @@
+task: "air_bench_chat_sound"
+dataset_name: "Chat"
+test_split: sound
+doc_to_target: "answer_gt"
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_chat
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.2
+  top_p: 1.0
+  num_beams: 1
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Give a detail answer to the question in English."
+metric_list:
+  - metric: gpt_eval
+    aggregation: !function utils.air_bench_aggregate_results_chat
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_chat
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_chat_speech.yaml

@@ -0,0 +1,25 @@
+task: "air_bench_chat_speech"
+dataset_name: "Chat"
+test_split: speech
+doc_to_target: "answer_gt"
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_chat
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.2
+  top_p: 1.0
+  num_beams: 1
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Give a detail answer to the question in English."
+metric_list:
+  - metric: gpt_eval
+    aggregation: !function utils.air_bench_aggregate_results_chat
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_chat
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_foundation.yaml

@@ -0,0 +1,5 @@
+group: air_bench_foundation
+task:
+  - air_bench_foundation_sound
+  - air_bench_foundation_music
+  - air_bench_foundation_speech

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_foundation_music.yaml

@@ -0,0 +1,26 @@
+task: "air_bench_foundation_music"
+dataset_name: "Foundation"
+test_split: music
+doc_to_target: !function utils.air_bench_doc_to_target_foundation
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_foundation
+doc_to_choice: !function utils.air_bench_doc_to_choice_foundation
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  do_sample: False
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+metric_list:
+  - metric: accuracy
+    aggregation: !function utils.air_bench_aggregate_results_foundation
+    higher_is_better: true
+  - metric: submission
+    aggregation: !function utils.air_bench_aggregate_results_for_submission
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_foundation
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_foundation_sound.yaml

@@ -0,0 +1,26 @@
+task: "air_bench_foundation_sound"
+dataset_name: "Foundation"
+test_split: sound
+doc_to_target: !function utils.air_bench_doc_to_target_foundation
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_foundation
+doc_to_choice: !function utils.air_bench_doc_to_choice_foundation
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  do_sample: False
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+metric_list:
+  - metric: accuracy
+    aggregation: !function utils.air_bench_aggregate_results_foundation
+    higher_is_better: true
+  - metric: submission
+    aggregation: !function utils.air_bench_aggregate_results_for_submission
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_foundation
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/air_bench_foundation_speech.yaml

@@ -0,0 +1,26 @@
+task: "air_bench_foundation_speech"
+dataset_name: "Foundation"
+test_split: speech
+doc_to_target: !function utils.air_bench_doc_to_target_foundation
+doc_to_visual: !function utils.air_bench_doc_to_audio
+doc_to_text: !function utils.air_bench_doc_to_text_foundation
+doc_to_choice: !function utils.air_bench_doc_to_choice_foundation
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  do_sample: False
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+metric_list:
+  - metric: accuracy
+    aggregation: !function utils.air_bench_aggregate_results_foundation
+    higher_is_better: true
+  - metric: submission
+    aggregation: !function utils.air_bench_aggregate_results_for_submission
+    higher_is_better: true
+
+process_results: !function utils.air_bench_process_results_foundation
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/air_bench/utils.py

@@ -0,0 +1,281 @@
+import datetime
+import json
+import os
+import random
+import re
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import requests
+import yaml
+from loguru import logger as eval_logger
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+
+def air_bench_doc_to_audio(doc):
+    return [doc["audio"]]
+
+
+def air_bench_doc_to_text_chat(doc, lmms_eval_specific_kwargs):
+    question = doc["question"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+# specify api type and key in .env
+GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"]
+API_TYPE = os.getenv("API_TYPE", "azure")
+
+if API_TYPE == "openai":
+    API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+    API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "Authorization": f"Bearer {API_KEY}",
+        "Content-Type": "application/json",
+    }
+elif API_TYPE == "azure":
+    API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
+    API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "api-key": API_KEY,
+        "Content-Type": "application/json",
+    }
+
+# prompt taken from the AIR-Bench repo
+eval_prompt = (
+    "You are a helpful and precise assistant for checking the quality of the answer.\n"
+    "[Detailed Audio Description]\nXAudioX\n[Question]\nXQuestionX\n"
+    "[The Start of Assistant 1s Answer]\nXAssistant1X\n[The End of Assistant 1s Answer]\n"
+    "[The Start of Assistant 2s Answer]\nXAssistant2X\n[The End of Assistant 2s Answer]\n[System]\n"
+    "We would like to request your feedback on the performance of two AI assistants in response to the user question "
+    "and audio description displayed above. AI assistants are provided with detailed audio descriptions and questions.\n"
+    "Please rate the helpfulness, relevance, accuracy, and comprehensiveness of their responses. "
+    "Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance. "
+    "Please output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. "
+    "The two scores are separated by a space. Please only output the 2 required number and no text."
+)
+
+retries = 3
+NUM_SECONDS_TO_SLEEP = 5
+
+
+def get_eval(max_tokens: int, content: str, retries: int = retries):
+    global headers
+
+    messages = [
+        {"role": "user", "content": content},
+    ]
+
+    payload = {
+        "model": GPT_EVAL_MODEL_NAME,
+        "messages": messages,
+        "temperature": 0,
+        "max_tokens": max_tokens,
+    }
+
+    for attempt in range(retries):
+        try:
+            response = requests.post(API_URL, headers=headers, json=payload, timeout=60)
+            response.raise_for_status()
+            response_data = response.json()
+
+            content = response_data["choices"][0]["message"]["content"].strip()
+            if content != "":
+                return content, response_data["model"]
+            break  # If successful, break out of the loop
+
+        except Exception as e:
+            eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}")
+            if attempt < retries:  # If we have retries left, sleep and then continue to next attempt
+                time.sleep(NUM_SECONDS_TO_SLEEP)
+            else:  # If this was the last attempt, log and return empty
+                eval_logger.error(f"All {retries} attempts failed. Last error message: {e}")
+                return "", ""
+    return "", ""
+
+
+def air_bench_process_results_chat(doc, result):
+    path = doc["path"]
+    question = doc["question"]
+    answer_gt = doc["answer_gt"]
+    task_name = doc["task_name"]
+    dataset_name = doc["dataset_name"]
+    response = result[0]
+
+    if response == None:
+        exit(1)
+
+    if doc["meta_info"] == None:
+        print("lack meta info")
+        exit(1)
+    else:
+        meta_info = doc["meta_info"]
+
+    # Get the evaluation score 2 times: one with ourmodel as assistant 1 and the other with our model as assistant 2 to prevent position bias
+    content = eval_prompt.replace("XAudioX", meta_info).replace("XQuestionX", question).replace("XAssistant1X", answer_gt).replace("XAssistant2X", response)
+    eval_answer, model_name = get_eval(max_tokens=1024, content=content)
+    content = eval_prompt.replace("XAudioX", meta_info).replace("XQuestionX", question).replace("XAssistant1X", response).replace("XAssistant2X", answer_gt)
+    eval_answer2, model_name2 = get_eval(max_tokens=1024, content=content)
+
+    return {
+        "gpt_eval": {
+            "eval_answer": [eval_answer, eval_answer2],
+            "model_name": model_name,
+        },
+    }
+
+
+def air_bench_aggregate_results_chat(results):
+    score = 0
+    for result in results:
+        eval_answer = result["eval_answer"]
+        eval_answer = [eval_answer[i].strip().replace(".", "") for i in range(len(eval_answer))]
+        pattern = r"\b(?:[1-9]|10)\b"
+
+        # Find all matches
+        try:
+            matches1 = re.findall(pattern, eval_answer[0])
+            matches2 = re.findall(pattern, eval_answer[1])
+            # Get the first two occurrences
+            eval_score = float(matches1[1]) + float(matches2[0])
+            score += eval_score
+        except Exception as e:
+            eval_logger.error(f"Error parsing eval_score: {e}")
+            continue
+
+    return score / (2 * len(results))
+
+
+# Functions for Foundation tasks
+
+
+def air_bench_doc_to_text_foundation(doc, lmms_eval_specific_kwargs):
+    question = doc["question"]
+    answers = [doc["choice_a"], doc["choice_b"], doc.get("choice_c", None), doc.get("choice_d", None)]
+    question = f"{question}\nA. {answers[0]}\nB. {answers[1]}\nC. {answers[2]}\nD. {answers[3]}\n"
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+def air_bench_doc_to_target_foundation(doc):
+    return get_gt(doc)
+
+
+def air_bench_doc_to_choice_foundation(doc):
+    choices = []
+    for option in ["choice_a", "choice_b", "choice_c", "choice_d"]:
+        if doc.get(option) and doc[option].strip():
+            choices.append(option[-1].upper())
+    return choices
+
+
+def air_bench_process_results_foundation(doc, result):
+    response = result[0].strip()
+    all_choices = [choice[-1].upper() for choice in ["choice_a", "choice_b", "choice_c", "choice_d"] if doc.get(choice)]
+    pred = parse_multi_choice_response(response, all_choices)  # AdaptfromMMMU
+    gt_ans = get_gt(doc)
+    score = 1.0 if pred == gt_ans else 0.0
+    submission_dict = {}
+    submission_dict = {doc.get("uniq_id", "unknown"): pred}
+    return {
+        "accuracy": {"score": score, "task": doc["task_name"]},
+        "submission": submission_dict,
+    }
+
+
+def air_bench_aggregate_results_for_submission(results, args):
+    path = generate_submission_file("air_bench_test_submission.json", args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    eval_logger.info(f"Results saved to {path}.")
+
+
+def air_bench_aggregate_results_foundation(results):
+    score = 0
+    categorical_correct = {}
+    categorical_total = {}
+    for result in results:
+        score += result["score"]
+        if result["task"] not in categorical_correct.keys():
+            categorical_correct[result["task"]] = 0
+            categorical_total[result["task"]] = 0
+        categorical_correct[result["task"]] += result["score"]
+        categorical_total[result["task"]] += 1
+
+    overall_accuracy = round(score / len(results), 5)
+    categorical_accuracy = {task: round(categorical_correct[task] / categorical_total[task], 5) for task in categorical_correct.keys()}
+
+    eval_logger.info("=" * 50)
+    eval_logger.info(f"Overall accuracy: {overall_accuracy}")
+    for task, acc in categorical_accuracy.items():
+        eval_logger.info(f"{task} accuracy: {acc}")
+    eval_logger.info("=" * 50)
+    return overall_accuracy
+
+
+def parse_multi_choice_response(response, all_choices):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted choice letter e.g., A, B, C, D.
+    """
+    # Clean response of unwanted characters
+    for char in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(char)
+    response = " " + response + " "  # Add space to avoid partial match
+
+    candidates = []
+    # Look for choices with parentheses, e.g., (A)
+    for choice in all_choices:
+        if f"({choice})" in response:
+            candidates.append(choice)
+
+    # Look for simple choices, e.g., A, B, C
+    if len(candidates) == 0:
+        for choice in all_choices:
+            if f" {choice} " in response:
+                candidates.append(choice)
+
+    # Look for choices with periods, e.g., A., B., C.
+    if len(candidates) == 0:
+        for choice in all_choices:
+            if f"{choice}." in response:
+                candidates.append(choice)
+
+    # If no candidates, randomly choose one
+    if len(candidates) == 0:
+        pred_index = random.choice(all_choices)
+    elif len(candidates) > 1:
+        # If more than one candidate, choose the last one found
+        start_indexes = [response.rfind(f" {can} ") for can in candidates]
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:
+        # If only one candidate, use it
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+def get_gt(doc):
+    if doc["answer_gt"] == doc["choice_a"]:
+        return "A"
+    elif doc["answer_gt"] == doc["choice_b"]:
+        return "B"
+    elif doc["answer_gt"] == doc.get("choice_c", None):
+        return "C"
+    elif doc["answer_gt"] == doc.get("choice_d", None):
+        return "D"

LaViDa/eval/lmms_eval/tasks/arc/README.md

@@ -0,0 +1,58 @@
+# ARC
+
+### Paper
+
+Title: Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge
+
+Abstract: https://arxiv.org/abs/1803.05457
+
+The ARC dataset consists of 7,787 science exam questions drawn from a variety
+of sources, including science questions provided under license by a research
+partner affiliated with AI2. These are text-only, English language exam questions
+that span several grade levels as indicated in the files. Each question has a
+multiple choice structure (typically 4 answer options). The questions are sorted
+into a Challenge Set of 2,590 “hard” questions (those that both a retrieval and
+a co-occurrence method fail to answer correctly) and an Easy Set of 5,197 questions.
+
+Homepage: https://allenai.org/data/arc
+
+
+### Citation
+
+```
+@article{Clark2018ThinkYH,
+  title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
+  author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+  journal={ArXiv},
+  year={2018},
+  volume={abs/1803.05457}
+}
+```
+
+### Groups, Tags, and Tasks
+
+#### Groups
+
+None.
+
+#### Tags
+
+* `ai2_arc`: Evaluates `arc_easy` and `arc_challenge`
+
+#### Tasks
+
+* `arc_easy`
+* `arc_challenge`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

LaViDa/eval/lmms_eval/tasks/arc/arc_challenge.yaml

@@ -0,0 +1,3 @@
+include: arc_easy.yaml
+task: arc_challenge
+dataset_name: ARC-Challenge

LaViDa/eval/lmms_eval/tasks/arc/arc_easy.yaml

@@ -0,0 +1,23 @@
+tag:
+  - ai2_arc
+task: arc_easy
+dataset_path: allenai/ai2_arc
+dataset_name: ARC-Easy
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

LaViDa/eval/lmms_eval/tasks/av_odyssey/av_odyssey.yaml

@@ -0,0 +1,24 @@
+dataset_path: AV-Odyssey/AV_Odyssey_Bench_LMMs_Eval
+dataset_kwargs:
+  token: True
+  cache_dir: AV_Odyssey
+  video: True
+task: "av_odyssey"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.av_odyssey_doc_to_visual
+doc_to_text: !function utils.av_odyssey_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+# The return value of process_results will be used by metrics
+process_results: !function utils.av_odyssey_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: av_odyssey_score
+    aggregation: !function utils.av_odyssey_aggregate_results
+    higher_is_better: true

LaViDa/eval/lmms_eval/tasks/av_odyssey/utils.py

@@ -0,0 +1,264 @@
+import datetime
+import os
+import random
+import re
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import yaml
+from loguru import logger as eval_logger
+from PIL import Image
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+eval_type_dict = {
+    "Subfield": [
+        "Timbre",
+        "Tone",
+        "Melody",
+        "Space",
+        "Time",
+        "Hallucination",
+        "Intricacy",
+    ],
+}
+
+with open(Path(__file__).parent / "av_odyssey.yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+hf_home = os.getenv("HF_HOME", "~/.cache/huggingface/")
+cache_dir = os.path.join(hf_home, config["dataset_kwargs"]["cache_dir"])
+
+question_prompt = "Answer with the option's letter from the given choices directly."
+
+
+def split_media_tags(content):
+    pattern = r"\[(audio|video|img)(\d+)\]"
+
+    matches = list(re.finditer(pattern, content))
+    if not matches:
+        return [content]
+
+    result = []
+    last_end = 0
+
+    for match in matches:
+        if match.start() > last_end:
+            result.append(content[last_end : match.start()])
+
+        media_type = match.group(1)
+        media_num = int(match.group(2))
+        result.append((media_type, media_num))
+
+        last_end = match.end()
+
+    if last_end < len(content):
+        result.append(content[last_end:])
+
+    return result
+
+
+def av_odyssey_doc_to_visual(doc):
+    audio_data = []
+    image_data = []
+    video_data = []
+    result = []
+
+    # 处理 image 类型数据
+    if "image" in doc["data_type"]:
+        for relative_path in doc["image_path"]:
+            abs_path = os.path.join(cache_dir, relative_path)
+            if os.path.exists(abs_path):
+                image_data.append(abs_path)  # 保留路径以供后续处理
+            else:
+                print(f"Image path does not exist: {abs_path}")
+
+    # 处理 video 类型数据
+    elif "video" in doc["data_type"]:
+        for relative_path in doc["video_path"]:
+            abs_path = os.path.join(cache_dir, relative_path)
+            if os.path.exists(abs_path):
+                video_data.append(abs_path)  # 保留路径以供后续处理
+            else:
+                print(f"Video path does not exist: {abs_path}")
+
+    # 处理 audio 类型数据
+    for relative_path in doc["audio_path"]:
+        abs_path = os.path.join(cache_dir, relative_path)
+        if os.path.exists(abs_path):
+            audio_data.append(abs_path)  # 保留路径以供后续处理
+        else:
+            print(f"Audio path does not exist: {abs_path}")
+
+    question = get_text(doc)
+    for q in question:
+        if isinstance(q, str):
+            continue
+        else:
+            media_type, media_num = q
+            media_num = media_num - 1
+            if media_type == "audio":
+                result.append(audio_data[media_num])
+            elif media_type == "video":
+                result.append(video_data[media_num])
+            elif media_type == "img":
+                result.append(image_data[media_num])
+
+    return result
+
+
+def get_text(doc):
+    question = doc["question"]
+    options = doc["options"]
+    option_text = options[0] + "\n" + options[1] + "\n" + options[2] + "\n" + options[3] + "\n"
+    text = question + "\n" + option_text + question_prompt
+    return split_media_tags(text)
+
+
+def av_odyssey_doc_to_text(doc, lmms_eval_specific_kwargs=None):
+    text = get_text(doc)
+    id = 0
+    result = []
+    for t in text:
+        if isinstance(t, str):
+            result.append(t)
+        else:
+            result.append(f"<media_{id}>")
+            id += 1
+    return "".join(result)
+
+
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index e.g., A, B, C, D.
+    """
+    for char in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(char)
+    response = " " + response + " "  # add space to avoid partial match
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f"{choice}" in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A B C D
+            if f" {choice} " in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        # pred_index = random.choice(all_choices)
+        pred_index = "A"
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack:
+                for can in candidates:
+                    index = response.rfind(f"({can})")
+                    start_indexes.append(index)  # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f" {can} ")
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:  # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+def av_odyssey_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case av_odyssey score), value: metric value
+    """
+    pred = results[0]
+    options = doc["options"]
+    option_list = {"A": options[0][3:], "B": options[1][3:], "C": options[2][3:], "D": options[3][3:]}
+    answer = parse_multi_choice_response(pred, ["A", "B", "C", "D"], option_list)
+    gt_answer = doc["answer"]
+    assert answer in ["A", "B", "C", "D"]
+    assert gt_answer in ["A", "B", "C", "D"]
+    score = 1.0 if answer == gt_answer else 0.0
+    category = doc["subfield"]
+    key_name = "av_odyssey_score"
+    # Note: the key name here is very important. It decides which aggregation function will receive the results
+    # We note down the question id/category to help us aggregate the results later
+    return {key_name: {"question_id": doc["question_id"], "category": category, "score": score}}
+
+
+def av_odyssey_aggregate_results(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    category2score = defaultdict(dict)
+    for result in results:
+        question_id = result["question_id"]
+        score = result["score"]
+        category = result["category"]
+        if question_id not in category2score[category]:
+            category2score[category][question_id] = []
+        category2score[category][question_id].append(score)
+
+    # 计算每个 category 的平均分
+    category_avg_scores = {}
+    total_score = 0
+    total_questions = 0
+
+    # 遍历所有 category 来计算每个 category 的平均分
+    for category, questions in category2score.items():
+        # import pdb
+        # pdb.set_trace()
+        category_total = 0  # 计算所有问题的总分
+        for question_id, score in questions.items():
+            category_total += score[0]  # 累加所有问题的平均分
+        category_avg_scores[category] = category_total / len(questions) * 100.0  # 当前类别的平均分
+
+        total_score += category_total  # 累加所有类别的问题总分
+        total_questions += len(questions)  # 累加所有问题的数量
+
+    # 计算所有问题的平均分（按问题的总数来平均）
+    overall_avg_score = total_score / total_questions * 100.0
+
+    # 输出每个 category 的平均分
+    print("Average scores per category:")
+    for category, avg_score in category_avg_scores.items():
+        print(f"{category}: {avg_score:.2f}")
+
+    # 输出所有问题的平均分
+    print(f"Overall average score (across all questions): {overall_avg_score:.2f}")
+
+    return overall_avg_score

LaViDa/eval/lmms_eval/tasks/chartqa/chartqa.yaml

@@ -0,0 +1,34 @@
+dataset_path: lmms-lab/ChartQA
+dataset_kwargs:
+  token: True
+task: "chartqa"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.chartqa_doc_to_visual
+doc_to_text: !function utils.chartqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+process_results: !function utils.chartqa_process_results
+metric_list:
+  - metric: relaxed_overall
+    aggregation: mean
+    higher_is_better: true
+  - metric: relaxed_human_split
+    aggregation: mean
+    higher_is_better: true
+  - metric: relaxed_augmented_split
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question with a single word."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+

LaViDa/eval/lmms_eval/tasks/chartqa/chartqa_lite.yaml

@@ -0,0 +1,35 @@
+dataset_path: lmms-lab/LMMs-Eval-Lite
+dataset_kwargs:
+  token: True
+task: "chartqa_lite"
+dataset_name: chartqa
+test_split: lite
+output_type: generate_until
+doc_to_visual: !function utils.chartqa_doc_to_visual
+doc_to_text: !function utils.chartqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+process_results: !function utils.chartqa_process_results
+metric_list:
+  - metric: relaxed_overall
+    aggregation: mean
+    higher_is_better: true
+  - metric: relaxed_human_split
+    aggregation: mean
+    higher_is_better: true
+  - metric: relaxed_augmented_split
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question with a single word."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+

LaViDa/eval/lmms_eval/tasks/chartqa/upload_chartqa.py

@@ -0,0 +1,129 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+
+import datasets
+from PIL import Image
+
+# Find for instance the citation on arxiv or on the dataset repo/website
+_CITATION = """\
+@inproceedings{masry-etal-2022-chartqa,
+    title = "{C}hart{QA}: A Benchmark for Question Answering about Charts with Visual and Logical Reasoning",
+    author = "Masry, Ahmed  and
+      Long, Do  and
+      Tan, Jia Qing  and
+      Joty, Shafiq  and
+      Hoque, Enamul",
+    booktitle = "Findings of the Association for Computational Linguistics: ACL 2022",
+    month = may,
+    year = "2022",
+    address = "Dublin, Ireland",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2022.findings-acl.177",
+    doi = "10.18653/v1/2022.findings-acl.177",
+    pages = "2263--2279",
+}
+"""
+_DESCRIPTION = "A largescale benchmark covering 9.6K human-written questions as well as 23.1K questions generated from human-written chart summaries."
+
+
+def get_builder_config(VERSION):
+    builder_config = [
+        datasets.BuilderConfig(
+            name=f"ChartQA",
+            version=VERSION,
+            description=f"ChartQA",
+        )
+    ]
+    return builder_config
+
+
+dataset_features = {
+    "type": datasets.Value("string"),
+    "question": datasets.Value("string"),
+    "answer": datasets.Value("string"),
+    "image": datasets.Image(),
+}
+
+
+class ChartQA(datasets.GeneratorBasedBuilder):
+    VERSION = datasets.Version("1.0.0")
+
+    BUILDER_CONFIGS = get_builder_config(VERSION)
+
+    def _info(self):
+        features = datasets.Features(dataset_features)
+        return datasets.DatasetInfo(
+            # This is the description that will appear on the datasets page.
+            description=_DESCRIPTION,
+            # This defines the different columns of the dataset and their types
+            features=features,  # Here we define them above because they are different between the two configurations
+            # If there's a common (input, target) tuple from the features, uncomment supervised_keys line below and
+            # specify them. They'll be used if as_supervised=True in builder.as_dataset.
+            # supervised_keys=("sentence", "label"),
+            # Homepage of the dataset for documentation
+            # Citation for the dataset
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name
+
+        # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
+        # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
+        # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
+        image_path = "/home/yhzhang/ChartQA/ChartQA Dataset/test/png/"
+        human_annotation_path = "/home/yhzhang/ChartQA/ChartQA Dataset/test/test_human.json"
+        augmented_annotation_path = "/home/yhzhang/ChartQA/ChartQA Dataset/test/test_augmented.json"
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TEST,
+                gen_kwargs={
+                    "human_annotation": human_annotation_path,
+                    "augmented_annotation": augmented_annotation_path,
+                    "images": image_path,
+                },
+            ),
+        ]
+
+    # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
+    def _generate_examples(self, human_annotation, augmented_annotation, images):
+        # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
+        with open(human_annotation, encoding="utf-8") as f:
+            human_data = json.load(f)
+        with open(augmented_annotation, encoding="utf-8") as f:
+            augmented_data = json.load(f)
+        index = -1
+        for data in [human_data, augmented_data]:
+            for row in data["data"]:
+                index += 1
+                image_path = os.path.join(images, row["imgname"])
+                now_data = {}
+                now_data["type"] = "human_test" if data == human_data else "augmented_test"
+                now_data["image"] = Image.open(image_path)
+                now_data["question"] = row["query"]
+                now_data["answer"] = row["label"]
+                yield index, now_data
+
+
+if __name__ == "__main__":
+    from datasets import load_dataset
+
+    data = load_dataset(
+        "/home/yhzhang/lmms-eval/lmms_eval/tasks/chartqa/upload_chartqa.py",
+    )
+    data.push_to_hub("lmms-lab/chartqa", private=True)

LaViDa/eval/lmms_eval/tasks/chartqa/utils.py

@@ -0,0 +1,62 @@
+def chartqa_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def chartqa_doc_to_text(doc, lmms_eval_specific_kwargs):
+    question = doc["question"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+def chartqa_process_results(doc, results):
+    pred = results[0]
+    type = doc["type"]
+    score = relaxed_correctness(pred, doc["answer"])
+    score = 1.0 if score else 0.0
+    return_dict = {"relaxed_overall": score}
+    if type == "human_test":
+        return_dict["relaxed_human_split"] = score
+    else:
+        return_dict["relaxed_augmented_split"] = score
+    return return_dict
+
+
+def relaxed_correctness(prediction, target, max_relative_change: float = 0.05) -> bool:
+    """Calculates relaxed correctness.
+
+    The correctness tolerates certain error ratio defined by max_relative_change.
+    See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1:
+    “Following Methani et al. (2020), we use a relaxed accuracy measure for the
+    numeric answers to allow a minor inaccuracy that may result from the automatic
+    data extraction process. We consider an answer to be correct if it is within
+    5% of the gold answer. For non-numeric answers, we still need an exact match
+    to consider an answer to be correct.”
+
+    This funcion is taken from https://github.com/QwenLM/Qwen-VL/blob/34b4c0ee7b07726371b960911f249fe61b362ca3/eval_mm/evaluate_vqa.py#L113
+    Args:
+      target: List of target string.
+      prediction: List of predicted string.
+      max_relative_change: Maximum relative change.
+
+    Returns:
+      Whether the prediction was correct given the specified tolerance.
+    """
+
+    def _to_float(text: str):
+        try:
+            if text.endswith("%"):
+                # Convert percentages to floats.
+                return float(text.rstrip("%")) / 100.0
+            else:
+                return float(text)
+        except ValueError:
+            return None
+
+    prediction_float = _to_float(prediction)
+    target_float = _to_float(target)
+    if prediction_float is not None and target_float:
+        relative_change = abs(prediction_float - target_float) / abs(target_float)
+        return relative_change <= max_relative_change
+    else:
+        return prediction.lower() == target.lower()

LaViDa/eval/lmms_eval/tasks/clotho_aqa/_default_template_yaml

@@ -0,0 +1,10 @@
+dataset_path: lmms-lab/ClothoAQA
+dataset_kwargs:
+  token: True
+doc_to_target: "answer"
+doc_to_visual: !function utils.clotho_aqa_doc_to_audio
+doc_to_text: !function utils.clotho_aqa_doc_to_text
+
+metadata:
+  gpt_eval_model_name: gpt-4o
+  version: 0.0

LaViDa/eval/lmms_eval/tasks/clotho_aqa/clotho_aqa.yaml

@@ -0,0 +1,4 @@
+group: clotho_aqa
+task:
+  - clotho_aqa_val
+  - clotho_aqa_test

LaViDa/eval/lmms_eval/tasks/clotho_aqa/clotho_aqa_test.yaml

@@ -0,0 +1,19 @@
+task: "clotho_aqa_test"
+dataset_name: "clotho_aqa"
+test_split: clotho_aqa_test_filtered
+generation_kwargs:
+  max_new_tokens: 8
+  temperature: 0
+  do_sample: False
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word only. "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/clotho_aqa/clotho_aqa_val.yaml

@@ -0,0 +1,19 @@
+task: "clotho_aqa_val"
+dataset_name: "clotho_aqa"
+test_split: clotho_aqa_val_filtered
+generation_kwargs:
+  max_new_tokens: 8
+  temperature: 0
+  do_sample: False
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word only. "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/clotho_aqa/clotho_asqa_test_v2.yaml

@@ -0,0 +1,19 @@
+task: "clotho_asqa_test_v2"
+dataset_name: "clotho_asqa_test_v2"
+test_split: test
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  do_sample: False
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
+metric_list:
+  - metric: gpt_eval
+    aggregation: !function utils.clotho_aqa_v2_aggregate_results
+    higher_is_better: true
+
+process_results: !function utils.clotho_aqa_v2_process_results
+
+include: _default_template_yaml

LaViDa/eval/lmms_eval/tasks/clotho_aqa/utils.py

@@ -0,0 +1,142 @@
+import datetime
+import json
+import os
+import re
+import sys
+import time
+from pathlib import Path
+
+import requests
+import yaml
+from loguru import logger as eval_logger
+
+import lmms_eval.tasks._task_utils.file_utils as file_utils
+from lmms_eval.filters.extraction import ExtendedRegexFilter
+
+
+def clotho_aqa_doc_to_audio(doc):
+    return [doc["audio"]]
+
+
+def clotho_aqa_doc_to_text(doc, lmms_eval_specific_kwargs):
+    question = doc["question"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+# functions for the clotho_asqa_v2 task, need to be tested later
+
+with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+
+NUM_SECONDS_TO_SLEEP = 2
+GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"]
+API_TYPE = os.getenv("API_TYPE", "azure")
+
+if API_TYPE == "openai":
+    API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+    API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "Authorization": f"Bearer {API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+elif API_TYPE == "azure":
+    API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
+    API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "api-key": API_KEY,
+        "Content-Type": "application/json",
+    }
+
+eval_prompt = """
+            [Question]
+            {question}
+
+            [Reference Answer]
+            {ground_truth}
+
+            [Model Answer]
+            {model_response}
+
+            [Task]
+            Rate the model's answer based on its alignment with the reference answer, focusing on accuracy and relevance to the reference provided. Please be critical on the details.
+            Criteria: Assess if the model's response mirrors the reference in terms of content, accuracy, and relevance.
+            Score0: The answer is completely misaligned, providing incorrect or irrelevant information compared to the reference.
+            Score1: The answer shows minimal alignment, often misunderstanding or providing irrelevant details unrelated to the reference.
+            Score2: The answer recognizes the topic but diverges significantly from the reference in accuracy or relevance.
+            Score3: The answer aligns with the reference generally but lacks detail or precise accuracy in some aspects.
+            Score4: The answer is mostly accurate and relevant, closely following the reference but could be clearer or more detailed.
+            Score5: The answer is highly accurate, detailed, and matches the reference answer perfectly, capturing its essence and detail.
+
+            Your response should be formatted as follows:
+            Explanation: (Provide a concise explanation of your rating, comparing the reference answer with the model's response. "The reference answer is [XXX], while the model's answer is [YYY]. I think ...")
+            Rating: (int)"""
+
+
+retries = 3
+NUM_SECONDS_TO_SLEEP = 5
+
+
+def get_eval(max_tokens: int, content: str, retries: int = retries):
+    global headers
+
+    messages = [
+        {"role": "user", "content": content},
+    ]
+
+    payload = {"model": GPT_EVAL_MODEL_NAME, "messages": messages, "temperature": 0.7, "max_tokens": max_tokens, "top_p": 0.95, "frequency_penalty": 0, "presence_penalty": 0, "stop": None}
+
+    for attempt in range(retries):
+        try:
+            response = requests.post(API_URL, headers=headers, json=payload, timeout=60)
+            response.raise_for_status()
+            response_data = response.json()
+
+            content = response_data["choices"][0]["message"]["content"].strip()
+            if content != "":
+                return content, response_data["model"]
+            break  # If successful, break out of the loop
+
+        except Exception as e:
+            eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}")
+            if attempt < retries:  # If we have retries left, sleep and then continue to next attempt
+                time.sleep(NUM_SECONDS_TO_SLEEP)
+            else:  # If this was the last attempt, log and return empty
+                eval_logger.error(f"All {retries} attempts failed. Last error message: {e}")
+                return "", ""
+    return "", ""
+
+
+def clotho_aqa_v2_process_results(doc, result):
+    pred = result[0]
+    ground_truth_str = doc["answer"]
+    content = eval_prompt.format(model_response=pred, ground_truth=ground_truth_str, question=doc["question"])
+    eval_answer, model_name = get_eval(max_tokens=1024, content=content)
+    return {
+        "gpt_eval": {"eval_answer": eval_answer, "model_name": model_name},
+    }
+
+
+def clotho_aqa_v2_aggregate_results(results):
+    score = 0
+    for result in results:
+        eval_answer = result["eval_answer"]
+        eval_score = re.search(r"([0-5])", eval_answer).group(1)
+        try:
+            eval_score = float(eval_score)
+        except Exception as e:
+            eval_logger.error(f"Error parsing eval_score: {e}")
+            eval_score = 0.0
+        score += eval_score
+
+    return score / len(results) * 20

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2014_cap.yaml

@@ -0,0 +1,4 @@
+group : coco2014_cap
+task:
+  - coco2014_cap_val
+  - coco2014_cap_test

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2014_cap_test.yaml

@@ -0,0 +1,24 @@
+dataset_path: lmms-lab/COCO-Caption
+dataset_kwargs:
+  token: True
+task : "coco2014_cap_test"
+group : "coco_caption"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.coco_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.coco_test_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: coco_passthrough 
+    aggregation : !function utils.coco_test_aggregation_result
+    higher_is_better : true
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2014_cap_val.yaml

@@ -0,0 +1,45 @@
+dataset_path: lmms-lab/COCO-Caption
+dataset_kwargs:
+  token: True
+task: "coco2014_cap_val"
+group : "coco_caption"
+test_split: val
+output_type: generate_until
+doc_to_visual: !function utils.coco_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 32
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.coco_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: coco_Bleu_4 
+    aggregation : !function utils.coco_bleu4
+    higher_is_better : true
+  - metric: coco_Bleu_3
+    aggregation : !function utils.coco_bleu3
+    higher_is_better : true
+  - metric: coco_Bleu_2
+    aggregation : !function utils.coco_bleu2
+    higher_is_better : true
+  - metric: coco_Bleu_1
+    aggregation : !function utils.coco_bleu1
+    higher_is_better : true
+  - metric: coco_METEOR
+    aggregation : !function utils.coco_meteor
+    higher_is_better : true
+  - metric: coco_ROUGE_L
+    aggregation : !function utils.coco_rougel
+    higher_is_better : true
+  - metric: coco_CIDEr
+    aggregation : !function utils.coco_cider
+    higher_is_better : true
+  #- metric: coco_SPICE
+  #  aggregation : !function utils.coco_spice
+  #  higher_is_better : true
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2017_cap.yaml

@@ -0,0 +1,4 @@
+group : coco2017_cap
+task:
+  - coco2017_cap_val
+  - coco2017_cap_test

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2017_cap_test.yaml

@@ -0,0 +1,24 @@
+dataset_path: lmms-lab/COCO-Caption2017
+dataset_kwargs:
+  token: True
+task : "coco2017_cap_test"
+group : "coco_caption2017"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.coco_doc_to_visual
+doc_to_text: !function utils.coco_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.coco_test_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: coco_passthrough 
+    aggregation : !function utils.coco_test_aggregation_result
+    higher_is_better : true
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2017_cap_val.yaml

@@ -0,0 +1,45 @@
+dataset_path: lmms-lab/COCO-Caption2017
+dataset_kwargs:
+  token: True
+task: "coco2017_cap_val"
+group : "coco_caption2017"
+test_split: val
+output_type: generate_until
+doc_to_visual: !function utils.coco_doc_to_visual
+doc_to_text: !function utils.coco_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.coco_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: coco_Bleu_4 
+    aggregation : !function utils.coco_bleu4
+    higher_is_better : true
+  - metric: coco_Bleu_3
+    aggregation : !function utils.coco_bleu3
+    higher_is_better : true
+  - metric: coco_Bleu_2
+    aggregation : !function utils.coco_bleu2
+    higher_is_better : true
+  - metric: coco_Bleu_1
+    aggregation : !function utils.coco_bleu1
+    higher_is_better : true
+  - metric: coco_METEOR
+    aggregation : !function utils.coco_meteor
+    higher_is_better : true
+  - metric: coco_ROUGE_L
+    aggregation : !function utils.coco_rougel
+    higher_is_better : true
+  - metric: coco_CIDEr
+    aggregation : !function utils.coco_cider
+    higher_is_better : true
+  #- metric: coco_SPICE
+  #  aggregation : !function utils.coco_spice
+  #  higher_is_better : true
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/coco_cap/coco2017_cap_val_lite.yaml

@@ -0,0 +1,45 @@
+dataset_path: lmms-lab/LMMs-Eval-Lite
+dataset_kwargs:
+  token: True
+task: "coco2017_cap_val_lite"
+dataset_name: coco2017_cap_val
+test_split: lite
+output_type: generate_until
+doc_to_visual: !function utils.coco_doc_to_visual
+doc_to_text: !function utils.coco_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 32
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.coco_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: coco_Bleu_4 
+    aggregation : !function utils.coco_bleu4
+    higher_is_better : true
+  - metric: coco_Bleu_3
+    aggregation : !function utils.coco_bleu3
+    higher_is_better : true
+  - metric: coco_Bleu_2
+    aggregation : !function utils.coco_bleu2
+    higher_is_better : true
+  - metric: coco_Bleu_1
+    aggregation : !function utils.coco_bleu1
+    higher_is_better : true
+  - metric: coco_METEOR
+    aggregation : !function utils.coco_meteor
+    higher_is_better : true
+  - metric: coco_ROUGE_L
+    aggregation : !function utils.coco_rougel
+    higher_is_better : true
+  - metric: coco_CIDEr
+    aggregation : !function utils.coco_cider
+    higher_is_better : true
+  #- metric: coco_SPICE
+  #  aggregation : !function utils.coco_spice
+  #  higher_is_better : true
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/coco_cap/coco_cap.yaml

@@ -0,0 +1,6 @@
+group : coco_cap
+task:
+  - coco2014_cap_val
+  - coco2014_cap_test
+  - coco2017_cap_val
+  - coco2017_cap_test

LaViDa/eval/lmms_eval/tasks/coco_cap/utils.py

@@ -0,0 +1,155 @@
+import json
+import os
+
+from loguru import logger as eval_logger
+from pycocoevalcap.eval import Bleu, Cider, COCOEvalCap, Meteor, Rouge, Spice
+from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
+from pycocotools.coco import COCO
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+COCO_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"]  # , "SPICE"]
+
+
+def coco_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def coco_doc_to_text(doc):
+    return f"Provide a one-sentence caption for the provided image."
+
+
+def coco_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name, value: metric value
+    """
+    pred = result[0] if len(result) > 0 else ""
+    question_id = doc["question_id"]
+    # The question id in our dataset is the image file itself
+    image_id = int(question_id.split("_")[-1].split(".")[0])
+    id = doc["id"]
+
+    data_dict = {"answer": doc["answer"], "pred": pred, "image_id": image_id, "id": id}
+
+    return {f"coco_{metric}": data_dict for metric in COCO_METRICS}
+
+
+def coco_aggregation_result(results, metric, args):
+    scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]  # , (Spice(), "SPICE")]
+    scorers_dict = {s[1]: s for s in scorers}
+
+    stored_results = []
+    # In order to make the coco eval tools to successfully create index
+    # We need at least two dict in the dataset
+    # 'annotation' and 'images'
+    # 'annotation' exactly reproduce the original annotation
+    # 'images' however only need the image id which is contained in the file name
+    dataset = {"annotations": [], "images": []}
+    idx = 0
+    for result in results:
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+        for a in result["answer"]:
+            dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx})
+            idx += 1
+        dataset["images"].append({"id": result["image_id"]})
+
+    coco = COCO()
+    # Manually create index here
+    coco.dataset = dataset
+    coco.createIndex()
+
+    coco_result = coco.loadRes(stored_results)
+    coco_eval = COCOEvalCap(coco, coco_result)
+
+    imgIds = coco_eval.params["image_id"]
+    gts = {}
+    res = {}
+    for imgId in imgIds:
+        gts[imgId] = coco_eval.coco.imgToAnns[imgId]
+        res[imgId] = coco_eval.cocoRes.imgToAnns[imgId]
+
+    eval_logger.info("tokenization...")
+    tokenizer = PTBTokenizer()
+    gts = tokenizer.tokenize(gts)
+    res = tokenizer.tokenize(res)
+
+    eval_logger.info(f"Computing {metric} scores...")
+
+    score, scores = scorers_dict[metric][0].compute_score(gts, res)
+    # When metric is one of the Bleu, score will be a list
+    if type(score) == list:
+        n = int(metric.split("_")[-1])
+        score = score[n - 1]
+
+    path = generate_submission_file("coco_captions_val2014_alg_results.json", args)
+    if not os.path.exists(path):
+        eval_logger.info("Storing prediction that can be submitted to the server ...")
+        with open(path, "w") as f:
+            json.dump(stored_results, f, indent=4)
+
+    return score
+
+
+def coco_bleu4(results, args):
+    return coco_aggregation_result(results, "Bleu_4", args)
+
+
+def coco_bleu3(results, args):
+    return coco_aggregation_result(results, "Bleu_3", args)
+
+
+def coco_bleu2(results, args):
+    return coco_aggregation_result(results, "Bleu_2", args)
+
+
+def coco_bleu1(results, args):
+    return coco_aggregation_result(results, "Bleu_1", args)
+
+
+def coco_meteor(results, args):
+    return coco_aggregation_result(results, "METEOR", args)
+
+
+def coco_rougel(results, args):
+    return coco_aggregation_result(results, "ROUGE_L", args)
+
+
+def coco_cider(results, args):
+    return coco_aggregation_result(results, "CIDEr", args)
+
+
+def coco_spice(results, args):
+    return coco_aggregation_result(results, "SPICE", args)
+
+
+def coco_test_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case coco_passthrough), value: metric value
+    """
+    question_id = doc["question_id"]
+    # The question id in our dataset is the image file itself
+    image_id = int(question_id.split("_")[-1].split(".")[0])
+    return {"coco_passthrough": {"pred": result, "image_id": image_id}}
+
+
+def coco_test_aggregation_result(results, args):
+    stored_results = []
+    for result in results:
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+
+    path = generate_submission_file("coco_captions_test2014_alg_results.json", args)
+    eval_logger.info("Storing prediction that can be submitted to the server ...")
+    with open(path, "w") as f:
+        json.dump(stored_results, f, indent=4)
+
+    eval_logger.info(f"Your test result has been stored in to {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.")

LaViDa/eval/lmms_eval/tasks/common_voice_15/_default_template_yaml

@@ -0,0 +1,21 @@
+dataset_path: lmms-lab/common_voice_15
+dataset_kwargs:
+  token: True
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.common_voice_15_doc_to_audio
+doc_to_text: !function utils.common_voice_15_doc_to_text
+doc_to_target: "sentence"
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.common_voice_15_process_result
+metric_list:
+  - metric: wer 
+    aggregation : !function utils.common_voice_15_wer
+    higher_is_better : false
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/common_voice_15/common_voice_15.yaml

@@ -0,0 +1,5 @@
+group: common_voice_15
+task:
+- common_voice_15_zh-CN
+- common_voice_15_en
+- common_voice_15_fr

LaViDa/eval/lmms_eval/tasks/common_voice_15/common_voice_15_en.yaml

@@ -0,0 +1,10 @@
+task : "common_voice_15_en"
+dataset_name: en
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
+  qwen2_audio:
+    pre_prompt: ""
+    post_prompt: " <|en|>"
+include : _default_template_yaml

LaViDa/eval/lmms_eval/tasks/common_voice_15/common_voice_15_fr.yaml

@@ -0,0 +1,10 @@
+task : "common_voice_15_fr"
+dataset_name: fr
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
+  qwen2_audio:
+    pre_prompt: ""
+    post_prompt: " <|fr|>"
+include : _default_template_yaml

LaViDa/eval/lmms_eval/tasks/common_voice_15/common_voice_15_zh-CN.yaml

@@ -0,0 +1,10 @@
+task : "common_voice_15_zh-CN"
+dataset_name: zh-CN
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
+  qwen2_audio:
+    pre_prompt: ""
+    post_prompt: " <|zh|>"
+include : _default_template_yaml

LaViDa/eval/lmms_eval/tasks/common_voice_15/utils.py

@@ -0,0 +1,182 @@
+import os
+import re
+import unicodedata
+
+import editdistance as ed
+import zhconv
+
+from lmms_eval.tasks.librispeech.cn_tn import TextNorm
+from lmms_eval.tasks.librispeech.whisper_normalizer.basic import BasicTextNormalizer
+from lmms_eval.tasks.librispeech.whisper_normalizer.english import EnglishTextNormalizer
+
+# ImportError: To support decoding audio files, please install 'librosa' and 'soundfile'.
+english_normalizer = EnglishTextNormalizer()
+chinese_normalizer = TextNorm(
+    to_banjiao=False,
+    to_upper=False,
+    to_lower=False,
+    remove_fillers=False,
+    remove_erhua=False,
+    check_chars=False,
+    remove_space=False,
+    cc_mode="",
+)
+basic_normalizer = BasicTextNormalizer()
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+
+def common_voice_15_doc_to_audio(doc):
+    return [doc["audio"]]
+
+
+def common_voice_15_doc_to_text(doc, lmms_eval_specific_kwargs):
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    return f"{pre_prompt}Please recognize the speech and only output the recognized content:{post_prompt}"
+
+
+def common_voice_15_process_result(doc, result):
+    pred = result[0] if len(result) > 0 else ""
+
+    gt = doc["sentence"]
+    source = doc["path"]
+    task = doc["locale"]
+
+    data_dict = {"gt": gt, "pred": pred, "source": source, "task": task}
+
+    return {"wer": data_dict}
+
+
+PUNCS = "!,.?;:"
+
+
+def remove_sp(text, language):
+    gt = re.sub(r"<\|.*?\|>", " ", text)
+    gt = re.sub(rf"\s+", r" ", gt)  # Replace consecutive spaces in the text with a single space.
+    gt = re.sub(f" ?([{PUNCS}])", r"\1", gt)
+    gt = gt.lstrip(" ")
+    if language == "zh-CN":
+        gt = re.sub(rf"\s+", r"", gt)
+    return gt
+
+
+class EvaluationTokenizer(object):
+    """A generic evaluation-time tokenizer, which leverages built-in tokenizers
+    in sacreBLEU (https://github.com/mjpost/sacrebleu). It additionally provides
+    lowercasing, punctuation removal and character tokenization, which are
+    applied after sacreBLEU tokenization.
+
+    Args:
+        tokenizer_type (str): the type of sacreBLEU tokenizer to apply.
+        lowercase (bool): lowercase the text.
+        punctuation_removal (bool): remove punctuation (based on unicode
+        category) from text.
+        character_tokenization (bool): tokenize the text to characters.
+    """
+
+    SPACE = chr(32)
+    SPACE_ESCAPE = chr(9601)
+    # ALL_TOKENIZER_TYPES = ChoiceEnum(["none", "13a", "intl", "zh", "ja-mecab"])
+
+    def __init__(
+        self,
+        tokenizer_type: str = "13a",
+        lowercase: bool = False,
+        punctuation_removal: bool = False,
+        character_tokenization: bool = False,
+    ):
+        from sacrebleu.tokenizers.tokenizer_13a import Tokenizer13a
+        from sacrebleu.tokenizers.tokenizer_char import TokenizerChar
+        from sacrebleu.tokenizers.tokenizer_intl import TokenizerV14International
+        from sacrebleu.tokenizers.tokenizer_ja_mecab import TokenizerJaMecab
+        from sacrebleu.tokenizers.tokenizer_none import NoneTokenizer
+        from sacrebleu.tokenizers.tokenizer_zh import TokenizerZh
+
+        TOKENIZERS = {
+            "none": NoneTokenizer,
+            "13a": Tokenizer13a,
+            "intl": TokenizerV14International,
+            "zh": TokenizerZh,
+            "ja-mecab": TokenizerJaMecab,
+            "char": TokenizerChar,
+        }
+
+        assert tokenizer_type in TOKENIZERS, f"{tokenizer_type}, {TOKENIZERS}"
+        self.lowercase = lowercase
+        self.punctuation_removal = punctuation_removal
+        self.character_tokenization = character_tokenization
+        self.tokenizer = TOKENIZERS[tokenizer_type]
+        # self.tokenizer = tokenizer_none
+
+    @classmethod
+    def remove_punctuation(cls, sent: str):
+        """Remove punctuation based on Unicode category."""
+        return cls.SPACE.join(t for t in sent.split(cls.SPACE) if not all(unicodedata.category(c)[0] == "P" for c in t))
+
+    def tokenize(self, sent: str):
+        tokenized = self.tokenizer()(sent)
+
+        if self.punctuation_removal:
+            tokenized = self.remove_punctuation(tokenized)
+
+        if self.character_tokenization:
+            tokenized = self.SPACE.join(list(tokenized.replace(self.SPACE, self.SPACE_ESCAPE)))
+
+        if self.lowercase:
+            tokenized = tokenized.lower()
+
+        return tokenized
+
+
+def compute_wer(refs, hyps, language):
+    distance = 0
+    ref_length = 0
+    tokenizer = EvaluationTokenizer(
+        tokenizer_type="none",
+        lowercase=True,
+        punctuation_removal=True,
+        character_tokenization=False,
+    )
+    for i in range(len(refs)):
+        ref = refs[i]
+        pred = hyps[i]
+        if language in ["yue"]:
+            ref = zhconv.convert(ref, "zh-cn")
+            pred = zhconv.convert(pred, "zh-cn")
+        if language in ["en"]:
+            ref = english_normalizer(ref)
+            pred = english_normalizer(pred)
+        if language in ["zh-CN"]:
+            ref = chinese_normalizer(ref)
+            pred = chinese_normalizer(pred)
+        else:
+            ref = basic_normalizer(ref)
+            pred = basic_normalizer(pred)
+        ref_items = tokenizer.tokenize(ref).split()
+        pred_items = tokenizer.tokenize(pred).split()
+        if language in ["zh-CN", "yue"]:
+            ref_items = [x for x in "".join(ref_items)]
+            pred_items = [x for x in "".join(pred_items)]
+        if i == 0:
+            print(f"ref: {ref}")
+            print(f"pred: {pred}")
+            print(f"ref_items:\n{ref_items}\n{len(ref_items)}\n{ref_items[0]}")
+            print(f"pred_items:\n{pred_items}\n{len(ref_items)}\n{ref_items[0]}")
+        distance += ed.eval(ref_items, pred_items)
+        ref_length += len(ref_items)
+    return distance / ref_length
+
+
+def common_voice_15_wer(results, args):
+    refs, hyps = [], []
+    for result in results:
+        lan = result["task"]
+        gt = result["gt"]
+        response = result["pred"]
+        gt = remove_sp(gt, lan)
+        response = remove_sp(response, lan)
+        refs.append(gt)
+        hyps.append(response)
+    wer = compute_wer(refs, hyps, lan)
+    return wer * 100

LaViDa/eval/lmms_eval/tasks/conbench/conbench.yaml

@@ -0,0 +1,24 @@
+dataset_path: ConBench/ConBench_D
+dataset_kwargs:
+  token: True
+task: "ConBench"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.conbench_doc_to_visual
+doc_to_text: !function utils.conbench_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.2
+  top_p: 0
+  num_beams: 1
+  do_sample: True
+# The return value of process_results will be used by metrics
+process_results: !function utils.conbench_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: ConScore_D
+    aggregation: !function utils.conbench_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0

LaViDa/eval/lmms_eval/tasks/conbench/utils.py

@@ -0,0 +1,96 @@
+import os
+from collections import defaultdict
+
+from anls import anls_score
+from loguru import logger as eval_logger
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+# 19 classes
+eval_type_dict = {
+    "Sensation": ["count", "color", "scene", "poster", "attribute_recognition", "ocr", "position"],
+    "Cognition": ["calculation", "code", "translation", "math", "cross_instance_reason", "attribute_reason"],
+    "Knowledge": ["celebrity", "chemistry", "physics", "biology", "landmark", "artwork"],
+}
+
+
+def conbench_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def conbench_doc_to_text(doc):
+    question = doc["question"].strip()
+    return question
+
+
+def parse_pred_ans_NY(pred_ans):
+    pred_label = None
+    if pred_ans in ["yes", "no"]:
+        pred_label = pred_ans
+    else:
+        prefix_pred_ans = pred_ans[:4]
+
+        if "yes" in prefix_pred_ans:
+            pred_label = "yes"
+        elif "no" in prefix_pred_ans:
+            pred_label = "no"
+        else:
+            pred_label = "other"
+    return pred_label
+
+
+def parse_pred_ans_choice(pred_ans):
+    return pred_ans.replace(" ", "")[0]
+
+
+def conbench_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case mme score), value: metric value
+    """
+    pred = results[0]
+    pred = pred.replace("\n", "").lower()
+    # parser
+    if doc["question_field"] == "N/Y":
+        pred_ans = parse_pred_ans_NY(pred)
+    elif doc["question_field"] == "Choices":
+        pred_ans = parse_pred_ans_choice(pred)
+    else:
+        pred_ans = pred
+
+    gt_ans = doc["answer"].lower()
+
+    # score
+    score = 1 if (doc["question_field"] == "Q/A" and anls_score(prediction=pred_ans, gold_labels=[gt_ans], threshold=0.95) >= 0.4) or (gt_ans == pred_ans) else 0
+    # Note: the key name here is very important. It decides which aggregation function will receive the results
+    # We note down the question id/category to help us aggregate the results later
+    return {"ConScore_D": {"image_id": doc["image_id"], "question_field": doc["question_field"], "score": score}}
+
+
+def conbench_aggregate_results(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    summary = defaultdict(dict)
+    for result in results:
+        image_id = result["image_id"]
+        score = result["score"]
+        if image_id not in summary.keys():
+            summary[image_id] = 0
+        summary[image_id] += score
+
+    cnt_con = 0
+    for image_id, score in summary.items():
+        if score == 3:
+            cnt_con += 1
+
+    print("Consistency Cases are ", cnt_con)
+    cnt_con = cnt_con / (len(results) / 3)
+    eval_logger.info(f"ConScore_D: {cnt_con:.2f}")
+    return cnt_con

LaViDa/eval/lmms_eval/tasks/covost2/_default_template_en_zh_yaml

@@ -0,0 +1,28 @@
+dataset_path: lmms-lab/covost2_en-zh
+dataset_name: en_zh
+dataset_kwargs:
+  token: True
+output_type: generate_until
+doc_to_visual: !function utils.covost2_doc_to_audio
+doc_to_text: !function utils.covost2_doc_to_text
+doc_to_target: "translation"
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.covost2_en_zh_process_result
+metric_list:
+  - metric: bleu
+    aggregation: !function utils.covost2_bleu
+    higher_is_better: true
+metadata:
+  - version: 0.0
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the speech into Chinese and only output the translation:"
+    post_prompt: ""
+  qwen2_audio:
+    pre_prompt: "Translate the speech into Chinese and only output the translation: <|en|>"
+    post_prompt: ""

LaViDa/eval/lmms_eval/tasks/covost2/_default_template_zh_en_yaml

@@ -0,0 +1,28 @@
+dataset_path: lmms-lab/covost2
+dataset_name: zh_en
+dataset_kwargs:
+  token: True
+output_type: generate_until
+doc_to_visual: !function utils.covost2_doc_to_audio
+doc_to_text: !function utils.covost2_doc_to_text
+doc_to_target: "gt"
+generation_kwargs:
+  max_new_tokens: 256
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.covost2_zh_en_process_result
+metric_list:
+  - metric: bleu
+    aggregation: !function utils.covost2_bleu_en
+    higher_is_better: true
+metadata:
+  - version: 0.0
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the speech into English and only output the translation:"
+    post_prompt: ""
+  qwen2_audio:
+    pre_prompt: "Translate the speech into English and only output the translation: <|zh|>"
+    post_prompt: ""

LaViDa/eval/lmms_eval/tasks/covost2/covost2.yaml

@@ -0,0 +1,4 @@
+group: covost2
+task:
+  - covost2_en_zh
+  - covost2_zh_en

LaViDa/eval/lmms_eval/tasks/covost2/covost2_en_zh.yaml

@@ -0,0 +1,4 @@
+group: covost2_en_zh
+task:
+  - covost2_en_zh_test
+  - covost2_en_zh_dev

LaViDa/eval/lmms_eval/tasks/covost2/covost2_en_zh_dev.yaml

@@ -0,0 +1,12 @@
+task: "covost2_en_zh_dev"
+include: _default_template_en_zh_yaml
+dataset_path: lmms-lab/covost2_en-zh
+dataset_name: en_zh
+test_split: dev
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the speech into Chinese and only output the translation:"
+    post_prompt: ""
+  qwen2_audio:
+    pre_prompt: "Translate the speech into Chinese and only output the translation: <|en|>"
+    post_prompt: ""