diff --git a/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c6bbe6432d343535236da0542497986672a2672 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml @@ -0,0 +1,9 @@ +output_type: generate_until +generation_kwargs: + until: + - "ASSISTANT:" + image_aspect_ratio: pad +metadata: + version: 0.0 + api_type : openai + gpt_eval_model_name: "gpt-3.5-turbo" \ No newline at end of file diff --git a/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml new file mode 100644 index 0000000000000000000000000000000000000000..602f79a4902d34d4c174d45984e813b7ae40b724 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml @@ -0,0 +1,26 @@ +dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench +dataset_name: in_domain +dataset_kwargs: + token: True +task: "llava_interleave_bench_in_domain" +test_split: test +doc_to_target: "answer" +doc_to_visual: !function utils.doc_to_visual +doc_to_text: !function utils.doc_to_text +process_results: !function utils.interleave_process_results + +metric_list: + - metric: overall_score + aggregation: !function utils.overall_score + higher_is_better: true + +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + do_sample: False + image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy. + +lmms_eval_specific_kwargs: + default: + oe_post_prompt: "" + mcq_post_prompt: "" diff --git a/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d68410beb1699d36311261548e4bab458a34683 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/interleave_bench.yaml @@ -0,0 +1,5 @@ +group: llava_interleave_bench +task: +- llava_interleave_bench_in_domain +- llava_interleave_bench_out_domain +- llava_interleave_bench_multi_view \ No newline at end of file diff --git a/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38627c236ff904c4cfa0c4fa4408ea8911bf3e3f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml @@ -0,0 +1,26 @@ +dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench +dataset_name: multi_view_in_domain +dataset_kwargs: + token: True +task: "llava_interleave_bench_multi_view" +test_split: test +doc_to_target: "answer" +doc_to_visual: !function utils.doc_to_visual +doc_to_text: !function utils.doc_to_text_conversation +process_results: !function utils.interleave_process_results + +metric_list: + - metric: overall_score + aggregation: !function utils.overall_score + higher_is_better: true + +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + do_sample: False + image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy. + +lmms_eval_specific_kwargs: + default: + oe_post_prompt: "" + mcq_post_prompt: "" diff --git a/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76f217213cf9fb7c793135b8ec989fda80a3c016 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml @@ -0,0 +1,26 @@ +dataset_path: lmms-lab/LLaVA-NeXT-Interleave-Bench +dataset_name: out_of_domain +dataset_kwargs: + token: True +task: "llava_interleave_bench_out_domain" +test_split: test +doc_to_target: "answer" +doc_to_visual: !function utils.doc_to_visual +doc_to_text: !function utils.doc_to_text +process_results: !function utils.interleave_process_results + +metric_list: + - metric: overall_score + aggregation: !function utils.overall_score + higher_is_better: true + +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + do_sample: False + image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy. + +lmms_eval_specific_kwargs: + default: + oe_post_prompt: "" + mcq_post_prompt: "" diff --git a/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..52bca00e67ce5cfd120f44514d49575bcca4c769 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/llava_interleave_bench/utils.py @@ -0,0 +1,383 @@ +import json +import os +import re +import time +from collections import defaultdict +from pathlib import Path + +import requests +import yaml +from loguru import logger as eval_logger +from PIL import Image +from rouge import Rouge + +spot_the_diff = ["Spot-the-Diff", "Birds-to-Words", "CLEVR-Change"] +image_edit_instruct = ["IEdit", "HQ-Edit", "MagicBrush"] +visual_story_telling = ["AESOP", "FlintstonesSV", "PororoSV", "VIST"] +visual_cloze = ["COMICS_Dialogue", "RecipeQA_VisualCloze"] +text_rich_vqa = ["WebQA", "TQA", "OCR-VQA", "DocVQA"] +multi_image_vqa = ["MIT-States_StateCoherence", "MIT-States_PropertyCoherence", "VISION", "RecipeQA_ImageCoherence"] + +puzzle = ["RAVEN"] +nlrv2 = ["NLVR2_Mantis"] +qbench = ["QBench"] + +scan_qa = ["ScanQA"] +alfred = ["ALFRED"] +nuscenes = ["nuscenes"] +scannet_chat = ["ScanNet_chat"] +scannet_task = ["ScanNet_task"] +blink = ["BLINK"] +math_verse = ["MathVerse"] +sci_verse = ["SciVerse"] +mantis = ["Mantis"] + + +def doc_to_visual(doc): + max_visual_count = 16 + visuals = [] + for i in range(max_visual_count): + if f"image_{i}" in doc: + image = doc[f"image_{i}"] + if image is None: + continue # Skip this image if it's None + if isinstance(image, Image.Image): + visuals.append(image.copy().convert("RGB")) + else: + try: + # If the image is not already a PIL Image, try to open it + visuals.append(Image.open(image).convert("RGB")) + except Exception as e: + print(f"Error opening image_{i}: {e}") + # Optionally, you can add a placeholder image or just continue + continue + + return visuals + + +# This is the place where you format your question +def doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + + oe_post_prompt = "" + if "oe_post_prompt" in lmms_eval_specific_kwargs: + oe_post_prompt = lmms_eval_specific_kwargs["oe_post_prompt"] + + mcq_post_prompt = "" + if "mcq_post_prompt" in lmms_eval_specific_kwargs: + mcq_post_prompt = lmms_eval_specific_kwargs["mcq_post_prompt"] + + user_prompt = doc["question"] + + if mcq_post_prompt != "" and doc["question_type"] == "multi-choice": + user_prompt = user_prompt.split("Your answer is:")[0].split("\n")[0].strip() + user_prompt = f"{user_prompt}\n{mcq_post_prompt}" + + if oe_post_prompt != "" and doc["question_type"] == "open-ended": + user_prompt = f"{user_prompt}\n{oe_post_prompt}" + + return user_prompt + + +def doc_to_text_conversation(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + + conversations = doc["conversations"] + + if isinstance(conversations, list): + user_prompt = json.dumps(conversations) + else: + user_prompt = conversations + + return user_prompt + + +def doc_to_text_multi_turn(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + + return doc["conversations"] + + +def interleave_process_results(doc, results): + pred = results[0] + sample_id = doc["sample_id"] + + if doc["question_type"] == "multi-choice": + score = mcq_acc(doc["answer"], pred) + model_response = {"sample_id": sample_id, "sub_task": doc["sub_task"], "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": pred, "score": score} + elif doc["question_type"] == "open-ended": + score = oe_rogue(doc["answer"], pred) + model_response = {"sample_id": sample_id, "sub_task": doc["sub_task"], "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": pred, "score": score} + else: + raise ValueError(f"Unknown question type: {doc['question_type']}") + + return { + "overall_score": model_response, + } + + +def mcq_acc(answer, pred): + periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") + commaStrip = re.compile("(\d)(\,)(\d)") + punct = [";", r"/", "[", "]", '"', "{", "}", "(", ")", "=", "+", "\\", "_", "-", ">", "<", "@", "`", ",", "?", "!"] + + def processPunctuation(inText): + outText = inText + for p in punct: + if (p + " " in inText or " " + p in inText) or (re.search(commaStrip, inText) != None): + outText = outText.replace(p, "") + else: + outText = outText.replace(p, " ") + outText = periodStrip.sub("", outText, re.UNICODE) + return outText + + def process(answer): + option_regex = re.compile(r"^([A-E])\.\s*(.+)$", re.IGNORECASE) + match = option_regex.match(answer.strip()) + + if match: + # If matched, return the option letter in uppercase + return match.group(1).upper() + else: + # If no match, process the answer as before + answer = answer.replace("\n", " ") + answer = answer.replace("\t", " ") + answer = answer.strip() + answer = processPunctuation(answer) + answer = answer.strip("'") + answer = answer.strip('"') + answer = answer.strip(")") + answer = answer.strip("(") + answer = answer.strip().lower() + + # Try to find any single letter (A-E) in the processed answer + letter_match = re.search(r"\b([A-E])\b", answer, re.IGNORECASE) + if letter_match: + return letter_match.group(1).upper() + + return answer + + pred = process(pred) + answer = process(answer) + + if pred == answer: + score = 1 + else: + score = 0 + + return score + + +def oe_rogue(answer, pred): + rouge = Rouge() + if pred == "": + score = 0 + else: + if len(pred) > 512: + pred = pred[:512] + score = rouge.get_scores(pred, answer)[0]["rouge-l"]["f"] + + return score + + +def overall_score(results): + categories = { + "Spot-the-Diff": spot_the_diff, + "Image-Edit": image_edit_instruct, + "Visual-Story-Telling": visual_story_telling, + "Visual-Cloze": visual_cloze, + "Text-Rich-VQA": text_rich_vqa, + "Multi-Image-VQA": multi_image_vqa, + "Puzzle": puzzle, + "NLVR2": nlrv2, + "QBench": qbench, + "ScanQA": scan_qa, + "ALFRED": alfred, + "nuscenes": nuscenes, + "ScanNet_chat": scannet_chat, + "ScanNet_task": scannet_task, + "BLINK": blink, + "MathVerse": math_verse, + "SciVerse": sci_verse, + "Mantis": mantis, + } + + category_scores = {} + matched_subtasks = set() + + eval_logger.info(f"Evaluation Sub-Task Results:") + for category, subtasks in categories.items(): + score = 0 + count = 0 + for result in results: + if result["sub_task"] in subtasks: + count += 1 + score += result["score"] + matched_subtasks.add(result["sub_task"]) + if count > 0: + avg_score = score / count + category_scores[category] = avg_score + eval_logger.info(f"{category}: {avg_score:.3f}") + + if not matched_subtasks: + raise ValueError("No subtasks were matched in the results. Check if the subtask names are correct.") + + # Calculate overall score + total_score = sum(category_scores.values()) + num_categories = len(category_scores) + overall_score = total_score / num_categories if num_categories > 0 else 0 + + return overall_score + + +# EVAL_PROMPT = """ +# [Question] +# {question} + +# [Assistant Response] +# {model_response} + +# [Ground Truth Response] +# {ground_truth} + +# [System] +# Rate whether the assistant response correctly matches the ground truth, it's about a question towards a sequence of images shared by the user. +# The rating should be 1-5, where 1 is incorrect and 5 is correct. +# Your response should be in the format: +# Explanation: (your explanation) +# Rating: (int) +# """ + +# NUM_SECONDS_TO_SLEEP = 5 +# dir_path = os.path.dirname(os.path.realpath(__file__)) +# with open(Path(__file__).parent / "_default_template_interleave_yaml", "r") as f: +# raw_data = f.readlines() +# safe_data = [] +# for i, line in enumerate(raw_data): +# # remove function definition since yaml load cannot handle it +# if "!function" not in line: +# safe_data.append(line) + +# config = yaml.safe_load("".join(safe_data)) + +# GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] +# API_TYPE = config["metadata"]["api_type"] + +# if API_TYPE == "openai": +# API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") +# API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") +# headers = { +# "Authorization": f"Bearer {API_KEY}", +# "Content-Type": "application/json", +# } +# elif API_TYPE == "azure": +# API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") +# API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") +# headers = { +# "api-key": API_KEY, +# "Content-Type": "application/json", +# } +# else: +# API_URL = "" +# API_KEY = "" + + +# def get_chat_response(prompt, max_retries=5, wait_time=10): +# headers = { +# "Authorization": f"Bearer {API_KEY}", +# "Content-Type": "application/json", +# } + +# payload = { +# "model": GPT_EVAL_MODEL_NAME, +# "messages": [ +# { +# "role": "user", +# "content": [ +# {"type": "text", "text": prompt}, +# ], +# } +# ], +# "max_tokens": 1024, +# "temperature": 0.0, +# } + +# for attempt in range(max_retries): +# try: +# response = requests.post(API_URL, headers=headers, json=payload, timeout=60) +# response.raise_for_status() +# response_data = response.json() +# return response_data["choices"][0]["message"]["content"], GPT_EVAL_MODEL_NAME +# except requests.exceptions.RequestException as e: +# eval_logger.warning(f"Request failed on attempt {attempt+1}: {e}") +# time.sleep(wait_time) +# if attempt == max_retries - 1: +# eval_logger.error(f"Failed to get response after {max_retries} attempts") +# return "", GPT_EVAL_MODEL_NAME +# except Exception as e: +# eval_logger.error(f"Error on attempt {attempt+1}: {e}") +# return "", GPT_EVAL_MODEL_NAME + + +# def in_domain_oe_gpt_eval(results, args): +# total_score = 0 +# available_count = 0 +# for result in results: +# if result["question_type"] == "open-ended": +# question = result["question"] +# model_response = result["parsed_pred"] +# ground_truth = result["answer"] +# content = EVAL_PROMPT.format(question=question, model_response=model_response, ground_truth=ground_truth) +# result["gpt_eval_input"] = content +# model_output, model_name = get_chat_response(content) +# try: +# explanation = re.search(r"Explanation: (.*)\n", model_output).group(1) +# rating = re.search(r"Rating: (\d+)\n", model_output).group(1) +# result["gpt_eval_explanation"] = explanation +# result["gpt_eval_rating"] = rating +# result["gpt_eval_model_name"] = model_name +# except: +# eval_logger.error(f"Error on evaluating {result['sample_id']}. Results: {results}") +# result["gpt_eval_explanation"] = "" +# result["gpt_eval_rating"] = 0 +# result["gpt_eval_model_name"] = model_name + +# total_score += result["gpt_eval_rating"] +# available_count += 1 + +# elif result["question_type"] == "multi-choice": +# pass + +# return (total_score / available_count) * 20.0 if available_count > 0 else 0 + + +# # class MultiChoiceRegexFilter(ExtendedRegexFilter): +# # def __init__(self, *args, **kwargs): +# # super().__init__(*args, **kwargs) + +# # def apply(self, resps, docs): +# # filtered_resps = [] + +# # for r, doc in zip(resps, docs): +# # # Regex to directly extract the option letter from the model response +# # option_letter_regex = re.compile(r"\b([A-Z])\.\s+([^\n]*)") + +# # # Process each response +# # filtered = [] +# # for resp in r: +# # # Try to match the option letter at the start of the response +# # match = option_letter_regex.match(resp) +# # if match: +# # # If a match is found, append the matched letter +# # filtered.append(match.group(1)) +# # else: +# # # If no match, return the original response +# # filtered.append(resp) + +# # # Assuming we need the first response that matches or the original response +# # filtered_resps.append(filtered[0]) + +# # return filtered_resps diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py b/LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..5965cce7b62065f96bdc7bd544b98c7f96de8e5f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/_generate_configs.py @@ -0,0 +1,149 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + +eval_logger = logging.getLogger("lm-eval") + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--base_yaml_path", required=True) + parser.add_argument("--save_prefix_path", default="mmlu") + parser.add_argument("--cot_prompt_path", default=None) + parser.add_argument("--task_prefix", default="") + parser.add_argument("--group_prefix", default="") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + with open(args.base_yaml_path, encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path, encoding="utf-8") as f: + cot_file = json.load(f) + + ALL_CATEGORIES = [] + for subject, category in tqdm(SUBJECTS.items()): + if category not in ALL_CATEGORIES: + ALL_CATEGORIES.append(category) + + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"mmlu_{args.task_prefix}_{category}" if args.task_prefix != "" else f"mmlu_{category}", + "task": f"mmlu_{args.task_prefix}_{subject}" if args.task_prefix != "" else f"mmlu_{subject}", + "task_alias": subject.replace("_", " "), + "dataset_name": subject, + "description": description, + } + + file_save_path = args.save_prefix_path + f"_{subject}.yaml" + eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + allow_unicode=True, + default_style='"', + ) + + if args.task_prefix != "": + mmlu_subcategories = [f"mmlu_{args.task_prefix}_{category}" for category in ALL_CATEGORIES] + else: + mmlu_subcategories = [f"mmlu_{category}" for category in ALL_CATEGORIES] + + if args.group_prefix != "": + file_save_path = args.group_prefix + ".yaml" + else: + file_save_path = args.save_prefix_path + ".yaml" + + eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": f"mmlu_{args.task_prefix}" if args.task_prefix != "" else "mmlu", + "task": mmlu_subcategories, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1452e0f5b34a899e537ad8889bf012c403ff58cc --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/_default_template_yaml @@ -0,0 +1,20 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}" +generation_kwargs: + until: + - "" + - "\n" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cff0f12f51ff241eafa389e3e47f81febc8402f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_generative +group_alias: mmlu (generative) +task: + - group: stem + task: + - mmlu_stem_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_other_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_social_sciences_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_humanities_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17bfcafb79b113cffe93f6e90c68562b7eae7c95 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_abstract_algebra_generative" +"task_alias": "abstract_algebra" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b41447e74a2b95732b102bfe5ed642d3d208d2b --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_astronomy_generative" +"task_alias": "astronomy" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c15d443691af36dcdc761eb41b8673f3782d0b --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_business_ethics_generative" +"task_alias": "business_ethics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24cd0b72d3f68fb00da90397979816b85ea1c76c --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_clinical_knowledge_generative" +"task_alias": "clinical_knowledge" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d9ce3eab1332fa202cf6f99a52785865aed1a7 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_chemistry_generative" +"task_alias": "college_chemistry" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15ae9dded855610af45a15bab8aa56596bfaddd4 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_mathematics_generative" +"task_alias": "college_mathematics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d997d8974c99a549a2216a9bd9237f05a619e21 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_physics_generative" +"task_alias": "college_physics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee64d20100e25fc4bcf7f446b1e98acf042c4ab8 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_computer_security_generative" +"task_alias": "computer_security" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75764a2cbf542ba09a99ae252c76a103bf534a9f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_conceptual_physics_generative" +"task_alias": "conceptual_physics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43fec80ad3f505bedb810df609a8c6e8d2c2c0ed --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_econometrics_generative" +"task_alias": "econometrics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..130ec2b2aa2210322c1e2f86cdf6be31dd72bffc --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_electrical_engineering_generative" +"task_alias": "electrical_engineering" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4afd087dc47f27653b54ff48a27a187bc9af07bc --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_elementary_mathematics_generative" +"task_alias": "elementary_mathematics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c28c0b188b8b8fd69ba9ed79595f0d173f71cf --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_formal_logic_generative" +"task_alias": "formal_logic" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b788025ad5ddf0d859fc12a0d0f139c0975b16ba --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_global_facts_generative" +"task_alias": "global_facts" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3677842dcfc091bb28525889479a48096cbb854d --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_biology_generative" +"task_alias": "high_school_biology" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2df93cab2a999a7d6d8e78d3ac9c3ce9aeddcf12 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_chemistry_generative" +"task_alias": "high_school_chemistry" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec5dc7f89abd7ddc57438c71e0502fce1ac47279 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_computer_science_generative" +"task_alias": "high_school_computer_science" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9732754bbd7352957dbe299494083e17b960c1bc --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_european_history_generative" +"task_alias": "high_school_european_history" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66b1a3c97a64f9ee7db414ab13d3146efba5612d --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_geography_generative" +"task_alias": "high_school_geography" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ada415922b2b777f153cf387f9095cce9c75304b --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_macroeconomics_generative" +"task_alias": "high_school_macroeconomics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b22a5888e61be187f5bbbca1e38171eecd6252d --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_mathematics_generative" +"task_alias": "high_school_mathematics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c59ff16270084981614d6f01065851c005039413 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_microeconomics_generative" +"task_alias": "high_school_microeconomics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d846afb9c8c6b372d59ee462561bb8f67ae83e --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_physics_generative" +"task_alias": "high_school_physics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd1321a5f17efca463edbc6711c197fb18c3a81d --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_psychology_generative" +"task_alias": "high_school_psychology" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1442fb8df4168606151af5cc1dfd769bb2e70e3 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_statistics_generative" +"task_alias": "high_school_statistics" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4552a560f38e3ed5db503fa677548a11766873c2 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_us_history_generative" +"task_alias": "high_school_us_history" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d510f22ff39219829e6a9030cb39dc2c43062ca4 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_world_history_generative" +"task_alias": "high_school_world_history" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56352f4a8c86966853cdbafd68453d1ee85dbabb --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_human_aging_generative" +"task_alias": "human_aging" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a23559cfb36a380131573f46b30bbdb5f4656b42 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_human_sexuality_generative" +"task_alias": "human_sexuality" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..878df6f3cacb299a51afacca461204fdc4e3a782 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_international_law_generative" +"task_alias": "international_law" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5782d81551072a0ff03d79c930f02edb64488f3 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_jurisprudence_generative" +"task_alias": "jurisprudence" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43e8e0168b9f4638cc80b76ff1a4edc8893212b4 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_logical_fallacies_generative" +"task_alias": "logical_fallacies" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d51ea0d0aa41fb4b2579162111aa8ebd8ce8f6d --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_management_generative" +"task_alias": "management" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..744385a2ea524d6f651851856e15aaf190eb847e --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_marketing_generative" +"task_alias": "marketing" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e0fabc2536d4894526b680deba9a382ff9c3ff --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_miscellaneous_generative" +"task_alias": "miscellaneous" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61d2feee6a9cf4ed4d71b7c2f9aa68f5219c270a --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_moral_disputes_generative" +"task_alias": "moral_disputes" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aeb93f967f0811d3a2f1d886aedfb334a96714e --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_moral_scenarios_generative" +"task_alias": "moral_scenarios" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..149894b8484cb1fad9ddad1fc5cb2c07a659aea1 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_philosophy_generative" +"task_alias": "philosophy" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e130e1baacc3f8a8f558b568336896668e84dd4f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_prehistory_generative" +"task_alias": "prehistory" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a46792ec22d84ee3193996653f536084b9ab7861 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_accounting_generative" +"task_alias": "professional_accounting" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f087657e579524b35bf7de4c0f81cb5b697caed4 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_law_generative" +"task_alias": "professional_law" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc80878980195f58ac5ae26a0a70589a47b325d5 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_medicine_generative" +"task_alias": "professional_medicine" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b36ccde61e7edc33464a676d4fe0fcc25f3304 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology_generative" +"task_alias": "professional_psychology" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_public_relations.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37cdccba9b7cebbaa34c5f1e9da01655367477f6 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_public_relations_generative" +"task_alias": "public_relations" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_security_studies.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36c235feefd1548320400e7e8d9f3e03f2d478d0 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_security_studies_generative" +"task_alias": "security_studies" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_sociology.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7e2e592e4457118c9458ccb757b823f9adbb193 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_sociology_generative" +"task_alias": "sociology" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fb95366245eae638918270bff4353024195d5f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy_generative" +"task_alias": "us_foreign_policy" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_virology.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9954dc182f1bbd5030b94d2a08b2ddf4a135a6cf --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_virology_generative" +"task_alias": "virology" diff --git a/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_world_religions.yaml b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1db5128b43e615d0fc41f9c7448db3b5ea39942c --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/mmlu/generative/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_world_religions_generative" +"task_alias": "world_religions" diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/_default_template_yaml b/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a35beb5559ed1685d2d59bdf6a4e863479913f4 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/_default_template_yaml @@ -0,0 +1,9 @@ +dataset_path: lmms-lab/PerceptionTest +dataset_kwargs: + token: True + video: True + cache_dir: perceptiontest +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "" \ No newline at end of file diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/perceptiontest_mc.yaml b/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/perceptiontest_mc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d406fb85c11c44bab00d7fa2775720b7575f460c --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/perceptiontest_mc.yaml @@ -0,0 +1,13 @@ +dataset_name: "mc_question" +task: "perceptiontest_test_mc" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.perceptiontest_doc_to_visual +doc_to_text: !function utils.perceptiontest_doc_to_text +doc_to_target: !function utils.perceptiontest_doc_to_answer_mc +process_results: !function utils.perceptiontest_process_results_mc +metric_list: + - metric: submission + aggregation: !function utils.perceptiontest_aggregate_mc + higher_is_better: true +include: _default_template_yaml diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/utils.py b/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bdb0cb5a4ef3ad6b03e83414f39f32568b391b13 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/test/utils.py @@ -0,0 +1,126 @@ +import datetime +import json +import os +import sys +from pathlib import Path + +import numpy as np +import yaml +from decord import VideoReader, cpu + +import lmms_eval.tasks._task_utils.file_utils as file_utils + +with open(Path(__file__).parent / "_default_template_yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + +# We will unzip all the zip files +# To HF HOME cache dir +# And load it here +HF_HOME = os.environ["HF_HOME"] +cache_dir = config["dataset_kwargs"]["cache_dir"] +cache_dir = os.path.join(HF_HOME, cache_dir) +cache_dir = os.path.join(cache_dir, "videos") + +from loguru import logger as eval_logger + + +# Pass in video path here +# Can only work correctly with video llm +def perceptiontest_doc_to_visual(doc): + video_path = doc["video_name"] + ".mp4" + video_path = os.path.join(cache_dir, video_path) + if os.path.exists(video_path): + video_path = video_path + elif os.path.exists(video_path.replace("mp4", "MP4")): + video_path = video_path.replace("mp4", "MP4") + else: + sys.exit(f"video path:{video_path} does not exist, please check") + return [video_path] + + +# This is the place where you format your question +def perceptiontest_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = "" + post_prompt = "" + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + + question = doc["question"] + if "options" in doc: + index = 0 + for op in doc["options"]: + if index == 0: + question += "\n" + "A. " + op + elif index == 1: + question += "\n" + "B. " + op + else: + question += "\n" + "C. " + op + index += 1 + post_prompt = "\nAnswer with the option's letter from the given choices directly." + + return f"{pre_prompt}{question}{post_prompt}" + + +def perceptiontest_doc_to_answer_mc(doc): + return "" # pseudo answer + + +# Process result for mc_ppl +def perceptiontest_process_results_mc_ppl(doc, result): + # Initialize minimum value and index + min_value = float("inf") + min_index = -1 + + # Iterate through the results to find the index of the lowest value + for i, (value, _) in enumerate(result): + if value < min_value: + min_value = value + min_index = i + + # Return the result with the index of the lowest value + return {"submission": {"video_name": doc["video_name"], "question": doc["question"], "question_id": doc["question_id"], "pred_id": min_index}} + + +# Process result for generation +def perceptiontest_process_results_mc(doc, result): + pred = result[0] # string prediction "A", "B", "C" + + # Map the prediction to an index + pred_to_index = {"A": 0, "B": 1, "C": 2} + index = pred_to_index.get(pred, -1) # Default to -1 if the prediction is not found + + return {"submission": {"video_name": doc["video_name"], "question": doc["question"], "question_id": doc["question_id"], "pred_id": index}} + + +def perceptiontest_aggregate_submissions(results, args, task): + now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + submission_file_name = f"inference_results_perceptiontest_{task}_{now_date_time}.json" + path = file_utils.generate_submission_file(submission_file_name, args) + + with open(path, "w") as f: + json.dump(results, f, indent=4) + + eval_logger.info(f"Submission file saved to {path}") + + +def perceptiontest_aggregate_mc(results, args): + perceptiontest_aggregate_submissions(results, args, "MC") + + +def perceptiontest_aggregate_mc_ppl(results, args): + perceptiontest_aggregate_submissions(results, args, "MC_PPL") + + +def perceptiontest_doc_to_choice(doc): + return [op for op in doc["options"]] diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/_default_template_yaml b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c586beb97a0cc9b7c1eab1c26cedf9c0751414fd --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/_default_template_yaml @@ -0,0 +1,9 @@ +dataset_path: lmms-lab/PerceptionTest_Val +dataset_kwargs: + token: True + video: True + cache_dir: perceptiontest_val +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "" \ No newline at end of file diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/perceptiontest_mc.yaml b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/perceptiontest_mc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fce82a656844e30990b7df9befbdb33cc231648a --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/perceptiontest_mc.yaml @@ -0,0 +1,13 @@ +dataset_name: "mc_question_val" +task: "perceptiontest_val_mc" +test_split: validation +output_type: generate_until +doc_to_visual: !function utils.perceptiontest_val_doc_to_visual +doc_to_text: !function utils.perceptiontest_val_doc_to_text +doc_to_target: !function utils.perceptiontest_val_doc_to_answer +process_results: !function utils.perceptiontest_val_process_results_mc +metric_list: + - metric: accuracy + aggregation: !function utils.perceptiontest_val_aggregate_accuracy + higher_is_better: true +include: _default_template_yaml diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/perceptiontest_mcppl.yaml b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/perceptiontest_mcppl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2371cdb9446a87fb6d18cda9beab451c1a79f52 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/perceptiontest_mcppl.yaml @@ -0,0 +1,14 @@ +dataset_name: "mc_question_val" +task: "perceptiontest_val_mcppl" +test_split: validation +output_type: multiple_choice +doc_to_visual: !function utils.perceptiontest_val_doc_to_visual +doc_to_text: "question" +doc_to_target: !function utils.perceptiontest_val_doc_to_answer +doc_to_choice: !function utils.perceptiontest_val_doc_to_choice +process_results: !function utils.perceptiontest_val_process_results_mc_ppl +metric_list: + - metric: accuracy + aggregation: !function utils.perceptiontest_val_aggregate_accuracy + higher_is_better: true +include: _default_template_yaml diff --git a/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/utils.py b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..24a9d8842405a1f77f41c8ed93c2108cc101b573 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/perceptiontest/val/utils.py @@ -0,0 +1,153 @@ +import datetime +import json +import os +import sys +from pathlib import Path + +import numpy as np +import yaml +from decord import VideoReader, cpu + +import lmms_eval.tasks._task_utils.file_utils as file_utils + +with open(Path(__file__).parent / "_default_template_yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + +# We will unzip all the zip files +# To HF HOME cache dir +# And load it here +HF_HOME = os.environ["HF_HOME"] +cache_dir = config["dataset_kwargs"]["cache_dir"] +cache_dir = os.path.join(HF_HOME, cache_dir) +cache_dir = os.path.join(cache_dir, "videos") + +from loguru import logger as eval_logger + + +# Pass in video path here +# Can only work correctly with video llm +def perceptiontest_val_doc_to_visual(doc): + video_path = doc["video_name"] + ".mp4" + video_path = os.path.join(cache_dir, video_path) + if os.path.exists(video_path): + video_path = video_path + elif os.path.exists(video_path.replace("mp4", "MP4")): + video_path = video_path.replace("mp4", "MP4") + else: + sys.exit(f"video path:{video_path} does not exist, please check") + return [video_path] + + +# This is the place where you format your question +def perceptiontest_val_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = "" + post_prompt = "" + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + + question = doc["question"] + if "options" in doc: + index = 0 + for op in doc["options"]: + if index == 0: + question += "\n" + "A. " + op + elif index == 1: + question += "\n" + "B. " + op + else: + question += "\n" + "C. " + op + index += 1 + post_prompt = "\nAnswer with the option's letter from the given choices directly." + + return f"{pre_prompt}{question}{post_prompt}" + + +def perceptiontest_val_doc_to_answer(doc): + return doc["answer_id"] + + +# Process result for mc_ppl +def perceptiontest_val_process_results_mc_ppl(doc, result): + # Initialize minimum value and index + min_value = float("inf") + min_index = -1 + + # Iterate through the results to find the index of the lowest value + for i, (value, _) in enumerate(result): + if value < min_value: + min_value = value + min_index = i + + # Return the result with the index of the lowest value + return { + "accuracy": { + "video_name": doc["video_name"], + "question": doc["question"], + "question_id": doc["question_id"], + "pred_id": min_index, + "answer_id": doc["answer_id"], + "area": doc["area"], + "reasoning": doc["reasoning"], + "tag": doc["tag"], + } + } + + +# Process result for generation +import re + + +def perceptiontest_val_process_results_mc(doc, result): + pred = result[0].strip() # raw text prediction + + # Use regex to match A, B, C, or D + match = re.search(r"\b([A-D])\b", pred) + + if match: + pred = match.group(1) # Extract the matched letter + pred = pred.upper() + else: + pred = "" # Set to empty string if no match found + + # Map the prediction to an index + pred_to_index = {"A": 0, "B": 1, "C": 2, "D": 3} + index = pred_to_index.get(pred, -1) # Default to -1 if the prediction is not found + + correct = 1 if index == int(doc["answer_id"]) else 0 + + return { + "accuracy": { + "video_name": doc["video_name"], + "question": doc["question"], + "pred_id": index, + "answer_id": doc["answer_id"], + "correct": correct, + } + } + + +def perceptiontest_val_aggregate_accuracy(results, args): + yes_count = 0 + + # results is a list of dict + for answer_dict in results: + if answer_dict["correct"] == 1: + yes_count = yes_count + 1 + + accuracy = yes_count / len(results) + + return accuracy + + +def perceptiontest_val_doc_to_choice(doc): + return [op for op in doc["options"]] diff --git a/LaViDa/eval/lmms_eval/tasks/wavcaps/utils.py b/LaViDa/eval/lmms_eval/tasks/wavcaps/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..09a4333f7f069b545c3c8b5c344b4079ec5f232f --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/wavcaps/utils.py @@ -0,0 +1,134 @@ +import os +import re +import time +from pathlib import Path + +import requests +import yaml +from loguru import logger as eval_logger + + +def wavcaps_doc_to_audio(doc): + return [doc["context"]] + + +def wavcaps_doc_to_text(doc, lmms_eval_specific_kwargs): + question = doc["instruction"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + return f"{pre_prompt}{question}{post_prompt}" + + +# functions for the clotho_asqa_v2 task, need to be tested later + +with open(Path(__file__).parent / "wavcaps.yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + + +NUM_SECONDS_TO_SLEEP = 5 +GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] +API_TYPE = os.getenv("API_TYPE", "azure") + +if API_TYPE == "openai": + API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") + API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") + headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json", + } +elif API_TYPE == "azure": + API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") + API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") + headers = { + "api-key": API_KEY, + "Content-Type": "application/json", + } + +eval_prompt = """ + [Question] + {question} + + [Reference Answer] + {ground_truth} + + [Model Answer] + {model_response} + + [Task] + Rate the model's answer based on its alignment with the reference answer, focusing on accuracy and relevance to the reference provided. Please be critical on the details. + Criteria: Assess if the model's response mirrors the reference in terms of content, accuracy, and relevance. + Score0: The answer is completely misaligned, providing incorrect or irrelevant information compared to the reference. + Score1: The answer shows minimal alignment, often misunderstanding or providing irrelevant details unrelated to the reference. + Score2: The answer recognizes the topic but diverges significantly from the reference in accuracy or relevance. + Score3: The answer aligns with the reference generally but lacks detail or precise accuracy in some aspects. + Score4: The answer is mostly accurate and relevant, closely following the reference but could be clearer or more detailed. + Score5: The answer is highly accurate, detailed, and matches the reference answer perfectly, capturing its essence and detail. + + Your response should be formatted as follows: + Explanation: (Provide a concise explanation of your rating, comparing the reference answer with the model's response. "The reference answer is [XXX], while the model's answer is [YYY]. I think ...") + Rating: (int)""" + + +# gpt-4 +def get_eval(max_tokens: int, content: str): + global headers + + messages = [ + {"role": "user", "content": content}, + ] + + payload = {"model": GPT_EVAL_MODEL_NAME, "messages": messages, "temperature": 0, "max_tokens": max_tokens, "n": 1} + + for attempt in range(5): + try: + response = requests.post(API_URL, headers=headers, json=payload, timeout=60) + response.raise_for_status() + response_data = response.json() + + content = response_data["choices"][0]["message"]["content"].strip() + if content != "": + return content, response_data["model"] + break # If successful, break out of the loop + + except Exception as e: + eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}") + if attempt < 5: # If we have retries left, sleep and then continue to next attempt + time.sleep(NUM_SECONDS_TO_SLEEP) + else: # If this was the last attempt, log and return empty + eval_logger.error(f"All 5 attempts failed. Last error message: {e}") + return "", "" + return "", "" + + +def wavcaps_process_results(doc, results): + pred = results[0] + ground_truth_str = doc["answer"] + content = eval_prompt.format(model_response=pred, ground_truth=ground_truth_str, question=doc["instruction"]) + eval_answer, model_name = get_eval(max_tokens=1024, content=content) + return { + "gpt_eval": {"eval_answer": eval_answer, "model_name": model_name}, + } + + +def wavcaps_aggregate_results(results): + score = 0 + for result in results: + eval_answer = result["eval_answer"] + + try: + match = re.search(r"Rating:\s*([0-5])\s*$", eval_answer) + eval_score = match.group(1) if match else 0 + eval_score = float(eval_score) + except Exception as e: + eval_logger.error(f"Error parsing eval_score: {e}") + eval_score = 0.0 + score += eval_score + + return score / len(results) diff --git a/LaViDa/eval/lmms_eval/tasks/wavcaps/wavcaps.yaml b/LaViDa/eval/lmms_eval/tasks/wavcaps/wavcaps.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c142e2bb7e0ac17d3ea80068165c5c9438d1f105 --- /dev/null +++ b/LaViDa/eval/lmms_eval/tasks/wavcaps/wavcaps.yaml @@ -0,0 +1,27 @@ +dataset_path: AudioLLMs/wavcaps_test +dataset_kwargs: + token: True +task : "wavcaps" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.wavcaps_doc_to_audio +doc_to_text: !function utils.wavcaps_doc_to_text +doc_to_target: "answer" +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "" +generation_kwargs: + max_new_tokens: 256 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false +process_results: !function utils.wavcaps_process_results +metric_list: + - metric: gpt_eval + aggregation: !function utils.wavcaps_aggregate_results + higher_is_better: true +metadata: + gpt_eval_model_name: gpt-4o + version: 0.0 diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/eva_vit_model.py b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/eva_vit_model.py new file mode 100644 index 0000000000000000000000000000000000000000..23cb38c9230f92cf5e0601a95fb6a610a9e2185c --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/eva_vit_model.py @@ -0,0 +1,571 @@ +# -------------------------------------------------------- +# Adapted from https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- +import math +import os +import torch +import torch.nn as nn +import torch.nn.functional as F + +try: + from timm.models.layers import drop_path, to_2tuple, trunc_normal_ +except: + from timm.layers import drop_path, to_2tuple, trunc_normal_ + +from .transformer import PatchDropout +from .rope import VisionRotaryEmbedding, VisionRotaryEmbeddingFast + +if os.getenv("ENV_TYPE") == "deepspeed": + try: + from deepspeed.runtime.activation_checkpointing.checkpointing import checkpoint + except: + from torch.utils.checkpoint import checkpoint +else: + from torch.utils.checkpoint import checkpoint + +try: + import xformers.ops as xops +except ImportError: + xops = None + # print("Please 'pip install xformers'") + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return "p={}".format(self.drop_prob) + + +class Mlp(nn.Module): + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + drop=0.0, + subln=False, + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + + self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity() + + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + # x = self.drop(x) + # commit this for the orignal BERT implement + x = self.ffn_ln(x) + + x = self.fc2(x) + x = self.drop(x) + return x + + +class SwiGLU(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.0, norm_layer=nn.LayerNorm, subln=False): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + + self.w1 = nn.Linear(in_features, hidden_features) + self.w2 = nn.Linear(in_features, hidden_features) + + self.act = act_layer() + self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity() + self.w3 = nn.Linear(hidden_features, out_features) + + self.drop = nn.Dropout(drop) + + def forward(self, x): + x1 = self.w1(x) + x2 = self.w2(x) + hidden = self.act(x1) * x2 + x = self.ffn_ln(hidden) + x = self.w3(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.0, proj_drop=0.0, window_size=None, attn_head_dim=None, xattn=False, rope=None, subln=False, norm_layer=nn.LayerNorm): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.subln = subln + if self.subln: + self.q_proj = nn.Linear(dim, all_head_dim, bias=False) + self.k_proj = nn.Linear(dim, all_head_dim, bias=False) + self.v_proj = nn.Linear(dim, all_head_dim, bias=False) + else: + self.qkv = nn.Linear(dim, all_head_dim * 3, bias=False) + + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.v_bias = None + + if window_size: + self.window_size = window_size + self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3 + self.relative_position_bias_table = nn.Parameter(torch.zeros(self.num_relative_distance, num_heads)) # 2*Wh-1 * 2*Ww-1, nH + # cls to token & token 2 cls & cls to cls + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(window_size[0]) + coords_w = torch.arange(window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * window_size[1] - 1 + relative_position_index = torch.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype) + relative_position_index[1:, 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = self.num_relative_distance - 3 + relative_position_index[0:, 0] = self.num_relative_distance - 2 + relative_position_index[0, 0] = self.num_relative_distance - 1 + + self.register_buffer("relative_position_index", relative_position_index) + else: + self.window_size = None + self.relative_position_bias_table = None + self.relative_position_index = None + + self.attn_drop = nn.Dropout(attn_drop) + self.inner_attn_ln = norm_layer(all_head_dim) if subln else nn.Identity() + # self.proj = nn.Linear(all_head_dim, all_head_dim) + self.proj = nn.Linear(all_head_dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.xattn = xattn + self.xattn_drop = attn_drop + + self.rope = rope + + def forward(self, x, rel_pos_bias=None, attn_mask=None): + B, N, C = x.shape + if self.subln: + q = F.linear(input=x, weight=self.q_proj.weight, bias=self.q_bias) + k = F.linear(input=x, weight=self.k_proj.weight, bias=None) + v = F.linear(input=x, weight=self.v_proj.weight, bias=self.v_bias) + + q = q.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3) # B, num_heads, N, C + k = k.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3) + v = v.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3) + else: + + qkv_bias = None + if self.q_bias is not None: + qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias)) + + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) # 3, B, num_heads, N, C + q, k, v = qkv[0], qkv[1], qkv[2] + + if self.rope: + # slightly fast impl + q_t = q[:, :, 1:, :] + ro_q_t = self.rope(q_t) + q = torch.cat((q[:, :, :1, :], ro_q_t), -2).type_as(v) + + k_t = k[:, :, 1:, :] + ro_k_t = self.rope(k_t) + k = torch.cat((k[:, :, :1, :], ro_k_t), -2).type_as(v) + + if self.xattn: + q = q.permute(0, 2, 1, 3) # B, num_heads, N, C -> B, N, num_heads, C + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + + x = xops.memory_efficient_attention( + q, + k, + v, + p=self.xattn_drop, + scale=self.scale, + ) + x = x.reshape(B, N, -1) + x = self.inner_attn_ln(x) + x = self.proj(x) + x = self.proj_drop(x) + else: + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + if self.relative_position_bias_table is not None: + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(self.window_size[0] * self.window_size[1] + 1, self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0).type_as(attn) + + if rel_pos_bias is not None: + attn = attn + rel_pos_bias.type_as(attn) + + if attn_mask is not None: + attn_mask = attn_mask.bool() + attn = attn.masked_fill(~attn_mask[:, None, None, :], float("-inf")) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.inner_attn_ln(x) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + + def __init__( + self, + dim, + num_heads, + mlp_ratio=4.0, + qkv_bias=False, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + init_values=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + window_size=None, + attn_head_dim=None, + xattn=False, + rope=None, + postnorm=False, + subln=False, + naiveswiglu=False, + ): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, window_size=window_size, attn_head_dim=attn_head_dim, xattn=xattn, rope=rope, subln=subln, norm_layer=norm_layer + ) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + + if naiveswiglu: + self.mlp = SwiGLU( + in_features=dim, + hidden_features=mlp_hidden_dim, + subln=subln, + norm_layer=norm_layer, + ) + else: + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, subln=subln, drop=drop) + + if init_values is not None and init_values > 0: + self.gamma_1 = nn.Parameter(init_values * torch.ones((dim)), requires_grad=True) + self.gamma_2 = nn.Parameter(init_values * torch.ones((dim)), requires_grad=True) + else: + self.gamma_1, self.gamma_2 = None, None + + self.postnorm = postnorm + + def forward(self, x, rel_pos_bias=None, attn_mask=None): + if self.gamma_1 is None: + if self.postnorm: + x = x + self.drop_path(self.norm1(self.attn(x, rel_pos_bias=rel_pos_bias, attn_mask=attn_mask))) + x = x + self.drop_path(self.norm2(self.mlp(x))) + else: + x = x + self.drop_path(self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias, attn_mask=attn_mask)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + else: + if self.postnorm: + x = x + self.drop_path(self.gamma_1 * self.norm1(self.attn(x, rel_pos_bias=rel_pos_bias, attn_mask=attn_mask))) + x = x + self.drop_path(self.gamma_2 * self.norm2(self.mlp(x))) + else: + x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias, attn_mask=attn_mask)) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding""" + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) + self.patch_shape = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x, **kwargs): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + assert H == self.img_size[0] and W == self.img_size[1], f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x).flatten(2).transpose(1, 2) + return x + + +class RelativePositionBias(nn.Module): + + def __init__(self, window_size, num_heads): + super().__init__() + self.window_size = window_size + self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3 + self.relative_position_bias_table = nn.Parameter(torch.zeros(self.num_relative_distance, num_heads)) # 2*Wh-1 * 2*Ww-1, nH + # cls to token & token 2 cls & cls to cls + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(window_size[0]) + coords_w = torch.arange(window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * window_size[1] - 1 + relative_position_index = torch.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype) + relative_position_index[1:, 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = self.num_relative_distance - 3 + relative_position_index[0:, 0] = self.num_relative_distance - 2 + relative_position_index[0, 0] = self.num_relative_distance - 1 + + self.register_buffer("relative_position_index", relative_position_index) + + def forward(self): + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(self.window_size[0] * self.window_size[1] + 1, self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + return relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + + +class EVAVisionTransformer(nn.Module): + """Vision Transformer with support for patch or hybrid CNN input stage""" + + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + num_classes=1000, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=False, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=nn.LayerNorm, + init_values=None, + patch_dropout=0.0, + use_abs_pos_emb=True, + use_rel_pos_bias=False, + use_shared_rel_pos_bias=False, + rope=False, + use_mean_pooling=True, + init_scale=0.001, + grad_checkpointing=False, + xattn=False, + postnorm=False, + pt_hw_seq_len=16, + intp_freq=False, + naiveswiglu=False, + subln=False, + ): + super().__init__() + self.image_size = img_size + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + + self.patch_embed = PatchEmbed(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + # self.mask_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + if use_abs_pos_emb: + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim)) + else: + self.pos_embed = None + self.pos_drop = nn.Dropout(p=drop_rate) + + if use_shared_rel_pos_bias: + self.rel_pos_bias = RelativePositionBias(window_size=self.patch_embed.patch_shape, num_heads=num_heads) + else: + self.rel_pos_bias = None + + if rope: + half_head_dim = embed_dim // num_heads // 2 + hw_seq_len = img_size // patch_size + self.rope = VisionRotaryEmbeddingFast( + dim=half_head_dim, + pt_seq_len=pt_hw_seq_len, + ft_seq_len=hw_seq_len if intp_freq else None, + # patch_dropout=patch_dropout + ) + else: + self.rope = None + + self.naiveswiglu = naiveswiglu + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.use_rel_pos_bias = use_rel_pos_bias + self.blocks = nn.ModuleList( + [ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + init_values=init_values, + window_size=self.patch_embed.patch_shape if use_rel_pos_bias else None, + xattn=xattn, + rope=self.rope, + postnorm=postnorm, + subln=subln, + naiveswiglu=naiveswiglu, + ) + for i in range(depth) + ] + ) + self.norm = nn.Identity() if use_mean_pooling else norm_layer(embed_dim) + self.fc_norm = norm_layer(embed_dim) if use_mean_pooling else None + self.head = nn.Linear(embed_dim, num_classes, bias=qkv_bias) if num_classes > 0 else nn.Identity() + + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=0.02) + + trunc_normal_(self.cls_token, std=0.02) + + self.apply(self._init_weights) + self.fix_init_weight() + + if isinstance(self.head, nn.Linear): + trunc_normal_(self.head.weight, std=0.02) + self.head.weight.data.mul_(init_scale) + if self.head.bias is not None: + self.head.bias.data.mul_(init_scale) + + # setting a patch_dropout of 0. would mean it is disabled and this function would be the identity fn + self.patch_dropout = PatchDropout(patch_dropout) if patch_dropout > 0.0 else nn.Identity() + + self.grad_checkpointing = grad_checkpointing + + def fix_init_weight(self): + def rescale(param, layer_id): + param.div_(math.sqrt(2.0 * layer_id)) + + for layer_id, layer in enumerate(self.blocks): + rescale(layer.attn.proj.weight.data, layer_id + 1) + if self.naiveswiglu: + rescale(layer.mlp.w3.weight.data, layer_id + 1) + else: + rescale(layer.mlp.fc2.weight.data, layer_id + 1) + + def get_cast_dtype(self) -> torch.dtype: + return self.blocks[0].mlp.fc2.weight.dtype + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def get_num_layers(self): + return len(self.blocks) + + def lock(self, unlocked_groups=0, freeze_bn_stats=False): + assert unlocked_groups == 0, "partial locking not currently supported for this model" + for param in self.parameters(): + param.requires_grad = False + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.grad_checkpointing = enable + + @torch.jit.ignore + def no_weight_decay(self): + return {"pos_embed", "cls_token"} + + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes, global_pool=""): + self.num_classes = num_classes + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x, return_all_features=False): + + x = self.patch_embed(x) + batch_size, seq_len, _ = x.size() + + cls_tokens = self.cls_token.expand(batch_size, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + if self.pos_embed is not None: + x = x + self.pos_embed + x = self.pos_drop(x) + + # a patch_dropout of 0. would mean it is disabled and this function would do nothing but return what was passed in + # if os.getenv("RoPE") == "1": + # if self.training and not isinstance(self.patch_dropout, nn.Identity): + # x, patch_indices_keep = self.patch_dropout(x) + # self.rope.forward = partial(self.rope.forward, patch_indices_keep=patch_indices_keep) + # else: + # self.rope.forward = partial(self.rope.forward, patch_indices_keep=None) + # x = self.patch_dropout(x) + # else: + x = self.patch_dropout(x) + + rel_pos_bias = self.rel_pos_bias() if self.rel_pos_bias is not None else None + for blk in self.blocks: + if self.grad_checkpointing: + x = checkpoint(blk, x, (rel_pos_bias,)) + else: + x = blk(x, rel_pos_bias=rel_pos_bias) + + if not return_all_features: + x = self.norm(x) + if self.fc_norm is not None: + return self.fc_norm(x.mean(1)) + else: + return x[:, 0] + return x + + def forward(self, x, return_all_features=False): + if return_all_features: + return self.forward_features(x, return_all_features) + x = self.forward_features(x) + x = self.head(x) + return x diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/factory.py b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/factory.py new file mode 100644 index 0000000000000000000000000000000000000000..535eeacb7d780d473fad6cf360e38254af86cb13 --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/factory.py @@ -0,0 +1,528 @@ +import json +import logging +import os +import pathlib +import re +from copy import deepcopy +from pathlib import Path +from typing import Optional, Tuple, Union, Dict, Any +import torch + +try: + import deepspeed +except ImportError: + deepspeed = None + +from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD +from .model import CLIP, CustomCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict, get_cast_dtype +from .openai import load_openai_model +from .pretrained import is_pretrained_cfg, get_pretrained_cfg, download_pretrained, list_pretrained_tags_by_model +from .transform import image_transform +from .tokenizer import HFTokenizer, tokenize +from .utils import resize_clip_pos_embed, resize_evaclip_pos_embed, resize_visual_pos_embed, resize_eva_pos_embed + + +_MODEL_CONFIG_PATHS = [Path(__file__).parent / f"model_configs/"] +_MODEL_CONFIGS = {} # directory (model_name: config) of model architecture configs + + +def _natural_key(string_): + return [int(s) if s.isdigit() else s for s in re.split(r"(\d+)", string_.lower())] + + +def _rescan_model_configs(): + global _MODEL_CONFIGS + + config_ext = (".json",) + config_files = [] + for config_path in _MODEL_CONFIG_PATHS: + if config_path.is_file() and config_path.suffix in config_ext: + config_files.append(config_path) + elif config_path.is_dir(): + for ext in config_ext: + config_files.extend(config_path.glob(f"*{ext}")) + + for cf in config_files: + with open(cf, "r", encoding="utf8") as f: + model_cfg = json.load(f) + if all(a in model_cfg for a in ("embed_dim", "vision_cfg", "text_cfg")): + _MODEL_CONFIGS[cf.stem] = model_cfg + + _MODEL_CONFIGS = dict(sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))) + + +_rescan_model_configs() # initial populate of model config registry + + +def list_models(): + """enumerate available model architectures based on config files""" + return list(_MODEL_CONFIGS.keys()) + + +def add_model_config(path): + """add model config path or file and update registry""" + if not isinstance(path, Path): + path = Path(path) + _MODEL_CONFIG_PATHS.append(path) + _rescan_model_configs() + + +def get_model_config(model_name): + if model_name in _MODEL_CONFIGS: + return deepcopy(_MODEL_CONFIGS[model_name]) + else: + return None + + +def get_tokenizer(model_name): + config = get_model_config(model_name) + tokenizer = HFTokenizer(config["text_cfg"]["hf_tokenizer_name"]) if "hf_tokenizer_name" in config["text_cfg"] else tokenize + return tokenizer + + +# loading openai CLIP weights when is_openai=True for training +def load_state_dict(checkpoint_path: str, map_location: str = "cpu", model_key: str = "model|module|state_dict", is_openai: bool = False, skip_list: list = []): + if is_openai: + model = torch.jit.load(checkpoint_path, map_location="cpu").eval() + state_dict = model.state_dict() + for key in ["input_resolution", "context_length", "vocab_size"]: + state_dict.pop(key, None) + else: + checkpoint = torch.load(checkpoint_path, map_location=map_location) + for mk in model_key.split("|"): + if isinstance(checkpoint, dict) and mk in checkpoint: + state_dict = checkpoint[mk] + break + else: + state_dict = checkpoint + if next(iter(state_dict.items()))[0].startswith("module"): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + for k in skip_list: + if k in list(state_dict.keys()): + logging.info(f"Removing key {k} from pretrained checkpoint") + del state_dict[k] + + if os.getenv("RoPE") == "1": + for k in list(state_dict.keys()): + if "freqs_cos" in k or "freqs_sin" in k: + del state_dict[k] + return state_dict + + +def load_checkpoint(model, checkpoint_path, model_key="model|module|state_dict", strict=True): + state_dict = load_state_dict(checkpoint_path, model_key=model_key, is_openai=False) + # detect old format and make compatible with new format + if "positional_embedding" in state_dict and not hasattr(model, "positional_embedding"): + state_dict = convert_to_custom_text_state_dict(state_dict) + if "text.logit_scale" in state_dict and hasattr(model, "logit_scale"): + state_dict["logit_scale"] = state_dict["text.logit_scale"] + del state_dict["text.logit_scale"] + + # resize_clip_pos_embed for CLIP and open CLIP + if "visual.positional_embedding" in state_dict: + resize_clip_pos_embed(state_dict, model) + # specified to eva_vit_model + elif "visual.pos_embed" in state_dict: + resize_evaclip_pos_embed(state_dict, model) + + # resize_clip_pos_embed(state_dict, model) + incompatible_keys = model.load_state_dict(state_dict, strict=strict) + logging.info(f"incompatible_keys.missing_keys: {incompatible_keys.missing_keys}") + return incompatible_keys + + +def load_clip_visual_state_dict(checkpoint_path: str, map_location: str = "cpu", is_openai: bool = False, skip_list: list = []): + state_dict = load_state_dict(checkpoint_path, map_location=map_location, is_openai=is_openai, skip_list=skip_list) + + for k in list(state_dict.keys()): + if not k.startswith("visual."): + del state_dict[k] + for k in list(state_dict.keys()): + if k.startswith("visual."): + new_k = k[7:] + state_dict[new_k] = state_dict[k] + del state_dict[k] + return state_dict + + +def load_clip_text_state_dict(checkpoint_path: str, map_location: str = "cpu", is_openai: bool = False, skip_list: list = []): + state_dict = load_state_dict(checkpoint_path, map_location=map_location, is_openai=is_openai, skip_list=skip_list) + + for k in list(state_dict.keys()): + if k.startswith("visual."): + del state_dict[k] + return state_dict + + +def get_pretrained_tag(pretrained_model): + pretrained_model = pretrained_model.lower() + if "laion" in pretrained_model or "open_clip" in pretrained_model: + return "open_clip" + elif "openai" in pretrained_model: + return "clip" + elif "eva" in pretrained_model and "clip" in pretrained_model: + return "eva_clip" + else: + return "other" + + +def load_zero_partitions(model, state_dict, is_deepspeed_zero3_enabled, pretrained_model_path, ignore_mismatched_sizes=False): + """ + adept from pytorch lightning and transformers + with deepspeed.zero.Init(): + model = MyModel() + state_dict = torch.load(model_path, map_location="cpu") + load_zero_partitions(model, prefix="") + """ + + # because zero3 puts placeholders in model params, this context + # manager gathers (unpartitions) the params of the current layer, then loads from + # the state dict and then re-partitions them again + model_state_dict = model.state_dict() + expected_keys = list(model_state_dict.keys()) + loaded_keys = list(state_dict.keys()) + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + # Mistmatched keys contains tuples key/shape1/shape2 of weights in the checkpoint that have a shape not + # matching the weights in the model. + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + + if model_key in model_state_dict and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape: + mismatched_keys.append((checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape)) + del state_dict[checkpoint_key] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, "_metadata", None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + error_msgs = [] + + # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants + # so we need to apply the function recursively. + def load(module, prefix=""): + local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) + args = (state_dict, prefix, local_metadata, True, [], [], error_msgs) + if is_deepspeed_zero3_enabled: + # because zero3 puts placeholders in model params, this context + # manager gathers (unpartitions) the params of the current layer, then loads from + # the state dict and then re-partitions them again + with deepspeed.zero.GatheredParameters(list(module.parameters(recurse=False)), modifier_rank=0): + if torch.distributed.get_rank() == 0: + module._load_from_state_dict(*args) + else: + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + ".") + + # Make sure we are able to load base models as well as derived models (with heads) + start_prefix = "" + model_to_load = model + load(model_to_load, prefix=start_prefix) + del state_dict + if len(error_msgs) > 0: + error_msg = "\n\t".join(error_msgs) + if "size mismatch" in error_msg: + error_msg += "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method." + raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}") + if len(unexpected_keys) > 0: + logging.warning( + f"Some weights of the model checkpoint at {pretrained_model_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or" + " with another architecture (e.g. initializing a BertForSequenceClassification model from a" + " BertForPreTraining model).\n- This IS NOT expected if you are initializing" + f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly identical" + " (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)." + ) + else: + logging.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + if len(missing_keys) > 0: + logging.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + elif len(mismatched_keys) == 0: + logging.info( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_path}.\nIf your task is similar to the task the model of the checkpoint" + f" was trained on, you can already use {model.__class__.__name__} for predictions without further" + " training." + ) + if len(mismatched_keys) > 0: + mismatched_warning = "\n".join([f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated" for key, shape1, shape2 in mismatched_keys]) + logging.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_path} and are newly initialized because the shapes did not" + f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be able" + " to use it for predictions and inference." + ) + + +def load_pretrained_checkpoint(model, visual_checkpoint_path, text_checkpoint_path, strict=True, visual_model=None, text_model=None, model_key="model|module|state_dict", skip_list=[]): + visual_tag = get_pretrained_tag(visual_model) + text_tag = get_pretrained_tag(text_model) + + logging.info(f"num of model state_dict keys: {len(model.state_dict().keys())}") + visual_incompatible_keys, text_incompatible_keys = None, None + if visual_checkpoint_path: + if visual_tag == "eva_clip" or visual_tag == "open_clip": + visual_state_dict = load_clip_visual_state_dict(visual_checkpoint_path, is_openai=False, skip_list=skip_list) + elif visual_tag == "clip": + visual_state_dict = load_clip_visual_state_dict(visual_checkpoint_path, is_openai=True, skip_list=skip_list) + else: + visual_state_dict = load_state_dict(visual_checkpoint_path, model_key=model_key, is_openai=False, skip_list=skip_list) + + # resize_clip_pos_embed for CLIP and open CLIP + if "positional_embedding" in visual_state_dict: + resize_visual_pos_embed(visual_state_dict, model) + # specified to EVA model + elif "pos_embed" in visual_state_dict: + resize_eva_pos_embed(visual_state_dict, model) + + visual_incompatible_keys = model.visual.load_state_dict(visual_state_dict, strict=strict) + logging.info(f"num of loaded visual_state_dict keys: {len(visual_state_dict.keys())}") + logging.info(f"visual_incompatible_keys.missing_keys: {visual_incompatible_keys.missing_keys}") + + if text_checkpoint_path: + if text_tag == "eva_clip" or text_tag == "open_clip": + text_state_dict = load_clip_text_state_dict(text_checkpoint_path, is_openai=False, skip_list=skip_list) + elif text_tag == "clip": + text_state_dict = load_clip_text_state_dict(text_checkpoint_path, is_openai=True, skip_list=skip_list) + else: + text_state_dict = load_state_dict(visual_checkpoint_path, model_key=model_key, is_openai=False, skip_list=skip_list) + + text_incompatible_keys = model.text.load_state_dict(text_state_dict, strict=strict) + + logging.info(f"num of loaded text_state_dict keys: {len(text_state_dict.keys())}") + logging.info(f"text_incompatible_keys.missing_keys: {text_incompatible_keys.missing_keys}") + + return visual_incompatible_keys, text_incompatible_keys + + +def create_model( + model_name: str, + pretrained: Optional[str] = None, + precision: str = "fp32", + device: Union[str, torch.device] = "cpu", + jit: bool = False, + force_quick_gelu: bool = False, + force_custom_clip: bool = False, + force_patch_dropout: Optional[float] = None, + pretrained_image: str = "", + pretrained_text: str = "", + pretrained_hf: bool = True, + pretrained_visual_model: str = None, + pretrained_text_model: str = None, + cache_dir: Optional[str] = None, + skip_list: list = [], +): + model_name = model_name.replace("/", "-") # for callers using old naming with / in ViT names + if isinstance(device, str): + device = torch.device(device) + + if pretrained and pretrained.lower() == "openai": + logging.info(f"Loading pretrained {model_name} from OpenAI.") + model = load_openai_model( + model_name, + precision=precision, + device=device, + jit=jit, + cache_dir=cache_dir, + ) + else: + model_cfg = get_model_config(model_name) + if model_cfg is not None: + logging.info(f"Loaded {model_name} model config.") + else: + logging.error(f"Model config for {model_name} not found; available models {list_models()}.") + raise RuntimeError(f"Model config for {model_name} not found.") + + if "rope" in model_cfg.get("vision_cfg", {}): + if model_cfg["vision_cfg"]["rope"]: + os.environ["RoPE"] = "1" + else: + os.environ["RoPE"] = "0" + + if force_quick_gelu: + # override for use of QuickGELU on non-OpenAI transformer models + model_cfg["quick_gelu"] = True + + if force_patch_dropout is not None: + # override the default patch dropout value + model_cfg["vision_cfg"]["patch_dropout"] = force_patch_dropout + + cast_dtype = get_cast_dtype(precision) + custom_clip = model_cfg.pop("custom_text", False) or force_custom_clip or ("hf_model_name" in model_cfg["text_cfg"]) + + if custom_clip: + if "hf_model_name" in model_cfg.get("text_cfg", {}): + model_cfg["text_cfg"]["hf_model_pretrained"] = pretrained_hf + model = CustomCLIP(**model_cfg, cast_dtype=cast_dtype) + else: + model = CLIP(**model_cfg, cast_dtype=cast_dtype) + + pretrained_cfg = {} + if pretrained: + checkpoint_path = "" + pretrained_cfg = get_pretrained_cfg(model_name, pretrained) + if pretrained_cfg: + checkpoint_path = download_pretrained(pretrained_cfg, cache_dir=cache_dir) + elif os.path.exists(pretrained): + checkpoint_path = pretrained + + if checkpoint_path: + logging.info(f"Loading pretrained {model_name} weights ({pretrained}).") + load_checkpoint(model, checkpoint_path, model_key="model|module|state_dict", strict=False) + else: + error_str = f"Pretrained weights ({pretrained}) not found for model {model_name}." f"Available pretrained tags ({list_pretrained_tags_by_model(model_name)}." + logging.warning(error_str) + raise RuntimeError(error_str) + else: + visual_checkpoint_path = "" + text_checkpoint_path = "" + + if pretrained_image: + pretrained_visual_model = pretrained_visual_model.replace("/", "-") # for callers using old naming with / in ViT names + pretrained_image_cfg = get_pretrained_cfg(pretrained_visual_model, pretrained_image) + if "timm_model_name" in model_cfg.get("vision_cfg", {}): + # pretrained weight loading for timm models set via vision_cfg + model_cfg["vision_cfg"]["timm_model_pretrained"] = True + elif pretrained_image_cfg: + visual_checkpoint_path = download_pretrained(pretrained_image_cfg, cache_dir=cache_dir) + elif os.path.exists(pretrained_image): + visual_checkpoint_path = pretrained_image + else: + logging.warning(f"Pretrained weights ({visual_checkpoint_path}) not found for model {model_name}.visual.") + raise RuntimeError(f"Pretrained weights ({visual_checkpoint_path}) not found for model {model_name}.visual.") + + if pretrained_text: + pretrained_text_model = pretrained_text_model.replace("/", "-") # for callers using old naming with / in ViT names + pretrained_text_cfg = get_pretrained_cfg(pretrained_text_model, pretrained_text) + if pretrained_image_cfg: + text_checkpoint_path = download_pretrained(pretrained_text_cfg, cache_dir=cache_dir) + elif os.path.exists(pretrained_text): + text_checkpoint_path = pretrained_text + else: + logging.warning(f"Pretrained weights ({text_checkpoint_path}) not found for model {model_name}.text.") + raise RuntimeError(f"Pretrained weights ({text_checkpoint_path}) not found for model {model_name}.text.") + + if visual_checkpoint_path: + logging.info(f"Loading pretrained {model_name}.visual weights ({visual_checkpoint_path}).") + if text_checkpoint_path: + logging.info(f"Loading pretrained {model_name}.text weights ({text_checkpoint_path}).") + + if visual_checkpoint_path or text_checkpoint_path: + load_pretrained_checkpoint(model, visual_checkpoint_path, text_checkpoint_path, strict=False, visual_model=pretrained_visual_model, text_model=pretrained_text_model, model_key="model|module|state_dict", skip_list=skip_list) + + if "fp16" in precision or "bf16" in precision: + logging.info(f"convert precision to {precision}") + model = model.to(torch.bfloat16) if "bf16" in precision else model.to(torch.float16) + + # model.to(device=device) + + # set image / mean metadata from pretrained_cfg if available, or use default + model.visual.image_mean = pretrained_cfg.get("mean", None) or OPENAI_DATASET_MEAN + model.visual.image_std = pretrained_cfg.get("std", None) or OPENAI_DATASET_STD + + if jit: + model = torch.jit.script(model) + + return model + + +def create_model_and_transforms( + model_name: str, + pretrained: Optional[str] = None, + precision: str = "fp32", + device: Union[str, torch.device] = "cpu", + jit: bool = False, + force_quick_gelu: bool = False, + force_custom_clip: bool = False, + force_patch_dropout: Optional[float] = None, + pretrained_image: str = "", + pretrained_text: str = "", + pretrained_hf: bool = True, + pretrained_visual_model: str = None, + pretrained_text_model: str = None, + image_mean: Optional[Tuple[float, ...]] = None, + image_std: Optional[Tuple[float, ...]] = None, + cache_dir: Optional[str] = None, + skip_list: list = [], +): + model = create_model( + model_name, + pretrained, + precision=precision, + device=device, + jit=jit, + force_quick_gelu=force_quick_gelu, + force_custom_clip=force_custom_clip, + force_patch_dropout=force_patch_dropout, + pretrained_image=pretrained_image, + pretrained_text=pretrained_text, + pretrained_hf=pretrained_hf, + pretrained_visual_model=pretrained_visual_model, + pretrained_text_model=pretrained_text_model, + cache_dir=cache_dir, + skip_list=skip_list, + ) + + image_mean = image_mean or getattr(model.visual, "image_mean", None) + image_std = image_std or getattr(model.visual, "image_std", None) + preprocess_train = image_transform(model.visual.image_size, is_train=True, mean=image_mean, std=image_std) + preprocess_val = image_transform(model.visual.image_size, is_train=False, mean=image_mean, std=image_std) + + return model, preprocess_train, preprocess_val + + +def create_model_from_pretrained( + model_name: str, + pretrained: str, + precision: str = "fp32", + device: Union[str, torch.device] = "cpu", + jit: bool = False, + force_quick_gelu: bool = False, + force_custom_clip: bool = False, + force_patch_dropout: Optional[float] = None, + return_transform: bool = True, + image_mean: Optional[Tuple[float, ...]] = None, + image_std: Optional[Tuple[float, ...]] = None, + cache_dir: Optional[str] = None, + is_frozen: bool = False, +): + if not is_pretrained_cfg(model_name, pretrained) and not os.path.exists(pretrained): + raise RuntimeError(f"{pretrained} is not a valid pretrained cfg or checkpoint for {model_name}." f" Use open_clip.list_pretrained() to find one.") + + model = create_model( + model_name, + pretrained, + precision=precision, + device=device, + jit=jit, + force_quick_gelu=force_quick_gelu, + force_custom_clip=force_custom_clip, + force_patch_dropout=force_patch_dropout, + cache_dir=cache_dir, + ) + + if is_frozen: + for param in model.parameters(): + param.requires_grad = False + + if not return_transform: + return model + + image_mean = image_mean or getattr(model.visual, "image_mean", None) + image_std = image_std or getattr(model.visual, "image_std", None) + preprocess = image_transform(model.visual.image_size, is_train=False, mean=image_mean, std=image_std) + + return model, preprocess diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_configs.py b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..ddd2c672fdcc895ce2e437f6da5ee439c09840da --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_configs.py @@ -0,0 +1,57 @@ +# HF architecture dict: +arch_dict = { + # https://huggingface.co/docs/transformers/model_doc/roberta#roberta + "roberta": { + "config_names": { + "context_length": "max_position_embeddings", + "vocab_size": "vocab_size", + "width": "hidden_size", + "heads": "num_attention_heads", + "layers": "num_hidden_layers", + "layer_attr": "layer", + "token_embeddings_attr": "embeddings", + }, + "pooler": "mean_pooler", + }, + # https://huggingface.co/docs/transformers/model_doc/xlm-roberta#transformers.XLMRobertaConfig + "xlm-roberta": { + "config_names": { + "context_length": "max_position_embeddings", + "vocab_size": "vocab_size", + "width": "hidden_size", + "heads": "num_attention_heads", + "layers": "num_hidden_layers", + "layer_attr": "layer", + "token_embeddings_attr": "embeddings", + }, + "pooler": "mean_pooler", + }, + # https://huggingface.co/docs/transformers/model_doc/mt5#mt5 + "mt5": { + "config_names": { + # unlimited seqlen + # https://github.com/google-research/text-to-text-transfer-transformer/issues/273 + # https://github.com/huggingface/transformers/blob/v4.24.0/src/transformers/models/t5/modeling_t5.py#L374 + "context_length": "", + "vocab_size": "vocab_size", + "width": "d_model", + "heads": "num_heads", + "layers": "num_layers", + "layer_attr": "block", + "token_embeddings_attr": "embed_tokens", + }, + "pooler": "mean_pooler", + }, + "bert": { + "config_names": { + "context_length": "max_position_embeddings", + "vocab_size": "vocab_size", + "width": "hidden_size", + "heads": "num_attention_heads", + "layers": "num_hidden_layers", + "layer_attr": "layer", + "token_embeddings_attr": "embeddings", + }, + "pooler": "mean_pooler", + }, +} diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B.json b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B.json new file mode 100644 index 0000000000000000000000000000000000000000..689492a25d365436fd85ed432e6fb7295ca1c7bd --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B.json @@ -0,0 +1,27 @@ +{ + "embed_dim": 1280, + "vision_cfg": { + "image_size": 224, + "layers": 32, + "width": 4096, + "head_width": 128, + "mlp_ratio": 5, + "patch_size": 14, + "eva_model_name": "eva-clip-8b-14-x", + "drop_path_rate": 0, + "qkv_bias": false, + "xattn": true, + "postnorm": false, + "fusedLN": false, + "use_rms_norm": true + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 1280, + "heads": 20, + "layers": 32, + "xattn": false, + "fusedLN": false + } +} \ No newline at end of file diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-B-16.json b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-B-16.json new file mode 100644 index 0000000000000000000000000000000000000000..e4a6e723f77033caa341ddf9b5be1787d64ad42c --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-B-16.json @@ -0,0 +1,29 @@ +{ + "embed_dim": 512, + "vision_cfg": { + "image_size": 224, + "layers": 12, + "width": 768, + "head_width": 64, + "patch_size": 16, + "mlp_ratio": 2.6667, + "eva_model_name": "eva-clip-b-16-X", + "drop_path_rate": 0.0, + "xattn": true, + "fusedLN": true, + "rope": true, + "pt_hw_seq_len": 16, + "intp_freq": true, + "naiveswiglu": true, + "subln": true + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12, + "xattn": true, + "fusedLN": true + } +} \ No newline at end of file diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/timm_model.py b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/timm_model.py new file mode 100644 index 0000000000000000000000000000000000000000..65de78df2892a4cbe01b9eef68f07f5f79b8b367 --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/timm_model.py @@ -0,0 +1,114 @@ +""" timm model adapter + +Wraps timm (https://github.com/rwightman/pytorch-image-models) models for use as a vision tower in CLIP model. +""" + +import logging +from collections import OrderedDict + +import torch +import torch.nn as nn + +try: + import timm + from timm.models.layers import Mlp, to_2tuple + + try: + # old timm imports < 0.8.1 + from timm.models.layers.attention_pool2d import RotAttentionPool2d + from timm.models.layers.attention_pool2d import AttentionPool2d as AbsAttentionPool2d + except ImportError: + # new timm imports >= 0.8.1 + from timm.layers import RotAttentionPool2d + from timm.layers import AttentionPool2d as AbsAttentionPool2d +except ImportError: + timm = None + +from .utils import freeze_batch_norm_2d + + +class TimmModel(nn.Module): + """timm model adapter + # FIXME this adapter is a work in progress, may change in ways that break weight compat + """ + + def __init__(self, model_name, embed_dim, image_size=224, pool="avg", proj="linear", proj_bias=False, drop=0.0, pretrained=False): + super().__init__() + if timm is None: + raise RuntimeError("Please `pip install timm` to use timm models.") + + self.image_size = to_2tuple(image_size) + self.trunk = timm.create_model(model_name, pretrained=pretrained) + feat_size = self.trunk.default_cfg.get("pool_size", None) + feature_ndim = 1 if not feat_size else 2 + if pool in ("abs_attn", "rot_attn"): + assert feature_ndim == 2 + # if attn pooling used, remove both classifier and default pool + self.trunk.reset_classifier(0, global_pool="") + else: + # reset global pool if pool config set, otherwise leave as network default + reset_kwargs = dict(global_pool=pool) if pool else {} + self.trunk.reset_classifier(0, **reset_kwargs) + prev_chs = self.trunk.num_features + + head_layers = OrderedDict() + if pool == "abs_attn": + head_layers["pool"] = AbsAttentionPool2d(prev_chs, feat_size=feat_size, out_features=embed_dim) + prev_chs = embed_dim + elif pool == "rot_attn": + head_layers["pool"] = RotAttentionPool2d(prev_chs, out_features=embed_dim) + prev_chs = embed_dim + else: + assert proj, "projection layer needed if non-attention pooling is used." + + # NOTE attention pool ends with a projection layer, so proj should usually be set to '' if such pooling is used + if proj == "linear": + head_layers["drop"] = nn.Dropout(drop) + head_layers["proj"] = nn.Linear(prev_chs, embed_dim, bias=proj_bias) + elif proj == "mlp": + head_layers["mlp"] = Mlp(prev_chs, 2 * embed_dim, embed_dim, drop=drop, bias=(True, proj_bias)) + + self.head = nn.Sequential(head_layers) + + def lock(self, unlocked_groups=0, freeze_bn_stats=False): + """lock modules + Args: + unlocked_groups (int): leave last n layer groups unlocked (default: 0) + """ + if not unlocked_groups: + # lock full model + for param in self.trunk.parameters(): + param.requires_grad = False + if freeze_bn_stats: + freeze_batch_norm_2d(self.trunk) + else: + # NOTE: partial freeze requires latest timm (master) branch and is subject to change + try: + # FIXME import here until API stable and in an official release + from timm.models.helpers import group_parameters, group_modules + except ImportError: + raise RuntimeError("Please install latest timm `pip install git+https://github.com/rwightman/pytorch-image-models`") + matcher = self.trunk.group_matcher() + gparams = group_parameters(self.trunk, matcher) + max_layer_id = max(gparams.keys()) + max_layer_id = max_layer_id - unlocked_groups + for group_idx in range(max_layer_id + 1): + group = gparams[group_idx] + for param in group: + self.trunk.get_parameter(param).requires_grad = False + if freeze_bn_stats: + gmodules = group_modules(self.trunk, matcher, reverse=True) + gmodules = {k for k, v in gmodules.items() if v <= max_layer_id} + freeze_batch_norm_2d(self.trunk, gmodules) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + try: + self.trunk.set_grad_checkpointing(enable) + except Exception as e: + logging.warning("grad checkpointing not supported for this timm image tower, continuing without...") + + def forward(self, x): + x = self.trunk(x) + x = self.head(x) + return x diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transform.py b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transform.py new file mode 100644 index 0000000000000000000000000000000000000000..8cad45a167ab85eba8f84eed7feaa132dedc48d4 --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transform.py @@ -0,0 +1,104 @@ +from typing import Optional, Sequence, Tuple + +import torch +import torch.nn as nn +import torchvision.transforms.functional as F + +from torchvision.transforms import Normalize, Compose, RandomResizedCrop, InterpolationMode, ToTensor, Resize, CenterCrop + +from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD + + +class ResizeMaxSize(nn.Module): + + def __init__(self, max_size, interpolation=InterpolationMode.BICUBIC, fn="max", fill=0): + super().__init__() + if not isinstance(max_size, int): + raise TypeError(f"Size should be int. Got {type(max_size)}") + self.max_size = max_size + self.interpolation = interpolation + self.fn = min if fn == "min" else min + self.fill = fill + + def forward(self, img): + if isinstance(img, torch.Tensor): + height, width = img.shape[:2] + else: + width, height = img.size + scale = self.max_size / float(max(height, width)) + if scale != 1.0: + new_size = tuple(round(dim * scale) for dim in (height, width)) + img = F.resize(img, new_size, self.interpolation) + pad_h = self.max_size - new_size[0] + pad_w = self.max_size - new_size[1] + img = F.pad(img, padding=[pad_w // 2, pad_h // 2, pad_w - pad_w // 2, pad_h - pad_h // 2], fill=self.fill) + return img + + +def _convert_to_rgb(image): + return image.convert("RGB") + + +# class CatGen(nn.Module): +# def __init__(self, num=4): +# self.num = num +# def mixgen_batch(image, text): +# batch_size = image.shape[0] +# index = np.random.permutation(batch_size) + +# cat_images = [] +# for i in range(batch_size): +# # image mixup +# image[i,:] = lam * image[i,:] + (1 - lam) * image[index[i],:] +# # text concat +# text[i] = tokenizer((str(text[i]) + " " + str(text[index[i]])))[0] +# text = torch.stack(text) +# return image, text + + +def image_transform( + image_size: int, + is_train: bool, + mean: Optional[Tuple[float, ...]] = None, + std: Optional[Tuple[float, ...]] = None, + resize_longest_max: bool = False, + fill_color: int = 0, +): + mean = mean or OPENAI_DATASET_MEAN + if not isinstance(mean, (list, tuple)): + mean = (mean,) * 3 + + std = std or OPENAI_DATASET_STD + if not isinstance(std, (list, tuple)): + std = (std,) * 3 + + if isinstance(image_size, (list, tuple)) and image_size[0] == image_size[1]: + # for square size, pass size as int so that Resize() uses aspect preserving shortest edge + image_size = image_size[0] + + normalize = Normalize(mean=mean, std=std) + if is_train: + return Compose( + [ + RandomResizedCrop(image_size, scale=(0.9, 1.0), interpolation=InterpolationMode.BICUBIC), + _convert_to_rgb, + ToTensor(), + normalize, + ] + ) + else: + if resize_longest_max: + transforms = [ResizeMaxSize(image_size, fill=fill_color)] + else: + transforms = [ + Resize(image_size, interpolation=InterpolationMode.BICUBIC), + CenterCrop(image_size), + ] + transforms.extend( + [ + _convert_to_rgb, + ToTensor(), + normalize, + ] + ) + return Compose(transforms) diff --git a/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/utils.py b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..73b6d66273fcbaf5b8c19df5e304db533ba88c49 --- /dev/null +++ b/LaViDa/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/utils.py @@ -0,0 +1,321 @@ +from itertools import repeat +import collections.abc +import logging +import math +import numpy as np + +import torch +from torch import nn as nn +from torchvision.ops.misc import FrozenBatchNorm2d +import torch.nn.functional as F + + +# open CLIP +def resize_clip_pos_embed(state_dict, model, interpolation: str = "bicubic", seq_dim=1): + # Rescale the grid of position embeddings when loading from state_dict + old_pos_embed = state_dict.get("visual.positional_embedding", None) + if old_pos_embed is None or not hasattr(model.visual, "grid_size"): + return + grid_size = to_2tuple(model.visual.grid_size) + extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more) + new_seq_len = grid_size[0] * grid_size[1] + extra_tokens + if new_seq_len == old_pos_embed.shape[0]: + return + + if extra_tokens: + pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:] + else: + pos_emb_tok, pos_emb_img = None, old_pos_embed + old_grid_size = to_2tuple(int(math.sqrt(len(pos_emb_img)))) + + logging.info("Resizing position embedding grid-size from %s to %s", old_grid_size, grid_size) + pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2) + pos_emb_img = F.interpolate( + pos_emb_img, + size=grid_size, + mode=interpolation, + align_corners=True, + ) + pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0] + if pos_emb_tok is not None: + new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0) + else: + new_pos_embed = pos_emb_img + state_dict["visual.positional_embedding"] = new_pos_embed + + +def resize_visual_pos_embed(state_dict, model, interpolation: str = "bicubic", seq_dim=1): + # Rescale the grid of position embeddings when loading from state_dict + old_pos_embed = state_dict.get("positional_embedding", None) + if old_pos_embed is None or not hasattr(model.visual, "grid_size"): + return + grid_size = to_2tuple(model.visual.grid_size) + extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more) + new_seq_len = grid_size[0] * grid_size[1] + extra_tokens + if new_seq_len == old_pos_embed.shape[0]: + return + + if extra_tokens: + pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:] + else: + pos_emb_tok, pos_emb_img = None, old_pos_embed + old_grid_size = to_2tuple(int(math.sqrt(len(pos_emb_img)))) + + logging.info("Resizing position embedding grid-size from %s to %s", old_grid_size, grid_size) + pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2) + pos_emb_img = F.interpolate( + pos_emb_img, + size=grid_size, + mode=interpolation, + align_corners=True, + ) + pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0] + if pos_emb_tok is not None: + new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0) + else: + new_pos_embed = pos_emb_img + state_dict["positional_embedding"] = new_pos_embed + + +def resize_evaclip_pos_embed(state_dict, model, interpolation: str = "bicubic", seq_dim=1): + all_keys = list(state_dict.keys()) + # interpolate position embedding + if "visual.pos_embed" in state_dict: + pos_embed_checkpoint = state_dict["visual.pos_embed"] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.visual.patch_embed.num_patches + # num_extra_tokens = model.visual.pos_embed.shape[-2] - num_patches + num_extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more) + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches**0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=(new_size, new_size), mode="bicubic", align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + state_dict["visual.pos_embed"] = new_pos_embed + + patch_embed_proj = state_dict["visual.patch_embed.proj.weight"] + patch_size = model.visual.patch_embed.patch_size + state_dict["visual.patch_embed.proj.weight"] = torch.nn.functional.interpolate(patch_embed_proj.float(), size=patch_size, mode="bicubic", align_corners=False) + + +def resize_eva_pos_embed(state_dict, model, interpolation: str = "bicubic", seq_dim=1): + all_keys = list(state_dict.keys()) + # interpolate position embedding + if "pos_embed" in state_dict: + pos_embed_checkpoint = state_dict["pos_embed"] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.visual.patch_embed.num_patches + # num_extra_tokens = model.visual.pos_embed.shape[-2] - num_patches + num_extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more) + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches**0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=(new_size, new_size), mode="bicubic", align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + state_dict["pos_embed"] = new_pos_embed + + patch_embed_proj = state_dict["patch_embed.proj.weight"] + patch_size = model.visual.patch_embed.patch_size + state_dict["patch_embed.proj.weight"] = torch.nn.functional.interpolate(patch_embed_proj.float(), size=patch_size, mode="bicubic", align_corners=False) + + +def resize_rel_pos_embed(state_dict, model, interpolation: str = "bicubic", seq_dim=1): + all_keys = list(state_dict.keys()) + for key in all_keys: + if "relative_position_index" in key: + state_dict.pop(key) + + if "relative_position_bias_table" in key: + rel_pos_bias = state_dict[key] + src_num_pos, num_attn_heads = rel_pos_bias.size() + dst_num_pos, _ = model.visual.state_dict()[key].size() + dst_patch_shape = model.visual.patch_embed.patch_shape + if dst_patch_shape[0] != dst_patch_shape[1]: + raise NotImplementedError() + num_extra_tokens = dst_num_pos - (dst_patch_shape[0] * 2 - 1) * (dst_patch_shape[1] * 2 - 1) + src_size = int((src_num_pos - num_extra_tokens) ** 0.5) + dst_size = int((dst_num_pos - num_extra_tokens) ** 0.5) + if src_size != dst_size: + print("Position interpolate for %s from %dx%d to %dx%d" % (key, src_size, src_size, dst_size, dst_size)) + extra_tokens = rel_pos_bias[-num_extra_tokens:, :] + rel_pos_bias = rel_pos_bias[:-num_extra_tokens, :] + + def geometric_progression(a, r, n): + return a * (1.0 - r**n) / (1.0 - r) + + left, right = 1.01, 1.5 + while right - left > 1e-6: + q = (left + right) / 2.0 + gp = geometric_progression(1, q, src_size // 2) + if gp > dst_size // 2: + right = q + else: + left = q + + # if q > 1.090307: + # q = 1.090307 + + dis = [] + cur = 1 + for i in range(src_size // 2): + dis.append(cur) + cur += q ** (i + 1) + + r_ids = [-_ for _ in reversed(dis)] + + x = r_ids + [0] + dis + y = r_ids + [0] + dis + + t = dst_size // 2.0 + dx = np.arange(-t, t + 0.1, 1.0) + dy = np.arange(-t, t + 0.1, 1.0) + + print("Original positions = %s" % str(x)) + print("Target positions = %s" % str(dx)) + + all_rel_pos_bias = [] + + for i in range(num_attn_heads): + z = rel_pos_bias[:, i].view(src_size, src_size).float().numpy() + f = F.interpolate.interp2d(x, y, z, kind="cubic") + all_rel_pos_bias.append(torch.Tensor(f(dx, dy)).contiguous().view(-1, 1).to(rel_pos_bias.device)) + + rel_pos_bias = torch.cat(all_rel_pos_bias, dim=-1) + + new_rel_pos_bias = torch.cat((rel_pos_bias, extra_tokens), dim=0) + state_dict[key] = new_rel_pos_bias + + # interpolate position embedding + if "pos_embed" in state_dict: + pos_embed_checkpoint = state_dict["pos_embed"] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.visual.patch_embed.num_patches + num_extra_tokens = model.visual.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches**0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=(new_size, new_size), mode="bicubic", align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + state_dict["pos_embed"] = new_pos_embed + + patch_embed_proj = state_dict["patch_embed.proj.weight"] + patch_size = model.visual.patch_embed.patch_size + state_dict["patch_embed.proj.weight"] = torch.nn.functional.interpolate(patch_embed_proj.float(), size=patch_size, mode="bicubic", align_corners=False) + + +def freeze_batch_norm_2d(module, module_match={}, name=""): + """ + Converts all `BatchNorm2d` and `SyncBatchNorm` layers of provided module into `FrozenBatchNorm2d`. If `module` is + itself an instance of either `BatchNorm2d` or `SyncBatchNorm`, it is converted into `FrozenBatchNorm2d` and + returned. Otherwise, the module is walked recursively and submodules are converted in place. + + Args: + module (torch.nn.Module): Any PyTorch module. + module_match (dict): Dictionary of full module names to freeze (all if empty) + name (str): Full module name (prefix) + + Returns: + torch.nn.Module: Resulting module + + Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762 + """ + res = module + is_match = True + if module_match: + is_match = name in module_match + if is_match and isinstance(module, (nn.modules.batchnorm.BatchNorm2d, nn.modules.batchnorm.SyncBatchNorm)): + res = FrozenBatchNorm2d(module.num_features) + res.num_features = module.num_features + res.affine = module.affine + if module.affine: + res.weight.data = module.weight.data.clone().detach() + res.bias.data = module.bias.data.clone().detach() + res.running_mean.data = module.running_mean.data + res.running_var.data = module.running_var.data + res.eps = module.eps + else: + for child_name, child in module.named_children(): + full_child_name = ".".join([name, child_name]) if name else child_name + new_child = freeze_batch_norm_2d(child, module_match, full_child_name) + if new_child is not child: + res.add_module(child_name, new_child) + return res + + +# From PyTorch internals +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = lambda n, x: _ntuple(n)(x) + + +def is_logging(args): + def is_global_master(args): + return args.rank == 0 + + def is_local_master(args): + return args.local_rank == 0 + + def is_master(args, local=False): + return is_local_master(args) if local else is_global_master(args) + + return is_master + + +class AllGather(torch.autograd.Function): + """An autograd function that performs allgather on a tensor. + Performs all_gather operation on the provided tensors. + *** Warning ***: torch.distributed.all_gather has no gradient. + """ + + @staticmethod + def forward(ctx, tensor, rank, world_size): + tensors_gather = [torch.empty_like(tensor) for _ in range(world_size)] + torch.distributed.all_gather(tensors_gather, tensor) + ctx.rank = rank + ctx.batch_size = tensor.shape[0] + return torch.cat(tensors_gather, 0) + + @staticmethod + def backward(ctx, grad_output): + return (grad_output[ctx.batch_size * ctx.rank : ctx.batch_size * (ctx.rank + 1)], None, None) + + +allgather = AllGather.apply