diff --git a/LaViDa/eval/lmms_eval/models/__init__.py b/LaViDa/eval/lmms_eval/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0a6a6f1c9feae092a6beec4eedeceb5bcddc4320 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/__init__.py @@ -0,0 +1,97 @@ +import importlib +import os +import sys + +import hf_transfer +from loguru import logger + +os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" + +logger.remove() +logger.add(sys.stdout, level="WARNING") + +AVAILABLE_MODELS = { + "aria": "Aria", + "auroracap": "AuroraCap", + "batch_gpt4": "BatchGPT4", + "claude": "Claude", + "cogvlm2": "CogVLM2", + "from_log": "FromLog", + "fuyu": "Fuyu", + "gemini_api": "GeminiAPI", + "gpt4v": "GPT4V", + "idefics2": "Idefics2", + "instructblip": "InstructBLIP", + "internvideo2": "InternVideo2", + "internvl": "InternVLChat", + "internvl2": "InternVL2", + "llama_vid": "LLaMAVid", + "llama_vision": "LlamaVision", + "llava": "Llava", + "llava_hf": "LlavaHf", + "llava_onevision": "Llava_OneVision", + "llava_onevision_moviechat": "Llava_OneVision_MovieChat", + "llava_sglang": "LlavaSglang", + "llava_vid": "LlavaVid", + "longva": "LongVA", + "mantis": "Mantis", + "minicpm_v": "MiniCPM_V", + "minimonkey": "MiniMonkey", + "moviechat": "MovieChat", + "mplug_owl_video": "mplug_Owl", + "ola": "Ola", + "openai_compatible": "OpenAICompatible", + "oryx": "Oryx", + "phi3v": "Phi3v", + "qwen2_5_vl": "Qwen2_5_VL", + "qwen2_5_vl_interleave": "Qwen2_5_VL_Interleave", + "qwen2_audio": "Qwen2_Audio", + "qwen2_vl": "Qwen2_VL", + "qwen_vl": "Qwen_VL", + "qwen_vl_api": "Qwen_VL_API", + "reka": "Reka", + "ross": "Ross", + "slime": "Slime", + "srt_api": "SRT_API", + "tinyllava": "TinyLlava", + "videoChatGPT": "VideoChatGPT", + "videochat2": "VideoChat2", + "video_llava": "VideoLLaVA", + "vila": "VILA", + "vita": "VITA", + "vllm": "VLLM", + "xcomposer2_4KHD": "XComposer2_4KHD", + "xcomposer2d5": "XComposer2D5", + "egogpt": "EgoGPT", + "internvideo2_5": "InternVideo2_5", + "videochat_flash": "VideoChat_Flash", + "whisper": "Whisper", + "whisper_vllm": "WhisperVllm", + "llava_llada":"Llava_Llada", + "llava_dream":"Llava_Dream" +} + + +def get_model(model_name): + if model_name not in AVAILABLE_MODELS: + raise ValueError(f"Model {model_name} not found in available models.") + + model_class = AVAILABLE_MODELS[model_name] + if "." not in model_class: + model_class = f"lmms_eval.models.{model_name}.{model_class}" + + try: + model_module, model_class = model_class.rsplit(".", 1) + module = __import__(model_module, fromlist=[model_class]) + return getattr(module, model_class) + except Exception as e: + logger.error(f"Failed to import {model_class} from {model_name}: {e}") + raise + + +if os.environ.get("LMMS_EVAL_PLUGINS", None): + # Allow specifying other packages to import models from + for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","): + m = importlib.import_module(f"{plugin}.models") + for model_name, model_class in getattr(m, "AVAILABLE_MODELS").items(): + AVAILABLE_MODELS[model_name] = f"{plugin}.models.{model_name}.{model_class}" diff --git a/LaViDa/eval/lmms_eval/models/aria.py b/LaViDa/eval/lmms_eval/models/aria.py new file mode 100644 index 0000000000000000000000000000000000000000..88c6bb076e4d84f61dd332ff194f16d1aed8043a --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/aria.py @@ -0,0 +1,334 @@ +import warnings +from typing import List, Optional, Tuple, Union + +import numpy as np +import PIL +import requests +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState +from decord import VideoReader, cpu +from PIL import Image +from tqdm import tqdm +from transformers import AutoModelForCausalLM, AutoProcessor + +from lmms_eval import utils +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + +warnings.filterwarnings("ignore") + +from loguru import logger as eval_logger + +DEFAULT_IMAGE_TOKEN = "" + + +@register_model("aria") +class Aria(lmms): + """ + Llava Model for Hugging Face Transformers: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava + + Adapted from the LLaVA-HF model in lmms_eval/models/llava_hf.py + + Example usage: + + accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \ + --model aria \ + --model_args pretrained=rhymes-ai/Aria \ + --tasks seedbench \ + --batch_size 1 \ + --output_path ./logs/ \ + --log_samples + """ + + def __init__( + self, + pretrained: str = "rhymes-ai/Aria", + revision: str = "main", + device: str = "cuda", + dtype: Optional[Union[str, torch.dtype]] = "auto", + batch_size: int = 1, + attn_implementation: Optional[str] = None, + device_map: str = "", + chat_template: Optional[str] = None, + use_cache: bool = True, + specified_eot_token_id: Optional[int] = None, + max_frames_num: Optional[int] = 64, + **kwargs, + ) -> None: + super().__init__() + # Do not use kwargs for now + assert kwargs == {}, f"Unexpected kwargs: {kwargs}" + + accelerator = Accelerator() + if accelerator.num_processes > 1 and device_map == "": + self._device = torch.device(f"cuda:{accelerator.local_process_index}") + self.device_map = f"cuda:{accelerator.local_process_index}" + else: + self._device = torch.device(device) + self.device_map = device_map + if isinstance(dtype, str) and dtype != "auto": + dtype = getattr(torch, dtype) + + self.max_frames_num = max_frames_num + self._model = AutoModelForCausalLM.from_pretrained(pretrained, revision=revision, device_map=self.device_map, torch_dtype=torch.bfloat16, trust_remote_code=True, attn_implementation=attn_implementation) + + self.pretrained = pretrained + self._image_processor = AutoProcessor.from_pretrained(pretrained, revision=revision, trust_remote_code=True) + self._tokenizer = self._image_processor.tokenizer + + self._config = self._model.config + self.batch_size_per_gpu = int(batch_size) + self.chat_template = chat_template + self.specified_eot_token_id = specified_eot_token_id + if accelerator.num_processes > 1 and device_map == "": + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model + # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works + # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work. + if accelerator.distributed_type == DistributedType.DEEPSPEED: + kwargs = { + "train_micro_batch_size_per_gpu": self.batch_size_per_gpu, + "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes, + } + AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs) + eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0") + if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED: + self._model = accelerator.prepare(self.model) + else: + self._model = accelerator.prepare_model(self.model, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + elif accelerator.num_processes == 1 and device_map == "auto": + eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism") + self._rank = 0 + self._world_size = 1 + else: + eval_logger.info(f"Using single device: {self._device}") + self.model.to(self._device) + self._rank = 0 + self._world_size = 1 + self.accelerator = accelerator + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def tokenizer(self): + return self._tokenizer + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def max_length(self): + return self._max_length + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]: + """ """ + add_special_tokens = False if add_special_tokens is None else add_special_tokens + encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens) + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + return encoding + + def tok_decode(self, tokens): + return self.tokenizer.decode(tokens) + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + raise NotImplementedError("Not implemented for Aria.") + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def load_video(self, video_path, max_frames_num): + if type(video_path) == str: + vr = VideoReader(video_path, ctx=cpu(0)) + else: + vr = VideoReader(video_path[0], ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + spare_frames = vr.get_batch(frame_idx).asnumpy() + spare_frames = [Image.fromarray(x) for x in spare_frames] + return spare_frames # (frames, height, width, channels) + + def generate_until(self, requests: List[Instance]) -> List[str]: + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1 + pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding") + for chunk in chunks: + contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk) + task = task[0] + split = split[0] + visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] + visuals = self.flatten(visuals) + if len(visuals) == 0: + task_type = "text" + elif isinstance(visuals[0], PIL.Image.Image): + task_type = "image" + elif isinstance(visuals[0], str): + task_type = "video" + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + + assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now" + text_context = contexts[0] + text_context = text_context.replace("\n\n", "\n") + + context = [] + + if task_type == "video": + try: + visuals = self.load_video(visuals, self.max_frames_num) + except Exception as e: + res.append("") + eval_logger.info(f"Error {e} when loading video : {visuals}") + pbar.update(1) + + if DEFAULT_IMAGE_TOKEN not in context: + context += [{"text": None, "type": "image"}] * len(visuals) + context += [{"text": "\n" + text_context, "type": "text"}] + else: + assert text_context.count(DEFAULT_IMAGE_TOKEN) == len(visuals) + for i, text_chunk in enumerate(text_context.split(DEFAULT_IMAGE_TOKEN)): + context += [{"text": text_chunk, "type": "text"}] + if i < len(visuals): + context += [{"text": None, "type": "image"}] * len(visuals) + context += [{"text": "\n", "type": "text"}] + + # Apply chat template + messages = [{"role": "user", "content": context}] + + text = self._image_processor.apply_chat_template(messages, add_generation_prompt=True) + + # removing redundant placeholders + text = re.sub(r"", "", text) + text = re.sub(r"", "", text) + + eval_logger.debug("DEBUGGING FOR ARIA:" + text) + + if self.accelerator.is_main_process and doc_id[0] % 100 == 0: + eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n") + + if task_type == "video": + inputs = self._image_processor(images=visuals, text=text, return_tensors="pt", max_image_size=490) + else: + inputs = self._image_processor(images=visuals, text=text, return_tensors="pt", max_image_size=980) + + inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype) + inputs = {k: v.to(self._device) for k, v in inputs.items()} + + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + if "top_p" not in gen_kwargs: + gen_kwargs["top_p"] = None + if "num_beams" not in gen_kwargs: + gen_kwargs["num_beams"] = 1 + + gen_kwargs["do_sample"] = False + gen_kwargs["max_new_tokens"] = 1024 + + if "until" in gen_kwargs: + gen_kwargs.pop("until") + + eval_logger.debug(f"generate kwargs: {gen_kwargs}") + + try: + with torch.inference_mode(), torch.cuda.amp.autocast(dtype=torch.bfloat16): + output = self.model.generate( + **inputs, + stop_strings=["<|im_end|>"], + tokenizer=self._image_processor.tokenizer, + **gen_kwargs, + ) + output_ids = output[0][inputs["input_ids"].shape[1] :] + text_outputs = self._image_processor.decode(output_ids, skip_special_tokens=True).replace("<|im_end|>", "") + + ### Basic Model-wise Parsing for CoT-alike Outputs + """ + keywords = [ + "Answer:", + "answer is:", "choice is:", "option is:", + "Answer is:", "Choice is:", "Option is:", + "answer is", "choice is", "option is", + "Answer is", "Choice is", "Option is" + ] + + for keyword in keywords: + if keyword in text_outputs: + eval_logger.debug(f"ARIA Original generated output simplified by keyword [{keyword}]: {text_outputs}") + text_outputs = text_outputs.split(keyword, 1)[-1] + break + """ + eval_logger.debug(f"Generated output: {text_outputs}") + except Exception as ex: + eval_logger.debug(f"Generation failed: {ex}") + if self.accelerator.is_main_process and doc_id[0] % 100 == 0: + eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n") + + res.append(text_outputs) + self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF") diff --git a/LaViDa/eval/lmms_eval/models/auroracap.py b/LaViDa/eval/lmms_eval/models/auroracap.py new file mode 100644 index 0000000000000000000000000000000000000000..f9ec9e69745187f2ec88123972fe583b2f3e49f4 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/auroracap.py @@ -0,0 +1,541 @@ +import copy +import json +import logging +import os +import os.path as osp +from typing import List, Optional, Tuple, Union + +import av +import numpy as np +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState +from huggingface_hub import snapshot_download +from peft import PeftModel +from PIL import Image +from tqdm import tqdm +from transformers import ( + AutoModel, + AutoModelForCausalLM, + AutoTokenizer, + BitsAndBytesConfig, + CLIPImageProcessor, +) + +from lmms_eval import utils +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model +from lmms_eval.models.model_utils.load_video import read_video_pyav +from lmms_eval.utils import stop_sequences_criteria + +try: + from lmms_eval.models.aurora_xtuner.model.aurora import ( + AuroraEncoder, + AuroraModel, + AuroraSigEncoder, + ) + from lmms_eval.models.aurora_xtuner.utils import PROMPT_TEMPLATE +except ImportError: + eval_logger.error("AuroraCap is not installed. Please install AuroraCap to use this model by `git clone https://github.com/rese1f/aurora.git` and link `src/xtuner/xtuner` to `lmms_eval/models/aurora_xtuner`") +import warnings + +warnings.filterwarnings("ignore") + +eval_logger = logging.getLogger("lmms-eval") + +try: + from llava.constants import ( + DEFAULT_IM_END_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IMAGE_TOKEN, + IGNORE_INDEX, + IMAGE_TOKEN_INDEX, + ) + from llava.conversation import SeparatorStyle, conv_templates + from llava.mm_utils import get_model_name_from_path, tokenizer_image_token +except ImportError: + eval_logger.error("LLaVA is not installed. Please install LLaVA to use this model.") + + +@register_model("auroracap") +class AuroraCap(lmms): + """ + auroracap Model + """ + + def __init__( + self, + pretrained_llm: str = "meta-llama/Meta-Llama-3-8B-Instruct", + pretrained_vit: str = "google/siglip-so400m-patch14-384", + pretrained: str = "model/PATH", + resolution: int = 378, + token_merge_ratio: float = 0.4, + device: Optional[str] = "cuda", + dtype: Optional[Union[str, torch.dtype]] = "auto", + batch_size: Optional[Union[int, str]] = 1, + conv_template="vicuna_v1", # vicuna_v1", + video_decode_backend: str = "pyav", + max_frames_num: int = 16, + slowfast: bool = False, + **kwargs, + ) -> None: + super().__init__() + # Do not use kwargs for now + assert kwargs == {}, f"Unexpected kwargs: {kwargs}" + + accelerator = Accelerator() + if accelerator.num_processes > 1: + self._device = torch.device(f"cuda:{accelerator.local_process_index}") + else: + self._device = device + + pretrained_pth = snapshot_download(repo_id=pretrained) if not osp.isdir(pretrained) else pretrained + pretrained_llm = pretrained_pth + pretrained_vit = osp.join(pretrained_pth, "visual_encoder") + + self._model = AuroraModel( + slowfast=slowfast, + llm=AutoModelForCausalLM.from_pretrained( + pretrained_model_name_or_path=pretrained_llm, + trust_remote_code=True, + torch_dtype=torch.float16, + ), + visual_encoder=AuroraEncoder.from_pretrained( + pretrained_model_name_or_path=pretrained_vit, + torch_dtype=torch.float16, + ), + ) + + projector_path = osp.join(pretrained_pth, "projector") + self.model.projector = AutoModel.from_pretrained(projector_path, torch_dtype=torch.float16, trust_remote_code=True) + + self._image_processor = CLIPImageProcessor.from_pretrained( + pretrained_model_name_or_path="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", # use standard CLIP processor + trust_remote_code=True, + size=resolution, + crop_size=resolution, + ) + self._tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=pretrained_llm, + trust_remote_code=True, + padding_side="right", + ) + # compute token merge ratio settings + self.patch_size = self._model.visual_encoder.config.patch_size + self.num_layers = self._model.visual_encoder.config.num_hidden_layers + self.token_merge_ratio = token_merge_ratio + + self._config = self._model.config + self.model.eval() + self.model.tie_weights() + self.batch_size_per_gpu = int(batch_size) + self.conv_template = conv_template + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model + # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works + # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work. + if accelerator.distributed_type == DistributedType.DEEPSPEED: + kwargs = { + "train_micro_batch_size_per_gpu": self.batch_size_per_gpu, + "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes, + } + AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs) + eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0") + if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED: + self._model = accelerator.prepare(self.model) + self._model.visual_encoder = accelerator.prepare(self.model.visual_encoder) + self._model.projector = accelerator.prepare(self.model.projector) + else: # DistributedType.MULTI_GPU + self._model = accelerator.prepare_model(self.model, evaluation_mode=True) + self._model.visual_encoder = accelerator.prepare_model(self.model.visual_encoder, evaluation_mode=True) + self._model.projector = accelerator.prepare_model(self.model.projector, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.model.to(self._device) + self._rank = 0 + self._world_size = 1 + + # For Video Caption + self.video_decode_backend = video_decode_backend + self.max_frames_num = int(max_frames_num) + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def tokenizer(self): + return self._tokenizer + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def max_length(self): + return self._max_length + + def pad_sequence(self, input_ids, batch_first, padding_value): + if self.tokenizer.padding_side == "left": + input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids] + input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value) + if self.tokenizer.padding_side == "left": + input_ids = torch.flip(input_ids, [1]) + return input_ids + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + def process_images(self, images, image_processor, model_cfg): + image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None) + new_images = [] + if image_aspect_ratio == "pad": + for image in images: + image = expand2square(image, tuple(int(x * 255) for x in image_processor.image_mean)) + image = image_processor.preprocess(image, return_tensors="pt")["pixel_values"][0] + new_images.append(image) + elif image_aspect_ratio == "anyres": + for image in images: + image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints) + new_images.append(image) + else: + return image_processor(images, return_tensors="pt")["pixel_values"] + if all(x.shape == new_images[0].shape for x in new_images): + new_images = torch.stack(new_images, dim=0) + return new_images + + def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]: + """ """ + add_special_tokens = False if add_special_tokens is None else add_special_tokens + encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens) + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + return encoding + + def tok_decode(self, tokens): + return self.tokenizer.decode(tokens) + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + # encode, pad, and truncate contexts for this batch + if type(doc_to_target) == str: + continuation = doc_to_target + else: + continuation = doc_to_target(self.task_dict[task][split][doc_id]) + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + if visuals: + image = self.process_images(visuals, self._image_processor, self._config) + if type(image) is list: + image = [_image.to(dtype=torch.float16, device=self.device) for _image in image] + else: + image = image.to(dtype=torch.float16, device=self.device) + else: + image = None + + prompts_input = contexts[0] + + if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input: + """ + Three senarios: + 1. No image, and there for, no image token should be added. + 2. image token is already specified in the context, so we don't need to add it. + 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line. + """ + image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals) + image_tokens = " ".join(image_tokens) + prompts_input = image_tokens + "\n" + contexts[0] + conv = conv_templates[self.conv_template].copy() + conv.append_message(conv.roles[0], prompts_input) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id + contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device) + # Add the answer of the second role + conv.messages[1][1] = continuation + + prompt = conv.get_prompt() + input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device) + labels = input_ids.clone() + # Context part no need to calculate for loss + labels[0, : contxt_id.shape[1]] = -100 + with torch.inference_mode(): + data = dict() + data["pixel_values"] = image_tensor + data["input_ids"] = input_ids + data["attention_mask"] = attention_masks + self.model.visual_encoder.reset_tome_r(self.token_merge_ratio) + output = self.model(data, mode="tensor") + + loss = outputs["loss"] + # loss = torch.exp(loss) + logits = outputs["logits"] + greedy_tokens = logits.argmax(dim=-1) + cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq] + greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq] + max_equal = (greedy_tokens == cont_toks).all() + res.append((float(loss.item()), bool(max_equal))) + pbar.update(1) + pbar.close() + return res + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def load_video(self, video_path, max_frames_num): + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + spare_frames = vr.get_batch(frame_idx).asnumpy() + return spare_frames # (frames, height, width, channels) + + def extract_keyframes(self, video_path, keyframes): + container = av.open(video_path) + video_stream = container.streams.video[0] + fps = video_stream.average_rate + time_base = video_stream.time_base + frames = [] + + for keyframe in keyframes: + keyframe_time = float(keyframe) + frame_number = int(keyframe_time * fps) + container.seek(int(keyframe_time / time_base)) + found = False + for packet in container.demux(video=0): + for frame in packet.decode(): + if frame.index >= frame_number: + frames.append(frame) + found = True + break + if found: + break + + if not found: + container.seek(-1, any_frame=False) + for packet in container.demux(video=0): + for frame in packet.decode(): + pass + frames.append(frame) + + return np.stack([x.to_ndarray(format="rgb24") for x in frames]) + + def generate_until(self, requests: List[Instance]) -> List[str]: + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1 + pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding") + + for chunk in chunks: + contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk) + task = task[0] + split = split[0] + visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # the length of visuals is 1, equal to batchsize + visuals = self.flatten(visuals) + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + + # Set default values for until and max_new_tokens + until = [self.tok_decode(self.eot_token_id)] + + # Update values from gen_kwargs if present + if "until" in gen_kwargs: + until = gen_kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}") + + if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__: + # here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation + self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio") + eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") + # encode, pad, and truncate contexts for this batch + if visuals: + if isinstance(visuals[0], dict): + video_path = visuals[0]["video_path"] + keyframe = visuals[0]["keyframe"] + video = self.extract_keyframes(video_path, keyframe) + image_tensor = self.process_images(video, self._image_processor, self._config).cuda() + elif isinstance(visuals, list): + print(visuals[0]) + if isinstance(visuals[0], Image.Image): + image_tensor = self.process_images(visuals, self._image_processor, self._config) + else: + if visuals[0].endswith("mp4"): + if self.video_decode_backend == "decord": + video = self.load_video(visuals[0], self.max_frames_num) + elif self.video_decode_backend == "pyav": + video = read_video_pyav(visuals[0], num_frm=self.max_frames_num) + image_tensor = self.process_images(video, self._image_processor, self._config).cuda() + elif visuals[0].endswith("mkv"): + assert self.video_decode_backend == "pyav", "we only tested this case, decord may not work" + video = read_video_pyav(visuals[0], num_frm=self.max_frames_num) + image_tensor = self.process_images(video, self._image_processor, self._config).cuda() + + if type(image_tensor) is list: + image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor] + else: + image_tensor = image_tensor.to(dtype=torch.float16, device=self.device) + + else: + image_tensor = None + + question_input = [] + + for visual, context in zip(visuals, contexts): + if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context: + """ + Three senarios: + 1. No image, and there for, no image token should be added. + 2. image token is already specified in the context, so we don't need to add it. + 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line. + """ + if isinstance(visuals[0], dict): + image_tokens = [DEFAULT_IMAGE_TOKEN] * len(video) + elif isinstance(visuals, list): + if isinstance(visuals[0], Image.Image): + image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN] + else: + if visual.endswith("mp4") or visual.endswith("mkv"): + image_tokens = [DEFAULT_IMAGE_TOKEN] * len(video) + + image_tokens = " ".join(image_tokens) + question = image_tokens + "\n" + context + + else: + question = context + conv = conv_templates[self.conv_template].copy() + conv.append_message(conv.roles[0], question) + conv.append_message(conv.roles[1], None) + prompt_question = conv.get_prompt() + question_input.append(prompt_question) + + # The above for loop has bugs. When there is no visuals, e.g. pure text, + # there will be no for loop execute resulting in an empty question_input (because no visuals) + # Scenario 1 won't even be execute + if len(visuals) == 0: + for context in contexts: + question = context + conv = conv_templates[self.conv_template].copy() + conv.append_message(conv.roles[0], question) + conv.append_message(conv.roles[1], None) + prompt_question = conv.get_prompt() + question_input.append(prompt_question) + + # preconfigure gen_kwargs with defaults + if isinstance(visuals[0], dict): + gen_kwargs["image_sizes"] = [video[idx].size for idx in range(len(video))] + elif isinstance(visuals, list): + if isinstance(visuals[0], Image.Image): + gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))] + else: + if visuals[0].endswith("mp4"): + gen_kwargs["image_sizes"] = [video[idx].size for idx in range(len(video))] + + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 1024 + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + if "top_p" not in gen_kwargs: + gen_kwargs["top_p"] = None + if "num_beams" not in gen_kwargs: + gen_kwargs["num_beams"] = 1 + + input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input] + pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id + input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device) + attention_masks = input_ids.ne(pad_token_ids).to(self.device) + # These steps are not in LLaVA's original code, but are necessary for generation to work + try: + data = dict() + if isinstance(visuals[0], dict): + data["pixel_values"] = image_tensor.unsqueeze(0) + elif isinstance(visuals, list): + if isinstance(visuals[0], Image.Image): + data["pixel_values"] = image_tensor + else: + if visuals[0].endswith("mp4") or visuals[0].endswith("mkv"): + data["pixel_values"] = image_tensor.unsqueeze(0) + + data["input_ids"] = input_ids + data["attention_mask"] = attention_masks + self.model.visual_encoder.reset_tome_r(self.token_merge_ratio) + output = self.model(data, mode="inference") + cont = self.model.llm.generate( + **output, + do_sample=True if gen_kwargs["temperature"] > 0 else False, + temperature=gen_kwargs["temperature"], + top_p=gen_kwargs["top_p"], + num_beams=gen_kwargs["num_beams"], + max_new_tokens=gen_kwargs["max_new_tokens"], + ) + text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True) + except Exception as e: + eval_logger.error(f"Error {e} in generating") + cont = "" + text_outputs = [""] + + print(text_outputs) + + res.extend(text_outputs) + self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res diff --git a/LaViDa/eval/lmms_eval/models/batch_gpt4.py b/LaViDa/eval/lmms_eval/models/batch_gpt4.py new file mode 100644 index 0000000000000000000000000000000000000000..e624310adcfa6ad4c455e53db1e4b50cb1d1dd15 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/batch_gpt4.py @@ -0,0 +1,207 @@ +# Standard library imports +import base64 +import json +import os +import time +from copy import deepcopy +from io import BytesIO + +import numpy as np +import requests as url_requests + +# Related third-party imports +from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs +from accelerate.state import AcceleratorState +from loguru import logger as eval_logger +from openai import OpenAI +from PIL import Image +from tqdm import tqdm + +# Local application/library specific imports +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + +# Conditional imports +try: + from decord import VideoReader, cpu +except ImportError: + eval_logger.warning("Decord is not installed. Video input will not be supported.") + +# Constants and global configurations +API_TYPE = os.getenv("API_TYPE", "openai") +NUM_SECONDS_TO_SLEEP = 5 + +if API_TYPE == "openai": + API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") + API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") + headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json", + } +elif API_TYPE == "azure": + API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") + API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") + headers = { + "api-key": API_KEY, + "Content-Type": "application/json", + } +else: + API_URL = "YOUR_API_URL" + API_KEY = "YOUR_API_KEY" + + +@register_model("batch_gpt4") +class BatchGPT4(lmms): + def __init__( + self, + model_version: str = "gpt-4o", + api_key: str = API_KEY, + api_url: str = API_URL, + modality: str = "image", + max_frames_num: int = 10, + timeout: int = 120, + **kwargs, + ) -> None: + super().__init__() + # Manually set a image token for GPT4V so that we can search for it + # and split the text and image + # Here we just use the same token as llava for convenient + self.model_version = model_version + self.modality = modality + self.max_frames_num = max_frames_num + self.image_token = "" + self.timeout = timeout + + self.api_key = api_key + self.api_url = api_url + self.client = OpenAI(api_key=api_key) + + accelerator = Accelerator() + assert accelerator.state.local_process_index == 0, "BatchGPT4 does not support distributed inference." + assert accelerator.state.num_processes == 1, "BatchGPT4 does not support distributed inference." + + # Function to encode the image + def encode_image(self, image: Image): + output_buffer = BytesIO() + image.save(output_buffer, format="PNG") + byte_data = output_buffer.getvalue() + base64_str = base64.b64encode(byte_data).decode("utf-8") + return base64_str + + # Function to encode the video + def encode_video(self, video_path, for_get_frames_num): + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + img = Image.fromarray(frame) + output_buffer = BytesIO() + img.save(output_buffer, format="PNG") + byte_data = output_buffer.getvalue() + base64_str = base64.b64encode(byte_data).decode("utf-8") + base64_frames.append(base64_str) + + return base64_frames + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def generate_until(self, requests): + # Prepare the batch requests data + requests_data = {} + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Batch Preparing") + for idx, (contexts, gen_kwargs, doc_to_visual, doc_id, task, split) in enumerate([reg.args for reg in requests]): + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + imgs = [] + for visual in visuals: + if self.modality == "image": + img = self.encode_image(visual) + imgs.append(img) + elif self.modality == "video": + frames = self.encode_video(visual, self.max_frames_num) + imgs.extend(frames) + + messages = [] + if self.image_token not in contexts: + messages.append({"role": "user", "content": contexts}) + for img in imgs: + messages.append({"role": "user", "content": f"data:image/jpeg;base64,{img}"}) + else: + contexts_split = contexts.split(self.image_token) + for idx, context in enumerate(contexts_split): + if idx < len(imgs): + messages.append({"role": "user", "content": context}) + messages.append({"role": "user", "content": f"data:image/jpeg;base64,{imgs[idx]}"}) + if len(contexts_split) > len(imgs): + messages.append({"role": "user", "content": contexts_split[-1]}) + + requests_data[f"request-{idx}"] = {"model": self.model_version, "messages": messages, "max_tokens": gen_kwargs.get("max_new_tokens", 1024)} + pbar.update(1) + + file_path = os.getenv("HF_HOME", "~/.cache/huggingface") + f"/batchinput_{len(requests_data)}.jsonl" + file_path = self.create_batch_input_file(requests_data, file_path) + file_id = self.upload_input_file(file_path) + + batch_response = self.create_batch(file_id, metadata={"description": "Batch Processing for GPT-4"}) + batch_status = self.check_batch_status(batch_response.id) + while True: + batch_status = self.check_batch_status(batch_response.id) + if batch_status.status == "completed": + eval_logger.info("Batch processing completed.") + batch_results = self.retrieve_batch_results(batch_status.output_file_id) + res = [result["response"]["choices"][0]["message"]["content"] for result in json.loads(batch_results)] + return res + elif batch_status.status == "failed": + eval_logger.info("Batch processing failed.") + res = ["Batch failed"] * len(requests) + return res + else: + eval_logger.info(f"Batch status: {batch_status.status}. Retrying in {NUM_SECONDS_TO_SLEEP} seconds.") + time.sleep(NUM_SECONDS_TO_SLEEP) + + def loglikelihood(self, requests): + # TODO + assert False, "GPT4V not support" + + def create_batch_input_file(self, requests_data, file_path="batchinput.jsonl"): + with open(file_path, "w") as file: + for request_id, data in requests_data.items(): + json_record = json.dumps({"custom_id": request_id, "method": "POST", "url": "/v1/chat/completions", "body": data}) + file.write(json_record + "\n") + return file_path + + def upload_input_file(self, file_path): + with open(file_path, "rb") as file: + response = self.client.files.create(file=file, purpose="batch") + return response.id + + def create_batch(self, file_id, metadata=None): + if metadata is None: + metadata = {} + response = self.client.batches.create(input_file_id=file_id, endpoint="/v1/chat/completions", completion_window="24h", metadata=metadata) + return response + + def check_batch_status(self, batch_id): + return self.client.batches.retrieve(batch_id) + + def retrieve_batch_results(self, file_id): + return self.client.files.content(file_id) + + def cancel_batch(self, batch_id): + return self.client.batches.cancel(batch_id) + + def list_batches(self, limit=10): + return self.client.batches.list(limit=limit) + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for BatchGPT4") diff --git a/LaViDa/eval/lmms_eval/models/cambrian.py b/LaViDa/eval/lmms_eval/models/cambrian.py new file mode 100644 index 0000000000000000000000000000000000000000..855b87f9e5aecf10ce748b299ee4851a72ca5c1b --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/cambrian.py @@ -0,0 +1,313 @@ +# code is migrated from https://github.com/kangreen0210/LIME-M/blob/main/lmms_eval/models/cambrian.py +import os +import uuid +import warnings +from typing import List, Optional, Tuple, Union + +import torch +from accelerate import Accelerator, DistributedType +from PIL import Image +from tqdm import tqdm +from transformers import PreTrainedTokenizer + +from lmms_eval import utils +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + +warnings.simplefilter("ignore", category=DeprecationWarning) +warnings.filterwarnings("ignore") + +from loguru import logger as eval_logger + +try: + from cambrian.conversation import conv_templates + from cambrian.mm_utils import ( + get_model_name_from_path, + process_images, + tokenizer_image_token, + ) + from cambrian.model.builder import load_pretrained_model +except ImportError: + eval_logger.error("Cambrian is not installed. Please install it by running `pip install cambrian`.") + +# Model Constants +IMAGE_TOKEN_INDEX = -200 +DEFAULT_IMAGE_TOKEN = "" +DEFAULT_IM_START_TOKEN = "" +DEFAULT_IM_END_TOKEN = "" + + +def process(image, question, tokenizer, image_processor, model_config, conv_mode): + qs = question + + if model_config.mm_use_im_start_end: + qs = f"{DEFAULT_IM_START_TOKEN}{DEFAULT_IMAGE_TOKEN}{DEFAULT_IM_END_TOKEN}\n{qs}" + else: + qs = f"{DEFAULT_IMAGE_TOKEN}\n{qs}" + + conv = conv_templates[conv_mode].copy() + conv.append_message(conv.roles[0], qs) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + image_size = [image.size] + image_tensor = process_images([image], image_processor, model_config) + + input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda() + + return input_ids, image_tensor, image_size, prompt + + +def make_context( + tokenizer: PreTrainedTokenizer, + query: str, + history: List[Tuple[str, str]] = None, + system: str = "", + max_window_size: int = 6144, + chat_format: str = "chatml", +): + if history is None: + history = [] + + if chat_format == "chatml": + im_start, im_end = "<|im_start|>", "<|im_end|>" + im_start_tokens = [tokenizer.im_start_id] + im_end_tokens = [tokenizer.im_end_id] + nl_tokens = tokenizer.encode("\n") + + def _tokenize_str(role, content): + return f"{role}\n{content}", tokenizer.encode(role, allowed_special=set(tokenizer.IMAGE_ST)) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST)) + + system_text, system_tokens_part = _tokenize_str("system", system) + system_tokens = im_start_tokens + system_tokens_part + im_end_tokens + + raw_text = "" + context_tokens = [] + + for turn_query, turn_response in reversed(history): + query_text, query_tokens_part = _tokenize_str("user", turn_query) + query_tokens = im_start_tokens + query_tokens_part + im_end_tokens + if turn_response is not None: + response_text, response_tokens_part = _tokenize_str("assistant", turn_response) + response_tokens = im_start_tokens + response_tokens_part + im_end_tokens + + next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens + prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}" + else: + next_context_tokens = nl_tokens + query_tokens + nl_tokens + prev_chat = f"\n{im_start}{query_text}{im_end}\n" + + current_context_size = len(system_tokens) + len(next_context_tokens) + len(context_tokens) + if current_context_size < max_window_size: + context_tokens = next_context_tokens + context_tokens + raw_text = prev_chat + raw_text + else: + break + + context_tokens = system_tokens + context_tokens + raw_text = f"{im_start}{system_text}{im_end}" + raw_text + context_tokens += nl_tokens + im_start_tokens + _tokenize_str("user", query)[1] + im_end_tokens + nl_tokens + im_start_tokens + tokenizer.encode("assistant") + nl_tokens + raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n" + + elif chat_format == "raw": + raw_text = query + context_tokens = tokenizer.encode(raw_text) + else: + raise NotImplementedError(f"Unknown chat format {chat_format!r}") + + return raw_text, context_tokens + + +@register_model("cambrian") +class Cambrian(lmms): + def __init__( + self, + pretrained: str = "nyu-visionx/cambrian-8b", + device: Optional[str] = "cuda", + device_map="auto", + batch_size: Optional[Union[int, str]] = 1, + trust_remote_code: Optional[bool] = True, + use_cache=True, + **kwargs, + ) -> None: + super().__init__() + assert not kwargs, f"Unexpected kwargs: {kwargs}" + + accelerator = Accelerator() + self._device = torch.device(f"cuda:{accelerator.local_process_index}") if accelerator.num_processes > 1 else device + + self.model_name = get_model_name_from_path(pretrained) + tokenizer, model, self.image_processor, context_len = load_pretrained_model(pretrained, None, self.model_name, device_map=self._device) + + self.conv_mode = {"cambrian-8b": "llama_3", "cambrian-13b": "vicuna_v1", "cambrian-34b": "chatml_direct"}.get(self.model_name) + + if not self.conv_mode: + raise ValueError(f"Unsupported model: {self.model_name}") + + self._model = model + self._tokenizer = tokenizer + self._model.eval() + self.batch_size_per_gpu = int(batch_size) + self.use_cache = use_cache + self._rank = 0 + self._world_size = 1 + + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU], "Unsupported distributed type. Only DDP and FSDP are supported." + self._model = accelerator.prepare(self.model) if accelerator.distributed_type == DistributedType.FSDP else accelerator.prepare_model(self.model, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.model.to(self._device) + + self.accelerator = accelerator + + @property + def model(self): + return self.accelerator.unwrap_model(self._model) if hasattr(self, "accelerator") else self._model + + @property + def eot_token_id(self): + return self.tokenizer.eos_token_id + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + continuation = doc_to_target if isinstance(doc_to_target, str) else doc_to_target(self.task_dict[task][split][doc_id]) + visuals = self.flatten([doc_to_visual(self.task_dict[task][split][doc_id])]) + + query = [] + visual_paths = [] + for visual in visuals: + name = uuid.uuid4().hex.upper()[0:6] + visual_path = f"/tmp/{name}.png" + visual.save(visual_path) + visual_paths.append(visual_path) + query.append({"image": visual_path}) + + context_query = query.copy() + context_query.append({"text": contexts}) + query.append({"text": contexts + continuation}) + + context_query = self.tokenizer.from_list_format(context_query) + query = self.tokenizer.from_list_format(query) + + _, context_tokens = make_context( + self.tokenizer, context_query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format + ) + context_tokens = torch.tensor([context_tokens]) + + _, continuation_tokens = make_context(self.tokenizer, query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format) + continuation_tokens = torch.tensor([continuation_tokens]).to(self.model.device) + attn_mask = torch.ones_like(continuation_tokens).to(self.model.device) + labels = continuation_tokens.clone().to(self.model.device) + labels[:, : context_tokens.shape[1]] = -100 + + with torch.inference_mode(): + outputs = self.model(input_ids=continuation_tokens, labels=labels, attention_mask=attn_mask) + + loss = outputs.loss + logits = outputs["logits"] + greedy_tokens = logits.argmax(dim=-1) + cont_toks = continuation_tokens[:, context_tokens.shape[1] :] + greedy_tokens = greedy_tokens[:, context_tokens.shape[1] : continuation_tokens.shape[1]] + max_equal = (greedy_tokens == cont_toks).all() + res.append((float(loss.item()), bool(max_equal))) + pbar.update(1) + + pbar.close() + return res + + @staticmethod + def flatten(input_list): + return [item for sublist in input_list for item in sublist] + + def generate_until(self, requests: List[Instance]) -> List[str]: + res = [] + + def _collate(x): + toks = self.tokenizer.encode(x[0]) + return -len(toks), x[0] + + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + + for chunk in chunks: + contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk) + task = task[0] + split = split[0] + visuals = self.flatten([doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]) + + visual_paths = [] + for visual in visuals: + name = uuid.uuid4().hex.upper()[0:6] + visual_path = f"/xpfs/public/gezhang/zk/lmms-eval/lmms_eval/tmp/{name}.png" + visual.save(visual_path) + visual_paths.append(visual_path) + + gen_kwargs = all_gen_kwargs[0] + until = [self.tokenizer.decode(self.eot_token_id)] + + if "until" in gen_kwargs: + until = gen_kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}") + + gen_kwargs.setdefault("image_sizes", [visuals[0].size] if visuals else None) + gen_kwargs.setdefault("max_new_tokens", 1024) + gen_kwargs.setdefault("temperature", 0) + gen_kwargs.setdefault("top_p", None) + gen_kwargs.setdefault("num_beams", 1) + + until.append("<|eot_id|>") + + image = Image.open(visual_paths[0]).convert("RGB") + question = contexts[0] + + input_ids, image_tensor, image_sizes, prompt = process(image, question, self.tokenizer, self.image_processor, self.model.config, self.conv_mode) + input_ids = input_ids.to(device=self.model.device, non_blocking=True) + + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids, + images=image_tensor, + image_sizes=image_sizes, + do_sample=gen_kwargs["temperature"] > 0, + temperature=gen_kwargs["temperature"], + num_beams=gen_kwargs["num_beams"], + max_new_tokens=gen_kwargs["max_new_tokens"], + use_cache=True, + ) + + text_outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() + + for term in until: + if term: + text_outputs = text_outputs.split(term)[0] + + print(text_outputs) + res.append(text_outputs) + + for visual_path in visual_paths: + try: + os.remove(visual_path) + except OSError: + pass + + pbar.update(1) + + res = re_ords.get_original(res) + pbar.close() + return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Cambrian") diff --git a/LaViDa/eval/lmms_eval/models/claude.py b/LaViDa/eval/lmms_eval/models/claude.py new file mode 100644 index 0000000000000000000000000000000000000000..cc6af6303b0f1a7d3e22a26eff7c14e2336baeeb --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/claude.py @@ -0,0 +1,272 @@ +import base64 +import json +import os +import time +from copy import deepcopy +from io import BytesIO +from typing import List, Tuple, Union + +from accelerate import Accelerator, DistributedType +from PIL import Image +from tqdm import tqdm + +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + +NUM_SECONDS_TO_SLEEP = 5 + +from loguru import logger + +eval_logger = logger + +try: + import anthropic + import numpy as np + from decord import VideoReader, cpu +except Exception as e: + eval_logger.warning(f"Error importing claude: {e}") + +API_URL = os.getenv("ANTHROPIC_API_URL", "https://api.anthropic.com/v1/complete") +API_KEY = os.getenv("ANTHROPIC_API_KEY", "YOUR_API_KEY") + + +@register_model("claude") +class Claude(lmms): + def __init__( + self, + model_version: str = "claude-3-opus-20240229", + image_token: str = "", # Use to separate interleaved image and text + system_prompt: str = "", # Whether you want some special system prompt here + modality: str = "image", + max_frames_num: int = 10, + continual_mode: bool = False, + response_persistent_folder: str = None, + **kwargs, + ) -> None: + super().__init__() + self.model_version = model_version + self.image_token = image_token + self.system_prompt = system_prompt + self.modality = modality + self.max_frames_num = max_frames_num + + self.continual_mode = continual_mode + if self.continual_mode: + if response_persistent_folder is None: + raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.") + + os.makedirs(response_persistent_folder, exist_ok=True) + self.response_persistent_folder = response_persistent_folder + self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json") + + if os.path.exists(self.response_persistent_file): + with open(self.response_persistent_file, "r") as f: + self.response_cache = json.load(f) + self.cache_mode = "resume" + else: + self.response_cache = {} + self.cache_mode = "start" + + accelerator = Accelerator() + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.accelerator = accelerator + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + + self.device = self.accelerator.device + + def encode_image(self, image): + output_buffer = BytesIO() + image.save(output_buffer, format="JPEG") + byte_data = output_buffer.getvalue() + base64_str = base64.b64encode(byte_data).decode("utf-8") + return base64_str + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def get_image_size(self, image): + # Create a BytesIO object to store the image bytes + img_byte_array = BytesIO() + + # Save the image to the BytesIO object + image.save(img_byte_array, format="PNG") + + # Get the size of the BytesIO object + img_size = img_byte_array.tell() + + return img_size + + # The max file size is 5MB for claude + def shrink_image_to_file_size(self, img: Image, max_file_size=4838990) -> Image: + # Get the current size of the image + original_size = self.get_image_size(img) + + # If the image size is already smaller than the desired size, return + if original_size <= max_file_size: + return img + + # Calculate the ratio to shrink the image + # Somehow I found out sqrt ratio is not enough to shrink the image + # below threshold, so I guess we do more + shrink_ratio = min(0.9, max_file_size / original_size) + + # Resize the image with the calculated ratio + new_width = int(img.width * shrink_ratio) + new_height = int(img.height * shrink_ratio) + img = img.resize((new_width, new_height), Image.LANCZOS) + + return self.shrink_image_to_file_size(img, max_file_size) + + def encode_video(self, video_path): + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, self.max_frames_num, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + img = Image.fromarray(frame) + output_buffer = BytesIO() + img.save(output_buffer, format="JPEG") + byte_data = output_buffer.getvalue() + base64_str = base64.b64encode(byte_data).decode("utf-8") + base64_frames.append(f"{base64_str}") + + return base64_frames + + def generate_until(self, requests) -> List[str]: + client = anthropic.Anthropic() + + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + empty_image_block = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + }, + } + empty_text_block = {"type": "text"} + empty_messages = [ + { + "role": "user", + "content": [], + } + ] + + for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + ###################### CONTINUAL MODE ###################### + if self.continual_mode is True and self.cache_mode == "resume": + doc_uuid = f"{task}___{split}___{doc_id}" + if doc_uuid in self.response_cache: + response_text = self.response_cache[doc_uuid] + if response_text: + res.append(response_text) + pbar.update(1) + continue + + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + imgs = [] + for visual in visuals: + if isinstance(visual, str) and os.path.exists(visual): # Assuming visual is a path to a video + visual = self.encode_video(visual) + for img in visual: + imgs.append(img) + else: + visual = self.shrink_image_to_file_size(visual) + img = self.encode_image(visual) + imgs.append(img) + + messages = deepcopy(empty_messages) + + if self.image_token not in contexts: + for img in imgs: + image_block = deepcopy(empty_image_block) + image_block["source"]["data"] = img + messages[0]["content"].append(image_block) + text_block = deepcopy(empty_text_block) + text_block["text"] = contexts + messages[0]["content"].append(text_block) + else: + contexts = contexts.split(self.image_token) + for idx, img in enumerate(imgs): + text_block = deepcopy(empty_text_block) + image_block = deepcopy(empty_image_block) + text_block["text"] = contexts + messages[0]["content"].append(text_block) + image_block["source"]["data"] = img + messages[0]["content"].append(image_block) + + # If n image tokens are in the contexts + # contexts will be splitted into n+1 chunks + # Manually add it into the messages + text_block = deepcopy(empty_text_block) + text_block["text"] = contexts + messages["content"].append(text_block) + + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 1024 + if gen_kwargs["max_new_tokens"] > 4096: + gen_kwargs["max_new_tokens"] = 4096 + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + if "top_p" not in gen_kwargs or gen_kwargs["top_p"] is None: + gen_kwargs["top_p"] = 1 + if "num_beams" not in gen_kwargs: + gen_kwargs["num_beams"] = 1 + + for attempt in range(5): + retry_flag = True + try: + message = client.messages.create(model=self.model_version, max_tokens=gen_kwargs["max_new_tokens"], system=self.system_prompt, temperature=gen_kwargs["temperature"], top_p=gen_kwargs["top_p"], messages=messages) + retry_flag = False + except Exception as e: + eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}") + if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt + time.sleep(NUM_SECONDS_TO_SLEEP) + else: # If this was the last attempt, log and return empty + eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}") + res.append("") + pbar.update(1) + continue + if not retry_flag: + break + eval_logger.info("Retrying...") + + response_text = message.content[0].text + res.append(message.content[0].text) + pbar.update(1) + + ###################### CONTINUAL MODE ###################### + if self.continual_mode is True: # Cache the response + response_text = message.content[0].text + doc_uuid = f"{task}___{split}___{doc_id}" + self.response_cache[doc_uuid] = response_text + with open(self.response_persistent_file, "w") as f: + json.dump(self.response_cache, f, indent=4) + + pbar.close() + + return res + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + assert False, "Not supported for claude" + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Claude") diff --git a/LaViDa/eval/lmms_eval/models/cogvlm2.py b/LaViDa/eval/lmms_eval/models/cogvlm2.py new file mode 100644 index 0000000000000000000000000000000000000000..b231e2f8f29baacc4698671d1f7bd1e8660d9a20 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/cogvlm2.py @@ -0,0 +1,229 @@ +import warnings +from typing import List, Optional, Tuple, Union + +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState +from tqdm import tqdm +from transformers import AutoModelForCausalLM, AutoTokenizer + +from lmms_eval import utils +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + +warnings.filterwarnings("ignore") + +from loguru import logger as eval_logger + + +@register_model("cogvlm2") +class CogVLM2(lmms): + """ + CogVLM2 Model + """ + + def __init__( + self, + pretrained: str = "THUDM/cogvlm2-llama3-chinese-chat-19B", + device: Optional[str] = "cuda", + dtype: Optional[Union[str, torch.dtype]] = torch.bfloat16, + batch_size: Optional[Union[int, str]] = 1, + trust_remote_code: Optional[bool] = True, + **kwargs, + ) -> None: + super().__init__() + # Do not use kwargs for now + assert kwargs == {}, f"Unexpected kwargs: {kwargs}" + + accelerator = Accelerator() + if accelerator.num_processes > 1: + self._device = torch.device(f"cuda:{accelerator.local_process_index}") + else: + self._device = device + self.dtype = dtype + self._model = AutoModelForCausalLM.from_pretrained(pretrained, trust_remote_code=trust_remote_code, torch_dtype=dtype, device_map=self._device) + self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code) + self._config = self._model.config + self.model.eval() + self.model.tie_weights() + self.batch_size_per_gpu = int(batch_size) + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model + # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works + # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work. + if accelerator.distributed_type == DistributedType.DEEPSPEED: + kwargs = { + "train_micro_batch_size_per_gpu": self.batch_size_per_gpu, + "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes, + } + AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs) + eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0") + if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED: + self._model = accelerator.prepare(self.model) + else: + self._model = accelerator.prepare_model(self.model, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self._rank = 0 + self._world_size = 1 + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def tokenizer(self): + return self._tokenizer + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def max_length(self): + return self._max_length + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]: + """ """ + add_special_tokens = False if add_special_tokens is None else add_special_tokens + encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens) + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + return encoding + + def tok_decode(self, tokens): + return self.tokenizer.decode(tokens) + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + # TODO + assert False, "We have not implemented this function for CogVLM2 yet" + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def generate_until(self, requests: List[Instance]) -> List[str]: + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1 + pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding") + for chunk in chunks: + contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk) + task = task[0] + split = split[0] + visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] + visuals = self.flatten(visuals) + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + + # Set default values for until and max_new_tokens + until = [self.tok_decode(self.eot_token_id)] + + # Update values from gen_kwargs if present + if "until" in gen_kwargs: + until = gen_kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}") + assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now" + assert len(visuals) == 1, "CogVLM2 interface does not support bn_image > 1 for now" + context = contexts[0] + if "" in context: + context = context.replace("", "") + + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 1024 + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + if "top_p" not in gen_kwargs: + gen_kwargs["top_p"] = None + if "num_beams" not in gen_kwargs: + gen_kwargs["num_beams"] = 1 + + image = visuals[0] + input_by_model = self.model.build_conversation_input_ids(self.tokenizer, query=context, history=[], images=[image]) + + inputs = { + "input_ids": input_by_model["input_ids"].unsqueeze(0).to(self.device), + "token_type_ids": input_by_model["token_type_ids"].unsqueeze(0).to(self.device), + "attention_mask": input_by_model["attention_mask"].unsqueeze(0).to(self.device), + "images": [[input_by_model["images"][0].to(self.device).to(self.dtype)]], + } + if "cross_images" in input_by_model and input_by_model["cross_images"]: + inputs["cross_images"] = [[input_by_model["cross_images"][0].to(self.device).to(self.dtype)]] + + try: + outputs = self.model.generate(**inputs, **gen_kwargs) + outputs = outputs[:, inputs["input_ids"].shape[1] :] + response = self.tokenizer.decode(outputs[0]) + response = response.split("")[0] + response = response.split("<|end_of_text|>")[0] + + context = [{"role": "user", "content": context}, {"role": "assistant", "content": response}] + except Exception as e: + eval_logger.error(f"Error {e} in generating") + cont = "" + res.append(response) + self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for CogVLM2") diff --git a/LaViDa/eval/lmms_eval/models/from_log.py b/LaViDa/eval/lmms_eval/models/from_log.py new file mode 100644 index 0000000000000000000000000000000000000000..f9f88f9139c5bfe9f940524440547cc58fbc2e47 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/from_log.py @@ -0,0 +1,119 @@ +import json +import os +import re +from datetime import datetime +from typing import List, Tuple + +from accelerate import Accelerator, DistributedType +from loguru import logger as eval_logger +from tqdm import tqdm + +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + + +@register_model("from_log") +class FromLog(lmms): + def __init__( + self, + logs: str = "logs", + model_name: str = None, + model_args: str = None, + have_limits: bool = False, + **kwargs, + ) -> None: + super().__init__() + + self.logs = {} + + log_folders = logs.split(",") + + def matched_model(_model_args): + if model_name and model_name != _model_args["model"]: + return False + + if model_args: + _model_args_list = model_args.split(",") + + for _model_arg in _model_args_list: + if _model_arg not in _model_args["model_args"]: + return False + + if not have_limits and _model_args["limit"] is not None: + return False + + return True + + for log_folder in log_folders: + for root, dirs, files in os.walk(log_folder): + for file in files: + if file.endswith(".json"): + try: + log_file = os.path.join(root, file) + + with open(log_file, "r") as f: + log_data = json.load(f) + + # check if model is matched + _model_args = log_data["args"] + if not matched_model(_model_args): + raise Exception("Model not matched") + + # load logs + logs = {} + for data in log_data["logs"]: + id = data["doc_id"] + response = data["resps"][0] + logs[id] = response + + task = log_data["model_configs"]["task"] + + pattern = re.compile(r"\d{4}_\d{4}") + + if "time" in log_data: + log_time = log_data["time"] + elif pattern.search(os.path.abspath(log_file)): + log_time = pattern.findall(os.path.abspath(log_file))[-1] + else: + log_time = "unknown" + + if task not in self.logs or (self.logs[task]["time"] == "unknown" or datetime.strptime(log_time, "%m%d_%H%M") > datetime.strptime(self.logs[task]["time"], "%m%d_%H%M")): + self.logs[task] = {"time": log_time, "logs": logs} + + except Exception as e: + pass + + accelerator = Accelerator() + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.accelerator = accelerator + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + + self.device = self.accelerator.device + + def generate_until(self, requests) -> List[str]: + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + response = self.logs[task]["logs"][doc_id] + res.append(response[0]) + pbar.update(1) + + pbar.close() + return res + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + # TODO + assert False, "not support" + + def generate_until_multi_round(self, requests) -> List[str]: + return generate_until(self, requests) diff --git a/LaViDa/eval/lmms_eval/models/fuyu.py b/LaViDa/eval/lmms_eval/models/fuyu.py new file mode 100644 index 0000000000000000000000000000000000000000..b453b2e47efb2d2a7d33b5fd0912aa072127b5b0 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/fuyu.py @@ -0,0 +1,269 @@ +import warnings + +warnings.simplefilter("ignore", category=DeprecationWarning) +warnings.filterwarnings("ignore") + +from typing import List, Optional, Tuple, Union + +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState +from loguru import logger as eval_logger +from PIL import Image +from tqdm import tqdm +from transformers import ( + AutoTokenizer, + FuyuForCausalLM, + FuyuImageProcessor, + FuyuProcessor, +) + +from lmms_eval import utils +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + + +@register_model("fuyu") +class Fuyu(lmms): + """ + Fuyu Model + """ + + def __init__( + self, + pretrained: str = "adept/fuyu-8b", + device: Optional[str] = "cuda", + max_new_tokens: int = 256, + batch_size: Optional[Union[int, str]] = 1, + **kwargs, + ) -> None: + super().__init__() + # Do not use kwargs for now + assert kwargs == {}, f"Unexpected kwargs: {kwargs}" + + accelerator = Accelerator() + if accelerator.num_processes > 1: + self._device = torch.device(f"cuda:{accelerator.local_process_index}") + else: + self._device = device + + self._model = FuyuForCausalLM.from_pretrained(pretrained, torch_dtype=torch.bfloat16, device_map=self.device) + self.model.eval() + self.model.tie_weights() + self._tokenizer = AutoTokenizer.from_pretrained(pretrained) + self._config = self.model.config + + self.image_processor = FuyuImageProcessor() + self.processor = FuyuProcessor(image_processor=self.image_processor, tokenizer=self.tokenizer) + self.max_new_tokens = max_new_tokens + self.batch_size_per_gpu = int(batch_size) + accelerator = Accelerator() + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model + # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works + # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work. + if accelerator.distributed_type == DistributedType.DEEPSPEED: + kwargs = { + "train_micro_batch_size_per_gpu": self.batch_size_per_gpu, + "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes, + } + AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs) + eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0") + if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED: + self._model = accelerator.prepare(self.model) + else: + self._model = accelerator.prepare_model(self.model, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.model.to(self._device) + self._rank = 0 + self._world_size = 1 + + """if accelerator.num_processes > 1: + assert accelerator.distributed_type in [ + DistributedType.FSDP, + DistributedType.MULTI_GPU, + ], "Unsupported distributed type provided. Only DDP and FSDP are supported." + if accelerator.distributed_type == DistributedType.FSDP: + self._model = accelerator.prepare(self.model) + else: + self._model = accelerator.prepare_model(self.model, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes""" + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def tokenizer(self): + return self._tokenizer + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def max_length(self): + # Assuming max_length is the sum of max context tokens and max new tokens + return self.tokenizer.model_max_length + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + def flatten(self, input, only_get_first=False): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + if only_get_first: + break + return new_list + + def generate_until(self, requests: List[Instance]) -> List[str]: + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1 + pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding") + + for chunk in chunks: + contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk) + task = task[0] + split = split[0] + visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] + visuals = self.flatten(visuals, only_get_first=True) + gen_kwargs = all_gen_kwargs[0] + + # if isinstance(visuals[0], list): + # visuals = [visuals[idx][0] for idx in range(len(visuals))] # get the first image in multi-image scenarios. + + # assert len(contexts) == self.batch_size_per_gpu, f"Expected contexts batch size {self.batch_size_per_gpu}, got {len(contexts)}" + # assert len(visuals) == self.batch_size_per_gpu, f"Expected visuals batch size {self.batch_size_per_gpu}, got {len(visuals)}" + formatted_contexts = [f"{context}\n" for context in contexts] + model_inputs = self.processor(text=formatted_contexts, images=visuals, device=self.device) + for k, v in model_inputs.items(): + model_inputs[k] = v.to(self.device, non_blocking=True) if isinstance(v, torch.Tensor) else [vv.to(self.device, non_blocking=True) for vv in v] + + for index in range(len(model_inputs["image_patches"])): + model_inputs["image_patches"][index] = model_inputs["image_patches"][index].to(dtype=next(self.model.parameters()).dtype) + + # preconfigure gen_kwargs with defaults + gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))] + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 256 + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + if "top_p" not in gen_kwargs: + gen_kwargs["top_p"] = None + if "num_beams" not in gen_kwargs: + gen_kwargs["num_beams"] = 1 + # generation_output = self.model.generate( + # **model_inputs, temperature=gen_kwargs["temperature"], max_new_tokens=gen_kwargs["max_new_tokens"], top_p=gen_kwargs["top_p"], num_beams=gen_kwargs["num_beams"], pad_token_id=self.tokenizer.eos_token_id + # ) + generation_output = self.model.generate(**model_inputs, max_new_tokens=gen_kwargs["max_new_tokens"], pad_token_id=self.tokenizer.eos_token_id) + generation_texts = self.processor.batch_decode(generation_output, skip_special_tokens=True) + response = [gen_text.split("\x04")[1].strip(" ").strip("\n") for gen_text in generation_texts] + res.extend(response) + pbar.update(1) + + pbar.close() + return res + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + # TODO + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + # encode, pad, and truncate contexts for this batch + if type(doc_to_target) == str: + continuation = doc_to_target + else: + continuation = doc_to_target(self.task_dict[task][split][doc_id]) + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + formatted_contexts = [f"{contexts}\n"] + formatted_continuation = [f"{contexts}\n{continuation}"] + model_inputs = self.processor(text=formatted_continuation, images=visuals, device=self.device) + for k, v in model_inputs.items(): + model_inputs[k] = v.to(self.device, non_blocking=True) if isinstance(v, torch.Tensor) else [vv.to(self.device, non_blocking=True) for vv in v] + + for index in range(len(model_inputs["image_patches"])): + model_inputs["image_patches"][index] = model_inputs["image_patches"][index].to(dtype=next(self.model.parameters()).dtype) + + labels = model_inputs["input_ids"].clone() + contxt_id = self.processor(text=formatted_contexts, return_tensors="pt")["input_ids"] + labels[: len(contxt_id)] = -100 + with torch.inference_mode(): + outputs = self.model(**model_inputs, labels=labels) + loss = outputs["loss"] + # loss = torch.exp(loss) + logits = outputs["logits"] + greedy_tokens = logits.argmax(dim=-1) + cont_toks = model_inputs["input_ids"][:, contxt_id.shape[1] :] # [1, seq] + greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : model_inputs["input_ids"].shape[1]] # [1, seq] + max_equal = (greedy_tokens == cont_toks).all() + res.append((float(loss.item()), bool(max_equal))) + pbar.update(1) + + pbar.close() + return res + + def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]: + """ """ + add_special_tokens = False if add_special_tokens is None else add_special_tokens + encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens) + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + return encoding + + def tok_decode(self, tokens): + return self.tokenizer.decode(tokens) + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Fuyu") diff --git a/LaViDa/eval/lmms_eval/models/gemini_api.py b/LaViDa/eval/lmms_eval/models/gemini_api.py new file mode 100644 index 0000000000000000000000000000000000000000..1d5a770697b2d94212e1c0a1667da30998042eb0 --- /dev/null +++ b/LaViDa/eval/lmms_eval/models/gemini_api.py @@ -0,0 +1,306 @@ +import io +import json +import os +import pathlib +import re +import time +from typing import List, Tuple + +import datasets +from accelerate import Accelerator, DistributedType +from loguru import logger as eval_logger +from PIL import Image +from tqdm import tqdm + +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + +try: + import google.generativeai as genai + from google.generativeai.types import HarmBlockThreshold, HarmCategory + + NUM_SECONDS_TO_SLEEP = 30 + GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") + genai.configure(api_key=GOOGLE_API_KEY) + +except Exception as e: + eval_logger.error(f"Error importing generativeai: {str(e)}") + genai = None + +try: + import soundfile as sf +except Exception as e: + eval_logger.warning(f"Error importing soundfile, audio generation will not work: {str(e)}") + + +@register_model("gemini_api") +class GeminiAPI(lmms): + def __init__( + self, + model_version: str = "gemini-1.5-pro", + # modality: str = "image", + timeout: int = 120, + continual_mode: bool = True, + response_persistent_folder: str = "./logs/gemini_persistent_folder", + interleave: bool = False, + # We will cache the Gemini API response in this path and use it for future requests + **kwargs, + ) -> None: + super().__init__() + self.model_version = model_version + self.timeout = timeout + self.model = genai.GenerativeModel(model_version) + self.continual_mode = continual_mode + self.response_persistent_file = "" + self.interleave = interleave + # if self.continual_mode and response_persistent_folder is None: + # raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.") + if self.continual_mode: + self.response_persistent_folder = response_persistent_folder + if not os.path.exists(self.response_persistent_folder): + os.makedirs(self.response_persistent_folder) + self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json") + + if os.path.exists(self.response_persistent_file): + with open(self.response_persistent_file, "r") as f: + self.response_cache = json.load(f) + self.cache_mode = "resume" + else: + self.response_cache = {} + self.cache_mode = "start" + + accelerator = Accelerator() + if accelerator.num_processes > 1: + assert self.continual_mode is False, "Continual mode is not supported with distributed inference." + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.accelerator = accelerator + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + + self.device = self.accelerator.device + + # self.modality = modality + + self.video_pool = [] + + def free_video(self): + for video in self.video_pool: + video.delete() + self.video_pool = [] + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def get_image_size(self, image): + # Create a BytesIO object to store the image bytes + img_byte_array = io.BytesIO() + + # Save the image to the BytesIO object + image.save(img_byte_array, format="PNG") + + # Get the size of the BytesIO object + img_size = img_byte_array.tell() + + return img_size + + def encode_video(self, video_path): + uploaded_obj = genai.upload_file(path=video_path) + time.sleep(5) + self.video_pool.append(uploaded_obj) + return uploaded_obj + + def encode_audio(self, audio): + audio_io = io.BytesIO() + sf.write(audio_io, audio["array"], audio["sampling_rate"], format="WAV") + return genai.upload_file(audio_io, mime_type="audio/wav") + + def convert_modality(self, images): + for idx, img in enumerate(images): + if isinstance(img, dict) and "sampling_rate" in img: # audio + audio = self.encode_audio(img) + images[idx] = audio + elif isinstance(img, str): # video + try: + images[idx] = self.encode_video(img) + except Exception as e: + eval_logger.error(f"Error converting video: {str(e)}") + return images + + def construct_interleaved_input(self, content, media): + pattern = r"" + parts = re.split(pattern, content) + result = [] + for i, part in enumerate(parts): + if i % 2 == 0: + if part == "": + continue + result.append(part) + else: + result.append(media[int(part)]) + + return result + + def generate_until(self, requests) -> List[str]: + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + def get_uuid(task, split, doc_id): + return f"{task}___{split}___{doc_id}" + + for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + if self.continual_mode and self.cache_mode == "resume": + doc_uuid = get_uuid(task, split, doc_id) + if doc_uuid in self.response_cache: + content = self.response_cache[doc_uuid] + if content: + res.append(content) + pbar.update(1) + continue + + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 1024 + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + + config = genai.GenerationConfig( + max_output_tokens=gen_kwargs["max_new_tokens"], + temperature=gen_kwargs["temperature"], + ) + + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + visuals = self.convert_modality(visuals) + + if self.interleave: + message = self.construct_interleaved_input(contexts, visuals) + else: + message = [contexts] + visuals + + for attempt in range(5): + try: + content = self.model.generate_content( + message, + generation_config=config, + safety_settings={ + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + }, + ) + content = content.text + break + except Exception as e: + eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}") + if isinstance(e, ValueError): + try: + eval_logger.info(f"Prompt feed_back: {content.prompt_feedback}") + content = "" + break + except Exception: + pass + if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt + time.sleep(NUM_SECONDS_TO_SLEEP) + else: # If this was the last attempt, log and return empty + eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}") + content = "" + res.append(content) + pbar.update(1) + + self.free_video() + + if self.continual_mode is True: # Cache the response + doc_uuid = get_uuid(task, split, doc_id) + self.response_cache[doc_uuid] = content + with open(self.response_persistent_file, "w") as f: + json.dump(self.response_cache, f) + + pbar.close() + return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Gemini API") + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + # TODO + assert False, "Gemini API not support" + + def get_image_audio_text_interleaved_messsage(self, image_path, audio_path, question): + # image_path for list of image path + # audio_path for list of audio path + # question for question + + # fixed image token and no audio in text + for index in range(1, 1 + len(image_path)): + question = question.replace(f"[img{index}]", "") + for index in range(1, 1 + len(audio_path)): + question = question.replace(f"[audio{index}]", "