Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- LaViDa/eval/lmms_eval/models/__init__.py +97 -0
- LaViDa/eval/lmms_eval/models/aria.py +334 -0
- LaViDa/eval/lmms_eval/models/auroracap.py +541 -0
- LaViDa/eval/lmms_eval/models/batch_gpt4.py +207 -0
- LaViDa/eval/lmms_eval/models/cambrian.py +313 -0
- LaViDa/eval/lmms_eval/models/claude.py +272 -0
- LaViDa/eval/lmms_eval/models/cogvlm2.py +229 -0
- LaViDa/eval/lmms_eval/models/from_log.py +119 -0
- LaViDa/eval/lmms_eval/models/fuyu.py +269 -0
- LaViDa/eval/lmms_eval/models/gemini_api.py +306 -0
- LaViDa/eval/lmms_eval/models/gpt4v.py +244 -0
- LaViDa/eval/lmms_eval/models/idefics2.py +249 -0
- LaViDa/eval/lmms_eval/models/instructblip.py +230 -0
- LaViDa/eval/lmms_eval/models/internvideo2.py +366 -0
- LaViDa/eval/lmms_eval/models/internvideo2_5.py +295 -0
- LaViDa/eval/lmms_eval/models/internvl.py +496 -0
- LaViDa/eval/lmms_eval/models/internvl2.py +334 -0
- LaViDa/eval/lmms_eval/models/llama_vid.py +282 -0
- LaViDa/eval/lmms_eval/models/llama_vision.py +218 -0
- LaViDa/eval/lmms_eval/models/llava.py +442 -0
- LaViDa/eval/lmms_eval/models/llava_dream.py +855 -0
- LaViDa/eval/lmms_eval/models/llava_hf.py +390 -0
- LaViDa/eval/lmms_eval/models/llava_llada.py +894 -0
- LaViDa/eval/lmms_eval/models/llava_onevision.py +785 -0
- LaViDa/eval/lmms_eval/models/llava_onevision_moviechat.py +526 -0
- LaViDa/eval/lmms_eval/models/llava_sglang.py +165 -0
- LaViDa/eval/lmms_eval/models/llava_vid.py +533 -0
- LaViDa/eval/lmms_eval/models/longva.py +476 -0
- LaViDa/eval/lmms_eval/models/mantis.py +312 -0
- LaViDa/eval/lmms_eval/models/minimonkey.py +334 -0
- LaViDa/eval/lmms_eval/models/model_utils/__init__.py +0 -0
- LaViDa/eval/lmms_eval/models/model_utils/audio_processing.py +7 -0
- LaViDa/eval/lmms_eval/models/model_utils/load_video.py +151 -0
- LaViDa/eval/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py +369 -0
- LaViDa/eval/lmms_eval/models/mplug_owl_video.py +199 -0
- LaViDa/eval/lmms_eval/models/mplug_owl_video/__init__.py +84 -0
- LaViDa/eval/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py +286 -0
- LaViDa/eval/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py +1850 -0
- LaViDa/eval/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py +263 -0
- LaViDa/eval/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py +59 -0
- LaViDa/eval/lmms_eval/models/ola.py +709 -0
- LaViDa/eval/lmms_eval/models/openai_compatible.py +225 -0
- LaViDa/eval/lmms_eval/models/oryx.py +475 -0
- LaViDa/eval/lmms_eval/models/phi3v.py +220 -0
- LaViDa/eval/lmms_eval/models/qwen2_5_vl.py +323 -0
- LaViDa/eval/lmms_eval/models/qwen2_5_vl_interleave.py +408 -0
- LaViDa/eval/lmms_eval/models/qwen2_audio.py +284 -0
- LaViDa/eval/lmms_eval/models/qwen2_vl.py +296 -0
- LaViDa/eval/lmms_eval/models/qwen_vl.py +312 -0
- LaViDa/eval/lmms_eval/models/qwen_vl_api.py +165 -0
LaViDa/eval/lmms_eval/models/__init__.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
|
| 5 |
+
import hf_transfer
|
| 6 |
+
from loguru import logger
|
| 7 |
+
|
| 8 |
+
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 9 |
+
|
| 10 |
+
logger.remove()
|
| 11 |
+
logger.add(sys.stdout, level="WARNING")
|
| 12 |
+
|
| 13 |
+
AVAILABLE_MODELS = {
|
| 14 |
+
"aria": "Aria",
|
| 15 |
+
"auroracap": "AuroraCap",
|
| 16 |
+
"batch_gpt4": "BatchGPT4",
|
| 17 |
+
"claude": "Claude",
|
| 18 |
+
"cogvlm2": "CogVLM2",
|
| 19 |
+
"from_log": "FromLog",
|
| 20 |
+
"fuyu": "Fuyu",
|
| 21 |
+
"gemini_api": "GeminiAPI",
|
| 22 |
+
"gpt4v": "GPT4V",
|
| 23 |
+
"idefics2": "Idefics2",
|
| 24 |
+
"instructblip": "InstructBLIP",
|
| 25 |
+
"internvideo2": "InternVideo2",
|
| 26 |
+
"internvl": "InternVLChat",
|
| 27 |
+
"internvl2": "InternVL2",
|
| 28 |
+
"llama_vid": "LLaMAVid",
|
| 29 |
+
"llama_vision": "LlamaVision",
|
| 30 |
+
"llava": "Llava",
|
| 31 |
+
"llava_hf": "LlavaHf",
|
| 32 |
+
"llava_onevision": "Llava_OneVision",
|
| 33 |
+
"llava_onevision_moviechat": "Llava_OneVision_MovieChat",
|
| 34 |
+
"llava_sglang": "LlavaSglang",
|
| 35 |
+
"llava_vid": "LlavaVid",
|
| 36 |
+
"longva": "LongVA",
|
| 37 |
+
"mantis": "Mantis",
|
| 38 |
+
"minicpm_v": "MiniCPM_V",
|
| 39 |
+
"minimonkey": "MiniMonkey",
|
| 40 |
+
"moviechat": "MovieChat",
|
| 41 |
+
"mplug_owl_video": "mplug_Owl",
|
| 42 |
+
"ola": "Ola",
|
| 43 |
+
"openai_compatible": "OpenAICompatible",
|
| 44 |
+
"oryx": "Oryx",
|
| 45 |
+
"phi3v": "Phi3v",
|
| 46 |
+
"qwen2_5_vl": "Qwen2_5_VL",
|
| 47 |
+
"qwen2_5_vl_interleave": "Qwen2_5_VL_Interleave",
|
| 48 |
+
"qwen2_audio": "Qwen2_Audio",
|
| 49 |
+
"qwen2_vl": "Qwen2_VL",
|
| 50 |
+
"qwen_vl": "Qwen_VL",
|
| 51 |
+
"qwen_vl_api": "Qwen_VL_API",
|
| 52 |
+
"reka": "Reka",
|
| 53 |
+
"ross": "Ross",
|
| 54 |
+
"slime": "Slime",
|
| 55 |
+
"srt_api": "SRT_API",
|
| 56 |
+
"tinyllava": "TinyLlava",
|
| 57 |
+
"videoChatGPT": "VideoChatGPT",
|
| 58 |
+
"videochat2": "VideoChat2",
|
| 59 |
+
"video_llava": "VideoLLaVA",
|
| 60 |
+
"vila": "VILA",
|
| 61 |
+
"vita": "VITA",
|
| 62 |
+
"vllm": "VLLM",
|
| 63 |
+
"xcomposer2_4KHD": "XComposer2_4KHD",
|
| 64 |
+
"xcomposer2d5": "XComposer2D5",
|
| 65 |
+
"egogpt": "EgoGPT",
|
| 66 |
+
"internvideo2_5": "InternVideo2_5",
|
| 67 |
+
"videochat_flash": "VideoChat_Flash",
|
| 68 |
+
"whisper": "Whisper",
|
| 69 |
+
"whisper_vllm": "WhisperVllm",
|
| 70 |
+
"llava_llada":"Llava_Llada",
|
| 71 |
+
"llava_dream":"Llava_Dream"
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def get_model(model_name):
|
| 76 |
+
if model_name not in AVAILABLE_MODELS:
|
| 77 |
+
raise ValueError(f"Model {model_name} not found in available models.")
|
| 78 |
+
|
| 79 |
+
model_class = AVAILABLE_MODELS[model_name]
|
| 80 |
+
if "." not in model_class:
|
| 81 |
+
model_class = f"lmms_eval.models.{model_name}.{model_class}"
|
| 82 |
+
|
| 83 |
+
try:
|
| 84 |
+
model_module, model_class = model_class.rsplit(".", 1)
|
| 85 |
+
module = __import__(model_module, fromlist=[model_class])
|
| 86 |
+
return getattr(module, model_class)
|
| 87 |
+
except Exception as e:
|
| 88 |
+
logger.error(f"Failed to import {model_class} from {model_name}: {e}")
|
| 89 |
+
raise
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
if os.environ.get("LMMS_EVAL_PLUGINS", None):
|
| 93 |
+
# Allow specifying other packages to import models from
|
| 94 |
+
for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","):
|
| 95 |
+
m = importlib.import_module(f"{plugin}.models")
|
| 96 |
+
for model_name, model_class in getattr(m, "AVAILABLE_MODELS").items():
|
| 97 |
+
AVAILABLE_MODELS[model_name] = f"{plugin}.models.{model_name}.{model_class}"
|
LaViDa/eval/lmms_eval/models/aria.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import PIL
|
| 6 |
+
import requests
|
| 7 |
+
import torch
|
| 8 |
+
from accelerate import Accelerator, DistributedType
|
| 9 |
+
from accelerate.state import AcceleratorState
|
| 10 |
+
from decord import VideoReader, cpu
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from tqdm import tqdm
|
| 13 |
+
from transformers import AutoModelForCausalLM, AutoProcessor
|
| 14 |
+
|
| 15 |
+
from lmms_eval import utils
|
| 16 |
+
from lmms_eval.api.instance import Instance
|
| 17 |
+
from lmms_eval.api.model import lmms
|
| 18 |
+
from lmms_eval.api.registry import register_model
|
| 19 |
+
|
| 20 |
+
warnings.filterwarnings("ignore")
|
| 21 |
+
|
| 22 |
+
from loguru import logger as eval_logger
|
| 23 |
+
|
| 24 |
+
DEFAULT_IMAGE_TOKEN = "<image>"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@register_model("aria")
|
| 28 |
+
class Aria(lmms):
|
| 29 |
+
"""
|
| 30 |
+
Llava Model for Hugging Face Transformers: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava
|
| 31 |
+
|
| 32 |
+
Adapted from the LLaVA-HF model in lmms_eval/models/llava_hf.py
|
| 33 |
+
|
| 34 |
+
Example usage:
|
| 35 |
+
|
| 36 |
+
accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
|
| 37 |
+
--model aria \
|
| 38 |
+
--model_args pretrained=rhymes-ai/Aria \
|
| 39 |
+
--tasks seedbench \
|
| 40 |
+
--batch_size 1 \
|
| 41 |
+
--output_path ./logs/ \
|
| 42 |
+
--log_samples
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
def __init__(
|
| 46 |
+
self,
|
| 47 |
+
pretrained: str = "rhymes-ai/Aria",
|
| 48 |
+
revision: str = "main",
|
| 49 |
+
device: str = "cuda",
|
| 50 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 51 |
+
batch_size: int = 1,
|
| 52 |
+
attn_implementation: Optional[str] = None,
|
| 53 |
+
device_map: str = "",
|
| 54 |
+
chat_template: Optional[str] = None,
|
| 55 |
+
use_cache: bool = True,
|
| 56 |
+
specified_eot_token_id: Optional[int] = None,
|
| 57 |
+
max_frames_num: Optional[int] = 64,
|
| 58 |
+
**kwargs,
|
| 59 |
+
) -> None:
|
| 60 |
+
super().__init__()
|
| 61 |
+
# Do not use kwargs for now
|
| 62 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 63 |
+
|
| 64 |
+
accelerator = Accelerator()
|
| 65 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 66 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 67 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 68 |
+
else:
|
| 69 |
+
self._device = torch.device(device)
|
| 70 |
+
self.device_map = device_map
|
| 71 |
+
if isinstance(dtype, str) and dtype != "auto":
|
| 72 |
+
dtype = getattr(torch, dtype)
|
| 73 |
+
|
| 74 |
+
self.max_frames_num = max_frames_num
|
| 75 |
+
self._model = AutoModelForCausalLM.from_pretrained(pretrained, revision=revision, device_map=self.device_map, torch_dtype=torch.bfloat16, trust_remote_code=True, attn_implementation=attn_implementation)
|
| 76 |
+
|
| 77 |
+
self.pretrained = pretrained
|
| 78 |
+
self._image_processor = AutoProcessor.from_pretrained(pretrained, revision=revision, trust_remote_code=True)
|
| 79 |
+
self._tokenizer = self._image_processor.tokenizer
|
| 80 |
+
|
| 81 |
+
self._config = self._model.config
|
| 82 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 83 |
+
self.chat_template = chat_template
|
| 84 |
+
self.specified_eot_token_id = specified_eot_token_id
|
| 85 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 86 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 87 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 88 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 89 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 90 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 91 |
+
kwargs = {
|
| 92 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 93 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 94 |
+
}
|
| 95 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 96 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 97 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 98 |
+
self._model = accelerator.prepare(self.model)
|
| 99 |
+
else:
|
| 100 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 101 |
+
self.accelerator = accelerator
|
| 102 |
+
if self.accelerator.is_local_main_process:
|
| 103 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 104 |
+
self._rank = self.accelerator.local_process_index
|
| 105 |
+
self._world_size = self.accelerator.num_processes
|
| 106 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 107 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
|
| 108 |
+
self._rank = 0
|
| 109 |
+
self._world_size = 1
|
| 110 |
+
else:
|
| 111 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 112 |
+
self.model.to(self._device)
|
| 113 |
+
self._rank = 0
|
| 114 |
+
self._world_size = 1
|
| 115 |
+
self.accelerator = accelerator
|
| 116 |
+
|
| 117 |
+
@property
|
| 118 |
+
def config(self):
|
| 119 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 120 |
+
return self._config
|
| 121 |
+
|
| 122 |
+
@property
|
| 123 |
+
def tokenizer(self):
|
| 124 |
+
return self._tokenizer
|
| 125 |
+
|
| 126 |
+
@property
|
| 127 |
+
def model(self):
|
| 128 |
+
# returns the model, unwrapping it if using Accelerate
|
| 129 |
+
if hasattr(self, "accelerator"):
|
| 130 |
+
return self.accelerator.unwrap_model(self._model)
|
| 131 |
+
else:
|
| 132 |
+
return self._model
|
| 133 |
+
|
| 134 |
+
@property
|
| 135 |
+
def eot_token_id(self):
|
| 136 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 137 |
+
return self.tokenizer.eos_token_id
|
| 138 |
+
|
| 139 |
+
@property
|
| 140 |
+
def max_length(self):
|
| 141 |
+
return self._max_length
|
| 142 |
+
|
| 143 |
+
@property
|
| 144 |
+
def batch_size(self):
|
| 145 |
+
return self.batch_size_per_gpu
|
| 146 |
+
|
| 147 |
+
@property
|
| 148 |
+
def device(self):
|
| 149 |
+
return self._device
|
| 150 |
+
|
| 151 |
+
@property
|
| 152 |
+
def rank(self):
|
| 153 |
+
return self._rank
|
| 154 |
+
|
| 155 |
+
@property
|
| 156 |
+
def world_size(self):
|
| 157 |
+
return self._world_size
|
| 158 |
+
|
| 159 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 160 |
+
""" """
|
| 161 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 162 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 163 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 164 |
+
if left_truncate_len:
|
| 165 |
+
encoding = encoding[-left_truncate_len:]
|
| 166 |
+
return encoding
|
| 167 |
+
|
| 168 |
+
def tok_decode(self, tokens):
|
| 169 |
+
return self.tokenizer.decode(tokens)
|
| 170 |
+
|
| 171 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 172 |
+
raise NotImplementedError("Not implemented for Aria.")
|
| 173 |
+
|
| 174 |
+
def flatten(self, input):
|
| 175 |
+
new_list = []
|
| 176 |
+
for i in input:
|
| 177 |
+
for j in i:
|
| 178 |
+
new_list.append(j)
|
| 179 |
+
return new_list
|
| 180 |
+
|
| 181 |
+
def load_video(self, video_path, max_frames_num):
|
| 182 |
+
if type(video_path) == str:
|
| 183 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 184 |
+
else:
|
| 185 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 186 |
+
total_frame_num = len(vr)
|
| 187 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 188 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 189 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 190 |
+
spare_frames = [Image.fromarray(x) for x in spare_frames]
|
| 191 |
+
return spare_frames # (frames, height, width, channels)
|
| 192 |
+
|
| 193 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 194 |
+
res = []
|
| 195 |
+
|
| 196 |
+
def _collate(x):
|
| 197 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 198 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 199 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 200 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 201 |
+
# automatic adaptive batches much much easier to implement
|
| 202 |
+
# - any OOMs will happen right away rather than near the end
|
| 203 |
+
toks = self.tok_encode(x[0])
|
| 204 |
+
return -len(toks), x[0]
|
| 205 |
+
|
| 206 |
+
# we group requests by their generation_kwargs,
|
| 207 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 208 |
+
# in the same batch.
|
| 209 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 210 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 211 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 212 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 213 |
+
for chunk in chunks:
|
| 214 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 215 |
+
task = task[0]
|
| 216 |
+
split = split[0]
|
| 217 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 218 |
+
visuals = self.flatten(visuals)
|
| 219 |
+
if len(visuals) == 0:
|
| 220 |
+
task_type = "text"
|
| 221 |
+
elif isinstance(visuals[0], PIL.Image.Image):
|
| 222 |
+
task_type = "image"
|
| 223 |
+
elif isinstance(visuals[0], str):
|
| 224 |
+
task_type = "video"
|
| 225 |
+
# we assume all gen kwargs in the batch are the same
|
| 226 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 227 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 228 |
+
|
| 229 |
+
assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
|
| 230 |
+
text_context = contexts[0]
|
| 231 |
+
text_context = text_context.replace("\n\n", "\n")
|
| 232 |
+
|
| 233 |
+
context = []
|
| 234 |
+
|
| 235 |
+
if task_type == "video":
|
| 236 |
+
try:
|
| 237 |
+
visuals = self.load_video(visuals, self.max_frames_num)
|
| 238 |
+
except Exception as e:
|
| 239 |
+
res.append("")
|
| 240 |
+
eval_logger.info(f"Error {e} when loading video : {visuals}")
|
| 241 |
+
pbar.update(1)
|
| 242 |
+
|
| 243 |
+
if DEFAULT_IMAGE_TOKEN not in context:
|
| 244 |
+
context += [{"text": None, "type": "image"}] * len(visuals)
|
| 245 |
+
context += [{"text": "\n" + text_context, "type": "text"}]
|
| 246 |
+
else:
|
| 247 |
+
assert text_context.count(DEFAULT_IMAGE_TOKEN) == len(visuals)
|
| 248 |
+
for i, text_chunk in enumerate(text_context.split(DEFAULT_IMAGE_TOKEN)):
|
| 249 |
+
context += [{"text": text_chunk, "type": "text"}]
|
| 250 |
+
if i < len(visuals):
|
| 251 |
+
context += [{"text": None, "type": "image"}] * len(visuals)
|
| 252 |
+
context += [{"text": "\n", "type": "text"}]
|
| 253 |
+
|
| 254 |
+
# Apply chat template
|
| 255 |
+
messages = [{"role": "user", "content": context}]
|
| 256 |
+
|
| 257 |
+
text = self._image_processor.apply_chat_template(messages, add_generation_prompt=True)
|
| 258 |
+
|
| 259 |
+
# removing redundant placeholders
|
| 260 |
+
text = re.sub(r"<image \d+>", "", text)
|
| 261 |
+
text = re.sub(r"<image>", "", text)
|
| 262 |
+
|
| 263 |
+
eval_logger.debug("DEBUGGING FOR ARIA:" + text)
|
| 264 |
+
|
| 265 |
+
if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
|
| 266 |
+
eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
|
| 267 |
+
|
| 268 |
+
if task_type == "video":
|
| 269 |
+
inputs = self._image_processor(images=visuals, text=text, return_tensors="pt", max_image_size=490)
|
| 270 |
+
else:
|
| 271 |
+
inputs = self._image_processor(images=visuals, text=text, return_tensors="pt", max_image_size=980)
|
| 272 |
+
|
| 273 |
+
inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype)
|
| 274 |
+
inputs = {k: v.to(self._device) for k, v in inputs.items()}
|
| 275 |
+
|
| 276 |
+
if "temperature" not in gen_kwargs:
|
| 277 |
+
gen_kwargs["temperature"] = 0
|
| 278 |
+
if "top_p" not in gen_kwargs:
|
| 279 |
+
gen_kwargs["top_p"] = None
|
| 280 |
+
if "num_beams" not in gen_kwargs:
|
| 281 |
+
gen_kwargs["num_beams"] = 1
|
| 282 |
+
|
| 283 |
+
gen_kwargs["do_sample"] = False
|
| 284 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 285 |
+
|
| 286 |
+
if "until" in gen_kwargs:
|
| 287 |
+
gen_kwargs.pop("until")
|
| 288 |
+
|
| 289 |
+
eval_logger.debug(f"generate kwargs: {gen_kwargs}")
|
| 290 |
+
|
| 291 |
+
try:
|
| 292 |
+
with torch.inference_mode(), torch.cuda.amp.autocast(dtype=torch.bfloat16):
|
| 293 |
+
output = self.model.generate(
|
| 294 |
+
**inputs,
|
| 295 |
+
stop_strings=["<|im_end|>"],
|
| 296 |
+
tokenizer=self._image_processor.tokenizer,
|
| 297 |
+
**gen_kwargs,
|
| 298 |
+
)
|
| 299 |
+
output_ids = output[0][inputs["input_ids"].shape[1] :]
|
| 300 |
+
text_outputs = self._image_processor.decode(output_ids, skip_special_tokens=True).replace("<|im_end|>", "")
|
| 301 |
+
|
| 302 |
+
### Basic Model-wise Parsing for CoT-alike Outputs
|
| 303 |
+
"""
|
| 304 |
+
keywords = [
|
| 305 |
+
"Answer:",
|
| 306 |
+
"answer is:", "choice is:", "option is:",
|
| 307 |
+
"Answer is:", "Choice is:", "Option is:",
|
| 308 |
+
"answer is", "choice is", "option is",
|
| 309 |
+
"Answer is", "Choice is", "Option is"
|
| 310 |
+
]
|
| 311 |
+
|
| 312 |
+
for keyword in keywords:
|
| 313 |
+
if keyword in text_outputs:
|
| 314 |
+
eval_logger.debug(f"ARIA Original generated output simplified by keyword [{keyword}]: {text_outputs}")
|
| 315 |
+
text_outputs = text_outputs.split(keyword, 1)[-1]
|
| 316 |
+
break
|
| 317 |
+
"""
|
| 318 |
+
eval_logger.debug(f"Generated output: {text_outputs}")
|
| 319 |
+
except Exception as ex:
|
| 320 |
+
eval_logger.debug(f"Generation failed: {ex}")
|
| 321 |
+
if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
|
| 322 |
+
eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n")
|
| 323 |
+
|
| 324 |
+
res.append(text_outputs)
|
| 325 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 326 |
+
pbar.update(1)
|
| 327 |
+
# reorder this group of results back to original unsorted form
|
| 328 |
+
res = re_ords.get_original(res)
|
| 329 |
+
|
| 330 |
+
pbar.close()
|
| 331 |
+
return res
|
| 332 |
+
|
| 333 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 334 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
|
LaViDa/eval/lmms_eval/models/auroracap.py
ADDED
|
@@ -0,0 +1,541 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import os
|
| 5 |
+
import os.path as osp
|
| 6 |
+
from typing import List, Optional, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import av
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
from accelerate import Accelerator, DistributedType
|
| 12 |
+
from accelerate.state import AcceleratorState
|
| 13 |
+
from huggingface_hub import snapshot_download
|
| 14 |
+
from peft import PeftModel
|
| 15 |
+
from PIL import Image
|
| 16 |
+
from tqdm import tqdm
|
| 17 |
+
from transformers import (
|
| 18 |
+
AutoModel,
|
| 19 |
+
AutoModelForCausalLM,
|
| 20 |
+
AutoTokenizer,
|
| 21 |
+
BitsAndBytesConfig,
|
| 22 |
+
CLIPImageProcessor,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
from lmms_eval import utils
|
| 26 |
+
from lmms_eval.api.instance import Instance
|
| 27 |
+
from lmms_eval.api.model import lmms
|
| 28 |
+
from lmms_eval.api.registry import register_model
|
| 29 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 30 |
+
from lmms_eval.utils import stop_sequences_criteria
|
| 31 |
+
|
| 32 |
+
try:
|
| 33 |
+
from lmms_eval.models.aurora_xtuner.model.aurora import (
|
| 34 |
+
AuroraEncoder,
|
| 35 |
+
AuroraModel,
|
| 36 |
+
AuroraSigEncoder,
|
| 37 |
+
)
|
| 38 |
+
from lmms_eval.models.aurora_xtuner.utils import PROMPT_TEMPLATE
|
| 39 |
+
except ImportError:
|
| 40 |
+
eval_logger.error("AuroraCap is not installed. Please install AuroraCap to use this model by `git clone https://github.com/rese1f/aurora.git` and link `src/xtuner/xtuner` to `lmms_eval/models/aurora_xtuner`")
|
| 41 |
+
import warnings
|
| 42 |
+
|
| 43 |
+
warnings.filterwarnings("ignore")
|
| 44 |
+
|
| 45 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 46 |
+
|
| 47 |
+
try:
|
| 48 |
+
from llava.constants import (
|
| 49 |
+
DEFAULT_IM_END_TOKEN,
|
| 50 |
+
DEFAULT_IM_START_TOKEN,
|
| 51 |
+
DEFAULT_IMAGE_TOKEN,
|
| 52 |
+
IGNORE_INDEX,
|
| 53 |
+
IMAGE_TOKEN_INDEX,
|
| 54 |
+
)
|
| 55 |
+
from llava.conversation import SeparatorStyle, conv_templates
|
| 56 |
+
from llava.mm_utils import get_model_name_from_path, tokenizer_image_token
|
| 57 |
+
except ImportError:
|
| 58 |
+
eval_logger.error("LLaVA is not installed. Please install LLaVA to use this model.")
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@register_model("auroracap")
|
| 62 |
+
class AuroraCap(lmms):
|
| 63 |
+
"""
|
| 64 |
+
auroracap Model
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
def __init__(
|
| 68 |
+
self,
|
| 69 |
+
pretrained_llm: str = "meta-llama/Meta-Llama-3-8B-Instruct",
|
| 70 |
+
pretrained_vit: str = "google/siglip-so400m-patch14-384",
|
| 71 |
+
pretrained: str = "model/PATH",
|
| 72 |
+
resolution: int = 378,
|
| 73 |
+
token_merge_ratio: float = 0.4,
|
| 74 |
+
device: Optional[str] = "cuda",
|
| 75 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 76 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 77 |
+
conv_template="vicuna_v1", # vicuna_v1",
|
| 78 |
+
video_decode_backend: str = "pyav",
|
| 79 |
+
max_frames_num: int = 16,
|
| 80 |
+
slowfast: bool = False,
|
| 81 |
+
**kwargs,
|
| 82 |
+
) -> None:
|
| 83 |
+
super().__init__()
|
| 84 |
+
# Do not use kwargs for now
|
| 85 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 86 |
+
|
| 87 |
+
accelerator = Accelerator()
|
| 88 |
+
if accelerator.num_processes > 1:
|
| 89 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 90 |
+
else:
|
| 91 |
+
self._device = device
|
| 92 |
+
|
| 93 |
+
pretrained_pth = snapshot_download(repo_id=pretrained) if not osp.isdir(pretrained) else pretrained
|
| 94 |
+
pretrained_llm = pretrained_pth
|
| 95 |
+
pretrained_vit = osp.join(pretrained_pth, "visual_encoder")
|
| 96 |
+
|
| 97 |
+
self._model = AuroraModel(
|
| 98 |
+
slowfast=slowfast,
|
| 99 |
+
llm=AutoModelForCausalLM.from_pretrained(
|
| 100 |
+
pretrained_model_name_or_path=pretrained_llm,
|
| 101 |
+
trust_remote_code=True,
|
| 102 |
+
torch_dtype=torch.float16,
|
| 103 |
+
),
|
| 104 |
+
visual_encoder=AuroraEncoder.from_pretrained(
|
| 105 |
+
pretrained_model_name_or_path=pretrained_vit,
|
| 106 |
+
torch_dtype=torch.float16,
|
| 107 |
+
),
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
projector_path = osp.join(pretrained_pth, "projector")
|
| 111 |
+
self.model.projector = AutoModel.from_pretrained(projector_path, torch_dtype=torch.float16, trust_remote_code=True)
|
| 112 |
+
|
| 113 |
+
self._image_processor = CLIPImageProcessor.from_pretrained(
|
| 114 |
+
pretrained_model_name_or_path="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", # use standard CLIP processor
|
| 115 |
+
trust_remote_code=True,
|
| 116 |
+
size=resolution,
|
| 117 |
+
crop_size=resolution,
|
| 118 |
+
)
|
| 119 |
+
self._tokenizer = AutoTokenizer.from_pretrained(
|
| 120 |
+
pretrained_model_name_or_path=pretrained_llm,
|
| 121 |
+
trust_remote_code=True,
|
| 122 |
+
padding_side="right",
|
| 123 |
+
)
|
| 124 |
+
# compute token merge ratio settings
|
| 125 |
+
self.patch_size = self._model.visual_encoder.config.patch_size
|
| 126 |
+
self.num_layers = self._model.visual_encoder.config.num_hidden_layers
|
| 127 |
+
self.token_merge_ratio = token_merge_ratio
|
| 128 |
+
|
| 129 |
+
self._config = self._model.config
|
| 130 |
+
self.model.eval()
|
| 131 |
+
self.model.tie_weights()
|
| 132 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 133 |
+
self.conv_template = conv_template
|
| 134 |
+
if accelerator.num_processes > 1:
|
| 135 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 136 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 137 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 138 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 139 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 140 |
+
kwargs = {
|
| 141 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 142 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 143 |
+
}
|
| 144 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 145 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 146 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 147 |
+
self._model = accelerator.prepare(self.model)
|
| 148 |
+
self._model.visual_encoder = accelerator.prepare(self.model.visual_encoder)
|
| 149 |
+
self._model.projector = accelerator.prepare(self.model.projector)
|
| 150 |
+
else: # DistributedType.MULTI_GPU
|
| 151 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 152 |
+
self._model.visual_encoder = accelerator.prepare_model(self.model.visual_encoder, evaluation_mode=True)
|
| 153 |
+
self._model.projector = accelerator.prepare_model(self.model.projector, evaluation_mode=True)
|
| 154 |
+
self.accelerator = accelerator
|
| 155 |
+
if self.accelerator.is_local_main_process:
|
| 156 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 157 |
+
self._rank = self.accelerator.local_process_index
|
| 158 |
+
self._world_size = self.accelerator.num_processes
|
| 159 |
+
else:
|
| 160 |
+
self.model.to(self._device)
|
| 161 |
+
self._rank = 0
|
| 162 |
+
self._world_size = 1
|
| 163 |
+
|
| 164 |
+
# For Video Caption
|
| 165 |
+
self.video_decode_backend = video_decode_backend
|
| 166 |
+
self.max_frames_num = int(max_frames_num)
|
| 167 |
+
|
| 168 |
+
@property
|
| 169 |
+
def config(self):
|
| 170 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 171 |
+
return self._config
|
| 172 |
+
|
| 173 |
+
@property
|
| 174 |
+
def tokenizer(self):
|
| 175 |
+
return self._tokenizer
|
| 176 |
+
|
| 177 |
+
@property
|
| 178 |
+
def model(self):
|
| 179 |
+
# returns the model, unwrapping it if using Accelerate
|
| 180 |
+
if hasattr(self, "accelerator"):
|
| 181 |
+
return self.accelerator.unwrap_model(self._model)
|
| 182 |
+
else:
|
| 183 |
+
return self._model
|
| 184 |
+
|
| 185 |
+
@property
|
| 186 |
+
def eot_token_id(self):
|
| 187 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 188 |
+
return self.tokenizer.eos_token_id
|
| 189 |
+
|
| 190 |
+
@property
|
| 191 |
+
def max_length(self):
|
| 192 |
+
return self._max_length
|
| 193 |
+
|
| 194 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 195 |
+
if self.tokenizer.padding_side == "left":
|
| 196 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 197 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 198 |
+
if self.tokenizer.padding_side == "left":
|
| 199 |
+
input_ids = torch.flip(input_ids, [1])
|
| 200 |
+
return input_ids
|
| 201 |
+
|
| 202 |
+
@property
|
| 203 |
+
def batch_size(self):
|
| 204 |
+
return self.batch_size_per_gpu
|
| 205 |
+
|
| 206 |
+
@property
|
| 207 |
+
def device(self):
|
| 208 |
+
return self._device
|
| 209 |
+
|
| 210 |
+
@property
|
| 211 |
+
def rank(self):
|
| 212 |
+
return self._rank
|
| 213 |
+
|
| 214 |
+
@property
|
| 215 |
+
def world_size(self):
|
| 216 |
+
return self._world_size
|
| 217 |
+
|
| 218 |
+
def process_images(self, images, image_processor, model_cfg):
|
| 219 |
+
image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None)
|
| 220 |
+
new_images = []
|
| 221 |
+
if image_aspect_ratio == "pad":
|
| 222 |
+
for image in images:
|
| 223 |
+
image = expand2square(image, tuple(int(x * 255) for x in image_processor.image_mean))
|
| 224 |
+
image = image_processor.preprocess(image, return_tensors="pt")["pixel_values"][0]
|
| 225 |
+
new_images.append(image)
|
| 226 |
+
elif image_aspect_ratio == "anyres":
|
| 227 |
+
for image in images:
|
| 228 |
+
image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints)
|
| 229 |
+
new_images.append(image)
|
| 230 |
+
else:
|
| 231 |
+
return image_processor(images, return_tensors="pt")["pixel_values"]
|
| 232 |
+
if all(x.shape == new_images[0].shape for x in new_images):
|
| 233 |
+
new_images = torch.stack(new_images, dim=0)
|
| 234 |
+
return new_images
|
| 235 |
+
|
| 236 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 237 |
+
""" """
|
| 238 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 239 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 240 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 241 |
+
if left_truncate_len:
|
| 242 |
+
encoding = encoding[-left_truncate_len:]
|
| 243 |
+
return encoding
|
| 244 |
+
|
| 245 |
+
def tok_decode(self, tokens):
|
| 246 |
+
return self.tokenizer.decode(tokens)
|
| 247 |
+
|
| 248 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 249 |
+
res = []
|
| 250 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 251 |
+
|
| 252 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 253 |
+
# encode, pad, and truncate contexts for this batch
|
| 254 |
+
if type(doc_to_target) == str:
|
| 255 |
+
continuation = doc_to_target
|
| 256 |
+
else:
|
| 257 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 258 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 259 |
+
visuals = self.flatten(visuals)
|
| 260 |
+
if visuals:
|
| 261 |
+
image = self.process_images(visuals, self._image_processor, self._config)
|
| 262 |
+
if type(image) is list:
|
| 263 |
+
image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
|
| 264 |
+
else:
|
| 265 |
+
image = image.to(dtype=torch.float16, device=self.device)
|
| 266 |
+
else:
|
| 267 |
+
image = None
|
| 268 |
+
|
| 269 |
+
prompts_input = contexts[0]
|
| 270 |
+
|
| 271 |
+
if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
|
| 272 |
+
"""
|
| 273 |
+
Three senarios:
|
| 274 |
+
1. No image, and there for, no image token should be added.
|
| 275 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 276 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 277 |
+
"""
|
| 278 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
|
| 279 |
+
image_tokens = " ".join(image_tokens)
|
| 280 |
+
prompts_input = image_tokens + "\n" + contexts[0]
|
| 281 |
+
conv = conv_templates[self.conv_template].copy()
|
| 282 |
+
conv.append_message(conv.roles[0], prompts_input)
|
| 283 |
+
conv.append_message(conv.roles[1], None)
|
| 284 |
+
prompt = conv.get_prompt()
|
| 285 |
+
pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 286 |
+
contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 287 |
+
# Add the answer of the second role
|
| 288 |
+
conv.messages[1][1] = continuation
|
| 289 |
+
|
| 290 |
+
prompt = conv.get_prompt()
|
| 291 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 292 |
+
labels = input_ids.clone()
|
| 293 |
+
# Context part no need to calculate for loss
|
| 294 |
+
labels[0, : contxt_id.shape[1]] = -100
|
| 295 |
+
with torch.inference_mode():
|
| 296 |
+
data = dict()
|
| 297 |
+
data["pixel_values"] = image_tensor
|
| 298 |
+
data["input_ids"] = input_ids
|
| 299 |
+
data["attention_mask"] = attention_masks
|
| 300 |
+
self.model.visual_encoder.reset_tome_r(self.token_merge_ratio)
|
| 301 |
+
output = self.model(data, mode="tensor")
|
| 302 |
+
|
| 303 |
+
loss = outputs["loss"]
|
| 304 |
+
# loss = torch.exp(loss)
|
| 305 |
+
logits = outputs["logits"]
|
| 306 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 307 |
+
cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
|
| 308 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
|
| 309 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 310 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 311 |
+
pbar.update(1)
|
| 312 |
+
pbar.close()
|
| 313 |
+
return res
|
| 314 |
+
|
| 315 |
+
def flatten(self, input):
|
| 316 |
+
new_list = []
|
| 317 |
+
for i in input:
|
| 318 |
+
for j in i:
|
| 319 |
+
new_list.append(j)
|
| 320 |
+
return new_list
|
| 321 |
+
|
| 322 |
+
def load_video(self, video_path, max_frames_num):
|
| 323 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 324 |
+
total_frame_num = len(vr)
|
| 325 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 326 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 327 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 328 |
+
return spare_frames # (frames, height, width, channels)
|
| 329 |
+
|
| 330 |
+
def extract_keyframes(self, video_path, keyframes):
|
| 331 |
+
container = av.open(video_path)
|
| 332 |
+
video_stream = container.streams.video[0]
|
| 333 |
+
fps = video_stream.average_rate
|
| 334 |
+
time_base = video_stream.time_base
|
| 335 |
+
frames = []
|
| 336 |
+
|
| 337 |
+
for keyframe in keyframes:
|
| 338 |
+
keyframe_time = float(keyframe)
|
| 339 |
+
frame_number = int(keyframe_time * fps)
|
| 340 |
+
container.seek(int(keyframe_time / time_base))
|
| 341 |
+
found = False
|
| 342 |
+
for packet in container.demux(video=0):
|
| 343 |
+
for frame in packet.decode():
|
| 344 |
+
if frame.index >= frame_number:
|
| 345 |
+
frames.append(frame)
|
| 346 |
+
found = True
|
| 347 |
+
break
|
| 348 |
+
if found:
|
| 349 |
+
break
|
| 350 |
+
|
| 351 |
+
if not found:
|
| 352 |
+
container.seek(-1, any_frame=False)
|
| 353 |
+
for packet in container.demux(video=0):
|
| 354 |
+
for frame in packet.decode():
|
| 355 |
+
pass
|
| 356 |
+
frames.append(frame)
|
| 357 |
+
|
| 358 |
+
return np.stack([x.to_ndarray(format="rgb24") for x in frames])
|
| 359 |
+
|
| 360 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 361 |
+
res = []
|
| 362 |
+
|
| 363 |
+
def _collate(x):
|
| 364 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 365 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 366 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 367 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 368 |
+
# automatic adaptive batches much much easier to implement
|
| 369 |
+
# - any OOMs will happen right away rather than near the end
|
| 370 |
+
toks = self.tok_encode(x[0])
|
| 371 |
+
return -len(toks), x[0]
|
| 372 |
+
|
| 373 |
+
# we group requests by their generation_kwargs,
|
| 374 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 375 |
+
# in the same batch.
|
| 376 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 377 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 378 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 379 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 380 |
+
|
| 381 |
+
for chunk in chunks:
|
| 382 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 383 |
+
task = task[0]
|
| 384 |
+
split = split[0]
|
| 385 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # the length of visuals is 1, equal to batchsize
|
| 386 |
+
visuals = self.flatten(visuals)
|
| 387 |
+
# we assume all gen kwargs in the batch are the same
|
| 388 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 389 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 390 |
+
|
| 391 |
+
# Set default values for until and max_new_tokens
|
| 392 |
+
until = [self.tok_decode(self.eot_token_id)]
|
| 393 |
+
|
| 394 |
+
# Update values from gen_kwargs if present
|
| 395 |
+
if "until" in gen_kwargs:
|
| 396 |
+
until = gen_kwargs.pop("until")
|
| 397 |
+
if isinstance(until, str):
|
| 398 |
+
until = [until]
|
| 399 |
+
elif not isinstance(until, list):
|
| 400 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 401 |
+
|
| 402 |
+
if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
|
| 403 |
+
# here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
|
| 404 |
+
self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
|
| 405 |
+
eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
|
| 406 |
+
# encode, pad, and truncate contexts for this batch
|
| 407 |
+
if visuals:
|
| 408 |
+
if isinstance(visuals[0], dict):
|
| 409 |
+
video_path = visuals[0]["video_path"]
|
| 410 |
+
keyframe = visuals[0]["keyframe"]
|
| 411 |
+
video = self.extract_keyframes(video_path, keyframe)
|
| 412 |
+
image_tensor = self.process_images(video, self._image_processor, self._config).cuda()
|
| 413 |
+
elif isinstance(visuals, list):
|
| 414 |
+
print(visuals[0])
|
| 415 |
+
if isinstance(visuals[0], Image.Image):
|
| 416 |
+
image_tensor = self.process_images(visuals, self._image_processor, self._config)
|
| 417 |
+
else:
|
| 418 |
+
if visuals[0].endswith("mp4"):
|
| 419 |
+
if self.video_decode_backend == "decord":
|
| 420 |
+
video = self.load_video(visuals[0], self.max_frames_num)
|
| 421 |
+
elif self.video_decode_backend == "pyav":
|
| 422 |
+
video = read_video_pyav(visuals[0], num_frm=self.max_frames_num)
|
| 423 |
+
image_tensor = self.process_images(video, self._image_processor, self._config).cuda()
|
| 424 |
+
elif visuals[0].endswith("mkv"):
|
| 425 |
+
assert self.video_decode_backend == "pyav", "we only tested this case, decord may not work"
|
| 426 |
+
video = read_video_pyav(visuals[0], num_frm=self.max_frames_num)
|
| 427 |
+
image_tensor = self.process_images(video, self._image_processor, self._config).cuda()
|
| 428 |
+
|
| 429 |
+
if type(image_tensor) is list:
|
| 430 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 431 |
+
else:
|
| 432 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 433 |
+
|
| 434 |
+
else:
|
| 435 |
+
image_tensor = None
|
| 436 |
+
|
| 437 |
+
question_input = []
|
| 438 |
+
|
| 439 |
+
for visual, context in zip(visuals, contexts):
|
| 440 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 441 |
+
"""
|
| 442 |
+
Three senarios:
|
| 443 |
+
1. No image, and there for, no image token should be added.
|
| 444 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 445 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 446 |
+
"""
|
| 447 |
+
if isinstance(visuals[0], dict):
|
| 448 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(video)
|
| 449 |
+
elif isinstance(visuals, list):
|
| 450 |
+
if isinstance(visuals[0], Image.Image):
|
| 451 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 452 |
+
else:
|
| 453 |
+
if visual.endswith("mp4") or visual.endswith("mkv"):
|
| 454 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(video)
|
| 455 |
+
|
| 456 |
+
image_tokens = " ".join(image_tokens)
|
| 457 |
+
question = image_tokens + "\n" + context
|
| 458 |
+
|
| 459 |
+
else:
|
| 460 |
+
question = context
|
| 461 |
+
conv = conv_templates[self.conv_template].copy()
|
| 462 |
+
conv.append_message(conv.roles[0], question)
|
| 463 |
+
conv.append_message(conv.roles[1], None)
|
| 464 |
+
prompt_question = conv.get_prompt()
|
| 465 |
+
question_input.append(prompt_question)
|
| 466 |
+
|
| 467 |
+
# The above for loop has bugs. When there is no visuals, e.g. pure text,
|
| 468 |
+
# there will be no for loop execute resulting in an empty question_input (because no visuals)
|
| 469 |
+
# Scenario 1 won't even be execute
|
| 470 |
+
if len(visuals) == 0:
|
| 471 |
+
for context in contexts:
|
| 472 |
+
question = context
|
| 473 |
+
conv = conv_templates[self.conv_template].copy()
|
| 474 |
+
conv.append_message(conv.roles[0], question)
|
| 475 |
+
conv.append_message(conv.roles[1], None)
|
| 476 |
+
prompt_question = conv.get_prompt()
|
| 477 |
+
question_input.append(prompt_question)
|
| 478 |
+
|
| 479 |
+
# preconfigure gen_kwargs with defaults
|
| 480 |
+
if isinstance(visuals[0], dict):
|
| 481 |
+
gen_kwargs["image_sizes"] = [video[idx].size for idx in range(len(video))]
|
| 482 |
+
elif isinstance(visuals, list):
|
| 483 |
+
if isinstance(visuals[0], Image.Image):
|
| 484 |
+
gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
|
| 485 |
+
else:
|
| 486 |
+
if visuals[0].endswith("mp4"):
|
| 487 |
+
gen_kwargs["image_sizes"] = [video[idx].size for idx in range(len(video))]
|
| 488 |
+
|
| 489 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 490 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 491 |
+
if "temperature" not in gen_kwargs:
|
| 492 |
+
gen_kwargs["temperature"] = 0
|
| 493 |
+
if "top_p" not in gen_kwargs:
|
| 494 |
+
gen_kwargs["top_p"] = None
|
| 495 |
+
if "num_beams" not in gen_kwargs:
|
| 496 |
+
gen_kwargs["num_beams"] = 1
|
| 497 |
+
|
| 498 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 499 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 500 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 501 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 502 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 503 |
+
try:
|
| 504 |
+
data = dict()
|
| 505 |
+
if isinstance(visuals[0], dict):
|
| 506 |
+
data["pixel_values"] = image_tensor.unsqueeze(0)
|
| 507 |
+
elif isinstance(visuals, list):
|
| 508 |
+
if isinstance(visuals[0], Image.Image):
|
| 509 |
+
data["pixel_values"] = image_tensor
|
| 510 |
+
else:
|
| 511 |
+
if visuals[0].endswith("mp4") or visuals[0].endswith("mkv"):
|
| 512 |
+
data["pixel_values"] = image_tensor.unsqueeze(0)
|
| 513 |
+
|
| 514 |
+
data["input_ids"] = input_ids
|
| 515 |
+
data["attention_mask"] = attention_masks
|
| 516 |
+
self.model.visual_encoder.reset_tome_r(self.token_merge_ratio)
|
| 517 |
+
output = self.model(data, mode="inference")
|
| 518 |
+
cont = self.model.llm.generate(
|
| 519 |
+
**output,
|
| 520 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 521 |
+
temperature=gen_kwargs["temperature"],
|
| 522 |
+
top_p=gen_kwargs["top_p"],
|
| 523 |
+
num_beams=gen_kwargs["num_beams"],
|
| 524 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 525 |
+
)
|
| 526 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 527 |
+
except Exception as e:
|
| 528 |
+
eval_logger.error(f"Error {e} in generating")
|
| 529 |
+
cont = ""
|
| 530 |
+
text_outputs = [""]
|
| 531 |
+
|
| 532 |
+
print(text_outputs)
|
| 533 |
+
|
| 534 |
+
res.extend(text_outputs)
|
| 535 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 536 |
+
pbar.update(1)
|
| 537 |
+
# reorder this group of results back to original unsorted form
|
| 538 |
+
res = re_ords.get_original(res)
|
| 539 |
+
|
| 540 |
+
pbar.close()
|
| 541 |
+
return res
|
LaViDa/eval/lmms_eval/models/batch_gpt4.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Standard library imports
|
| 2 |
+
import base64
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import time
|
| 6 |
+
from copy import deepcopy
|
| 7 |
+
from io import BytesIO
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import requests as url_requests
|
| 11 |
+
|
| 12 |
+
# Related third-party imports
|
| 13 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 14 |
+
from accelerate.state import AcceleratorState
|
| 15 |
+
from loguru import logger as eval_logger
|
| 16 |
+
from openai import OpenAI
|
| 17 |
+
from PIL import Image
|
| 18 |
+
from tqdm import tqdm
|
| 19 |
+
|
| 20 |
+
# Local application/library specific imports
|
| 21 |
+
from lmms_eval.api.instance import Instance
|
| 22 |
+
from lmms_eval.api.model import lmms
|
| 23 |
+
from lmms_eval.api.registry import register_model
|
| 24 |
+
|
| 25 |
+
# Conditional imports
|
| 26 |
+
try:
|
| 27 |
+
from decord import VideoReader, cpu
|
| 28 |
+
except ImportError:
|
| 29 |
+
eval_logger.warning("Decord is not installed. Video input will not be supported.")
|
| 30 |
+
|
| 31 |
+
# Constants and global configurations
|
| 32 |
+
API_TYPE = os.getenv("API_TYPE", "openai")
|
| 33 |
+
NUM_SECONDS_TO_SLEEP = 5
|
| 34 |
+
|
| 35 |
+
if API_TYPE == "openai":
|
| 36 |
+
API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
|
| 37 |
+
API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
|
| 38 |
+
headers = {
|
| 39 |
+
"Authorization": f"Bearer {API_KEY}",
|
| 40 |
+
"Content-Type": "application/json",
|
| 41 |
+
}
|
| 42 |
+
elif API_TYPE == "azure":
|
| 43 |
+
API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
|
| 44 |
+
API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
|
| 45 |
+
headers = {
|
| 46 |
+
"api-key": API_KEY,
|
| 47 |
+
"Content-Type": "application/json",
|
| 48 |
+
}
|
| 49 |
+
else:
|
| 50 |
+
API_URL = "YOUR_API_URL"
|
| 51 |
+
API_KEY = "YOUR_API_KEY"
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@register_model("batch_gpt4")
|
| 55 |
+
class BatchGPT4(lmms):
|
| 56 |
+
def __init__(
|
| 57 |
+
self,
|
| 58 |
+
model_version: str = "gpt-4o",
|
| 59 |
+
api_key: str = API_KEY,
|
| 60 |
+
api_url: str = API_URL,
|
| 61 |
+
modality: str = "image",
|
| 62 |
+
max_frames_num: int = 10,
|
| 63 |
+
timeout: int = 120,
|
| 64 |
+
**kwargs,
|
| 65 |
+
) -> None:
|
| 66 |
+
super().__init__()
|
| 67 |
+
# Manually set a image token for GPT4V so that we can search for it
|
| 68 |
+
# and split the text and image
|
| 69 |
+
# Here we just use the same token as llava for convenient
|
| 70 |
+
self.model_version = model_version
|
| 71 |
+
self.modality = modality
|
| 72 |
+
self.max_frames_num = max_frames_num
|
| 73 |
+
self.image_token = "<image>"
|
| 74 |
+
self.timeout = timeout
|
| 75 |
+
|
| 76 |
+
self.api_key = api_key
|
| 77 |
+
self.api_url = api_url
|
| 78 |
+
self.client = OpenAI(api_key=api_key)
|
| 79 |
+
|
| 80 |
+
accelerator = Accelerator()
|
| 81 |
+
assert accelerator.state.local_process_index == 0, "BatchGPT4 does not support distributed inference."
|
| 82 |
+
assert accelerator.state.num_processes == 1, "BatchGPT4 does not support distributed inference."
|
| 83 |
+
|
| 84 |
+
# Function to encode the image
|
| 85 |
+
def encode_image(self, image: Image):
|
| 86 |
+
output_buffer = BytesIO()
|
| 87 |
+
image.save(output_buffer, format="PNG")
|
| 88 |
+
byte_data = output_buffer.getvalue()
|
| 89 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 90 |
+
return base64_str
|
| 91 |
+
|
| 92 |
+
# Function to encode the video
|
| 93 |
+
def encode_video(self, video_path, for_get_frames_num):
|
| 94 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 95 |
+
total_frame_num = len(vr)
|
| 96 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int)
|
| 97 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 98 |
+
frames = vr.get_batch(frame_idx).asnumpy()
|
| 99 |
+
|
| 100 |
+
base64_frames = []
|
| 101 |
+
for frame in frames:
|
| 102 |
+
img = Image.fromarray(frame)
|
| 103 |
+
output_buffer = BytesIO()
|
| 104 |
+
img.save(output_buffer, format="PNG")
|
| 105 |
+
byte_data = output_buffer.getvalue()
|
| 106 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 107 |
+
base64_frames.append(base64_str)
|
| 108 |
+
|
| 109 |
+
return base64_frames
|
| 110 |
+
|
| 111 |
+
def flatten(self, input):
|
| 112 |
+
new_list = []
|
| 113 |
+
for i in input:
|
| 114 |
+
for j in i:
|
| 115 |
+
new_list.append(j)
|
| 116 |
+
return new_list
|
| 117 |
+
|
| 118 |
+
def generate_until(self, requests):
|
| 119 |
+
# Prepare the batch requests data
|
| 120 |
+
requests_data = {}
|
| 121 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Batch Preparing")
|
| 122 |
+
for idx, (contexts, gen_kwargs, doc_to_visual, doc_id, task, split) in enumerate([reg.args for reg in requests]):
|
| 123 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 124 |
+
visuals = self.flatten(visuals)
|
| 125 |
+
imgs = []
|
| 126 |
+
for visual in visuals:
|
| 127 |
+
if self.modality == "image":
|
| 128 |
+
img = self.encode_image(visual)
|
| 129 |
+
imgs.append(img)
|
| 130 |
+
elif self.modality == "video":
|
| 131 |
+
frames = self.encode_video(visual, self.max_frames_num)
|
| 132 |
+
imgs.extend(frames)
|
| 133 |
+
|
| 134 |
+
messages = []
|
| 135 |
+
if self.image_token not in contexts:
|
| 136 |
+
messages.append({"role": "user", "content": contexts})
|
| 137 |
+
for img in imgs:
|
| 138 |
+
messages.append({"role": "user", "content": f"data:image/jpeg;base64,{img}"})
|
| 139 |
+
else:
|
| 140 |
+
contexts_split = contexts.split(self.image_token)
|
| 141 |
+
for idx, context in enumerate(contexts_split):
|
| 142 |
+
if idx < len(imgs):
|
| 143 |
+
messages.append({"role": "user", "content": context})
|
| 144 |
+
messages.append({"role": "user", "content": f"data:image/jpeg;base64,{imgs[idx]}"})
|
| 145 |
+
if len(contexts_split) > len(imgs):
|
| 146 |
+
messages.append({"role": "user", "content": contexts_split[-1]})
|
| 147 |
+
|
| 148 |
+
requests_data[f"request-{idx}"] = {"model": self.model_version, "messages": messages, "max_tokens": gen_kwargs.get("max_new_tokens", 1024)}
|
| 149 |
+
pbar.update(1)
|
| 150 |
+
|
| 151 |
+
file_path = os.getenv("HF_HOME", "~/.cache/huggingface") + f"/batchinput_{len(requests_data)}.jsonl"
|
| 152 |
+
file_path = self.create_batch_input_file(requests_data, file_path)
|
| 153 |
+
file_id = self.upload_input_file(file_path)
|
| 154 |
+
|
| 155 |
+
batch_response = self.create_batch(file_id, metadata={"description": "Batch Processing for GPT-4"})
|
| 156 |
+
batch_status = self.check_batch_status(batch_response.id)
|
| 157 |
+
while True:
|
| 158 |
+
batch_status = self.check_batch_status(batch_response.id)
|
| 159 |
+
if batch_status.status == "completed":
|
| 160 |
+
eval_logger.info("Batch processing completed.")
|
| 161 |
+
batch_results = self.retrieve_batch_results(batch_status.output_file_id)
|
| 162 |
+
res = [result["response"]["choices"][0]["message"]["content"] for result in json.loads(batch_results)]
|
| 163 |
+
return res
|
| 164 |
+
elif batch_status.status == "failed":
|
| 165 |
+
eval_logger.info("Batch processing failed.")
|
| 166 |
+
res = ["Batch failed"] * len(requests)
|
| 167 |
+
return res
|
| 168 |
+
else:
|
| 169 |
+
eval_logger.info(f"Batch status: {batch_status.status}. Retrying in {NUM_SECONDS_TO_SLEEP} seconds.")
|
| 170 |
+
time.sleep(NUM_SECONDS_TO_SLEEP)
|
| 171 |
+
|
| 172 |
+
def loglikelihood(self, requests):
|
| 173 |
+
# TODO
|
| 174 |
+
assert False, "GPT4V not support"
|
| 175 |
+
|
| 176 |
+
def create_batch_input_file(self, requests_data, file_path="batchinput.jsonl"):
|
| 177 |
+
with open(file_path, "w") as file:
|
| 178 |
+
for request_id, data in requests_data.items():
|
| 179 |
+
json_record = json.dumps({"custom_id": request_id, "method": "POST", "url": "/v1/chat/completions", "body": data})
|
| 180 |
+
file.write(json_record + "\n")
|
| 181 |
+
return file_path
|
| 182 |
+
|
| 183 |
+
def upload_input_file(self, file_path):
|
| 184 |
+
with open(file_path, "rb") as file:
|
| 185 |
+
response = self.client.files.create(file=file, purpose="batch")
|
| 186 |
+
return response.id
|
| 187 |
+
|
| 188 |
+
def create_batch(self, file_id, metadata=None):
|
| 189 |
+
if metadata is None:
|
| 190 |
+
metadata = {}
|
| 191 |
+
response = self.client.batches.create(input_file_id=file_id, endpoint="/v1/chat/completions", completion_window="24h", metadata=metadata)
|
| 192 |
+
return response
|
| 193 |
+
|
| 194 |
+
def check_batch_status(self, batch_id):
|
| 195 |
+
return self.client.batches.retrieve(batch_id)
|
| 196 |
+
|
| 197 |
+
def retrieve_batch_results(self, file_id):
|
| 198 |
+
return self.client.files.content(file_id)
|
| 199 |
+
|
| 200 |
+
def cancel_batch(self, batch_id):
|
| 201 |
+
return self.client.batches.cancel(batch_id)
|
| 202 |
+
|
| 203 |
+
def list_batches(self, limit=10):
|
| 204 |
+
return self.client.batches.list(limit=limit)
|
| 205 |
+
|
| 206 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 207 |
+
raise NotImplementedError("TODO: Implement multi-round generation for BatchGPT4")
|
LaViDa/eval/lmms_eval/models/cambrian.py
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# code is migrated from https://github.com/kangreen0210/LIME-M/blob/main/lmms_eval/models/cambrian.py
|
| 2 |
+
import os
|
| 3 |
+
import uuid
|
| 4 |
+
import warnings
|
| 5 |
+
from typing import List, Optional, Tuple, Union
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from accelerate import Accelerator, DistributedType
|
| 9 |
+
from PIL import Image
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
from transformers import PreTrainedTokenizer
|
| 12 |
+
|
| 13 |
+
from lmms_eval import utils
|
| 14 |
+
from lmms_eval.api.instance import Instance
|
| 15 |
+
from lmms_eval.api.model import lmms
|
| 16 |
+
from lmms_eval.api.registry import register_model
|
| 17 |
+
|
| 18 |
+
warnings.simplefilter("ignore", category=DeprecationWarning)
|
| 19 |
+
warnings.filterwarnings("ignore")
|
| 20 |
+
|
| 21 |
+
from loguru import logger as eval_logger
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from cambrian.conversation import conv_templates
|
| 25 |
+
from cambrian.mm_utils import (
|
| 26 |
+
get_model_name_from_path,
|
| 27 |
+
process_images,
|
| 28 |
+
tokenizer_image_token,
|
| 29 |
+
)
|
| 30 |
+
from cambrian.model.builder import load_pretrained_model
|
| 31 |
+
except ImportError:
|
| 32 |
+
eval_logger.error("Cambrian is not installed. Please install it by running `pip install cambrian`.")
|
| 33 |
+
|
| 34 |
+
# Model Constants
|
| 35 |
+
IMAGE_TOKEN_INDEX = -200
|
| 36 |
+
DEFAULT_IMAGE_TOKEN = "<image>"
|
| 37 |
+
DEFAULT_IM_START_TOKEN = "<im_start>"
|
| 38 |
+
DEFAULT_IM_END_TOKEN = "<im_end>"
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def process(image, question, tokenizer, image_processor, model_config, conv_mode):
|
| 42 |
+
qs = question
|
| 43 |
+
|
| 44 |
+
if model_config.mm_use_im_start_end:
|
| 45 |
+
qs = f"{DEFAULT_IM_START_TOKEN}{DEFAULT_IMAGE_TOKEN}{DEFAULT_IM_END_TOKEN}\n{qs}"
|
| 46 |
+
else:
|
| 47 |
+
qs = f"{DEFAULT_IMAGE_TOKEN}\n{qs}"
|
| 48 |
+
|
| 49 |
+
conv = conv_templates[conv_mode].copy()
|
| 50 |
+
conv.append_message(conv.roles[0], qs)
|
| 51 |
+
conv.append_message(conv.roles[1], None)
|
| 52 |
+
prompt = conv.get_prompt()
|
| 53 |
+
|
| 54 |
+
image_size = [image.size]
|
| 55 |
+
image_tensor = process_images([image], image_processor, model_config)
|
| 56 |
+
|
| 57 |
+
input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
|
| 58 |
+
|
| 59 |
+
return input_ids, image_tensor, image_size, prompt
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def make_context(
|
| 63 |
+
tokenizer: PreTrainedTokenizer,
|
| 64 |
+
query: str,
|
| 65 |
+
history: List[Tuple[str, str]] = None,
|
| 66 |
+
system: str = "",
|
| 67 |
+
max_window_size: int = 6144,
|
| 68 |
+
chat_format: str = "chatml",
|
| 69 |
+
):
|
| 70 |
+
if history is None:
|
| 71 |
+
history = []
|
| 72 |
+
|
| 73 |
+
if chat_format == "chatml":
|
| 74 |
+
im_start, im_end = "<|im_start|>", "<|im_end|>"
|
| 75 |
+
im_start_tokens = [tokenizer.im_start_id]
|
| 76 |
+
im_end_tokens = [tokenizer.im_end_id]
|
| 77 |
+
nl_tokens = tokenizer.encode("\n")
|
| 78 |
+
|
| 79 |
+
def _tokenize_str(role, content):
|
| 80 |
+
return f"{role}\n{content}", tokenizer.encode(role, allowed_special=set(tokenizer.IMAGE_ST)) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST))
|
| 81 |
+
|
| 82 |
+
system_text, system_tokens_part = _tokenize_str("system", system)
|
| 83 |
+
system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
|
| 84 |
+
|
| 85 |
+
raw_text = ""
|
| 86 |
+
context_tokens = []
|
| 87 |
+
|
| 88 |
+
for turn_query, turn_response in reversed(history):
|
| 89 |
+
query_text, query_tokens_part = _tokenize_str("user", turn_query)
|
| 90 |
+
query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
|
| 91 |
+
if turn_response is not None:
|
| 92 |
+
response_text, response_tokens_part = _tokenize_str("assistant", turn_response)
|
| 93 |
+
response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
|
| 94 |
+
|
| 95 |
+
next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
|
| 96 |
+
prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
|
| 97 |
+
else:
|
| 98 |
+
next_context_tokens = nl_tokens + query_tokens + nl_tokens
|
| 99 |
+
prev_chat = f"\n{im_start}{query_text}{im_end}\n"
|
| 100 |
+
|
| 101 |
+
current_context_size = len(system_tokens) + len(next_context_tokens) + len(context_tokens)
|
| 102 |
+
if current_context_size < max_window_size:
|
| 103 |
+
context_tokens = next_context_tokens + context_tokens
|
| 104 |
+
raw_text = prev_chat + raw_text
|
| 105 |
+
else:
|
| 106 |
+
break
|
| 107 |
+
|
| 108 |
+
context_tokens = system_tokens + context_tokens
|
| 109 |
+
raw_text = f"{im_start}{system_text}{im_end}" + raw_text
|
| 110 |
+
context_tokens += nl_tokens + im_start_tokens + _tokenize_str("user", query)[1] + im_end_tokens + nl_tokens + im_start_tokens + tokenizer.encode("assistant") + nl_tokens
|
| 111 |
+
raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
|
| 112 |
+
|
| 113 |
+
elif chat_format == "raw":
|
| 114 |
+
raw_text = query
|
| 115 |
+
context_tokens = tokenizer.encode(raw_text)
|
| 116 |
+
else:
|
| 117 |
+
raise NotImplementedError(f"Unknown chat format {chat_format!r}")
|
| 118 |
+
|
| 119 |
+
return raw_text, context_tokens
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@register_model("cambrian")
|
| 123 |
+
class Cambrian(lmms):
|
| 124 |
+
def __init__(
|
| 125 |
+
self,
|
| 126 |
+
pretrained: str = "nyu-visionx/cambrian-8b",
|
| 127 |
+
device: Optional[str] = "cuda",
|
| 128 |
+
device_map="auto",
|
| 129 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 130 |
+
trust_remote_code: Optional[bool] = True,
|
| 131 |
+
use_cache=True,
|
| 132 |
+
**kwargs,
|
| 133 |
+
) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
assert not kwargs, f"Unexpected kwargs: {kwargs}"
|
| 136 |
+
|
| 137 |
+
accelerator = Accelerator()
|
| 138 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}") if accelerator.num_processes > 1 else device
|
| 139 |
+
|
| 140 |
+
self.model_name = get_model_name_from_path(pretrained)
|
| 141 |
+
tokenizer, model, self.image_processor, context_len = load_pretrained_model(pretrained, None, self.model_name, device_map=self._device)
|
| 142 |
+
|
| 143 |
+
self.conv_mode = {"cambrian-8b": "llama_3", "cambrian-13b": "vicuna_v1", "cambrian-34b": "chatml_direct"}.get(self.model_name)
|
| 144 |
+
|
| 145 |
+
if not self.conv_mode:
|
| 146 |
+
raise ValueError(f"Unsupported model: {self.model_name}")
|
| 147 |
+
|
| 148 |
+
self._model = model
|
| 149 |
+
self._tokenizer = tokenizer
|
| 150 |
+
self._model.eval()
|
| 151 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 152 |
+
self.use_cache = use_cache
|
| 153 |
+
self._rank = 0
|
| 154 |
+
self._world_size = 1
|
| 155 |
+
|
| 156 |
+
if accelerator.num_processes > 1:
|
| 157 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU], "Unsupported distributed type. Only DDP and FSDP are supported."
|
| 158 |
+
self._model = accelerator.prepare(self.model) if accelerator.distributed_type == DistributedType.FSDP else accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 159 |
+
self.accelerator = accelerator
|
| 160 |
+
if self.accelerator.is_local_main_process:
|
| 161 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 162 |
+
self._rank = self.accelerator.local_process_index
|
| 163 |
+
self._world_size = self.accelerator.num_processes
|
| 164 |
+
else:
|
| 165 |
+
self.model.to(self._device)
|
| 166 |
+
|
| 167 |
+
self.accelerator = accelerator
|
| 168 |
+
|
| 169 |
+
@property
|
| 170 |
+
def model(self):
|
| 171 |
+
return self.accelerator.unwrap_model(self._model) if hasattr(self, "accelerator") else self._model
|
| 172 |
+
|
| 173 |
+
@property
|
| 174 |
+
def eot_token_id(self):
|
| 175 |
+
return self.tokenizer.eos_token_id
|
| 176 |
+
|
| 177 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 178 |
+
res = []
|
| 179 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 180 |
+
|
| 181 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 182 |
+
continuation = doc_to_target if isinstance(doc_to_target, str) else doc_to_target(self.task_dict[task][split][doc_id])
|
| 183 |
+
visuals = self.flatten([doc_to_visual(self.task_dict[task][split][doc_id])])
|
| 184 |
+
|
| 185 |
+
query = []
|
| 186 |
+
visual_paths = []
|
| 187 |
+
for visual in visuals:
|
| 188 |
+
name = uuid.uuid4().hex.upper()[0:6]
|
| 189 |
+
visual_path = f"/tmp/{name}.png"
|
| 190 |
+
visual.save(visual_path)
|
| 191 |
+
visual_paths.append(visual_path)
|
| 192 |
+
query.append({"image": visual_path})
|
| 193 |
+
|
| 194 |
+
context_query = query.copy()
|
| 195 |
+
context_query.append({"text": contexts})
|
| 196 |
+
query.append({"text": contexts + continuation})
|
| 197 |
+
|
| 198 |
+
context_query = self.tokenizer.from_list_format(context_query)
|
| 199 |
+
query = self.tokenizer.from_list_format(query)
|
| 200 |
+
|
| 201 |
+
_, context_tokens = make_context(
|
| 202 |
+
self.tokenizer, context_query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format
|
| 203 |
+
)
|
| 204 |
+
context_tokens = torch.tensor([context_tokens])
|
| 205 |
+
|
| 206 |
+
_, continuation_tokens = make_context(self.tokenizer, query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format)
|
| 207 |
+
continuation_tokens = torch.tensor([continuation_tokens]).to(self.model.device)
|
| 208 |
+
attn_mask = torch.ones_like(continuation_tokens).to(self.model.device)
|
| 209 |
+
labels = continuation_tokens.clone().to(self.model.device)
|
| 210 |
+
labels[:, : context_tokens.shape[1]] = -100
|
| 211 |
+
|
| 212 |
+
with torch.inference_mode():
|
| 213 |
+
outputs = self.model(input_ids=continuation_tokens, labels=labels, attention_mask=attn_mask)
|
| 214 |
+
|
| 215 |
+
loss = outputs.loss
|
| 216 |
+
logits = outputs["logits"]
|
| 217 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 218 |
+
cont_toks = continuation_tokens[:, context_tokens.shape[1] :]
|
| 219 |
+
greedy_tokens = greedy_tokens[:, context_tokens.shape[1] : continuation_tokens.shape[1]]
|
| 220 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 221 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 222 |
+
pbar.update(1)
|
| 223 |
+
|
| 224 |
+
pbar.close()
|
| 225 |
+
return res
|
| 226 |
+
|
| 227 |
+
@staticmethod
|
| 228 |
+
def flatten(input_list):
|
| 229 |
+
return [item for sublist in input_list for item in sublist]
|
| 230 |
+
|
| 231 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 232 |
+
res = []
|
| 233 |
+
|
| 234 |
+
def _collate(x):
|
| 235 |
+
toks = self.tokenizer.encode(x[0])
|
| 236 |
+
return -len(toks), x[0]
|
| 237 |
+
|
| 238 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 239 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 240 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 241 |
+
|
| 242 |
+
for chunk in chunks:
|
| 243 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 244 |
+
task = task[0]
|
| 245 |
+
split = split[0]
|
| 246 |
+
visuals = self.flatten([doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id])
|
| 247 |
+
|
| 248 |
+
visual_paths = []
|
| 249 |
+
for visual in visuals:
|
| 250 |
+
name = uuid.uuid4().hex.upper()[0:6]
|
| 251 |
+
visual_path = f"/xpfs/public/gezhang/zk/lmms-eval/lmms_eval/tmp/{name}.png"
|
| 252 |
+
visual.save(visual_path)
|
| 253 |
+
visual_paths.append(visual_path)
|
| 254 |
+
|
| 255 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 256 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 257 |
+
|
| 258 |
+
if "until" in gen_kwargs:
|
| 259 |
+
until = gen_kwargs.pop("until")
|
| 260 |
+
if isinstance(until, str):
|
| 261 |
+
until = [until]
|
| 262 |
+
elif not isinstance(until, list):
|
| 263 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 264 |
+
|
| 265 |
+
gen_kwargs.setdefault("image_sizes", [visuals[0].size] if visuals else None)
|
| 266 |
+
gen_kwargs.setdefault("max_new_tokens", 1024)
|
| 267 |
+
gen_kwargs.setdefault("temperature", 0)
|
| 268 |
+
gen_kwargs.setdefault("top_p", None)
|
| 269 |
+
gen_kwargs.setdefault("num_beams", 1)
|
| 270 |
+
|
| 271 |
+
until.append("<|eot_id|>")
|
| 272 |
+
|
| 273 |
+
image = Image.open(visual_paths[0]).convert("RGB")
|
| 274 |
+
question = contexts[0]
|
| 275 |
+
|
| 276 |
+
input_ids, image_tensor, image_sizes, prompt = process(image, question, self.tokenizer, self.image_processor, self.model.config, self.conv_mode)
|
| 277 |
+
input_ids = input_ids.to(device=self.model.device, non_blocking=True)
|
| 278 |
+
|
| 279 |
+
with torch.inference_mode():
|
| 280 |
+
output_ids = self.model.generate(
|
| 281 |
+
input_ids,
|
| 282 |
+
images=image_tensor,
|
| 283 |
+
image_sizes=image_sizes,
|
| 284 |
+
do_sample=gen_kwargs["temperature"] > 0,
|
| 285 |
+
temperature=gen_kwargs["temperature"],
|
| 286 |
+
num_beams=gen_kwargs["num_beams"],
|
| 287 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 288 |
+
use_cache=True,
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
text_outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
|
| 292 |
+
|
| 293 |
+
for term in until:
|
| 294 |
+
if term:
|
| 295 |
+
text_outputs = text_outputs.split(term)[0]
|
| 296 |
+
|
| 297 |
+
print(text_outputs)
|
| 298 |
+
res.append(text_outputs)
|
| 299 |
+
|
| 300 |
+
for visual_path in visual_paths:
|
| 301 |
+
try:
|
| 302 |
+
os.remove(visual_path)
|
| 303 |
+
except OSError:
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
pbar.update(1)
|
| 307 |
+
|
| 308 |
+
res = re_ords.get_original(res)
|
| 309 |
+
pbar.close()
|
| 310 |
+
return res
|
| 311 |
+
|
| 312 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 313 |
+
raise NotImplementedError("TODO: Implement multi-round generation for Cambrian")
|
LaViDa/eval/lmms_eval/models/claude.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
from copy import deepcopy
|
| 6 |
+
from io import BytesIO
|
| 7 |
+
from typing import List, Tuple, Union
|
| 8 |
+
|
| 9 |
+
from accelerate import Accelerator, DistributedType
|
| 10 |
+
from PIL import Image
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
|
| 13 |
+
from lmms_eval.api.instance import Instance
|
| 14 |
+
from lmms_eval.api.model import lmms
|
| 15 |
+
from lmms_eval.api.registry import register_model
|
| 16 |
+
|
| 17 |
+
NUM_SECONDS_TO_SLEEP = 5
|
| 18 |
+
|
| 19 |
+
from loguru import logger
|
| 20 |
+
|
| 21 |
+
eval_logger = logger
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
import anthropic
|
| 25 |
+
import numpy as np
|
| 26 |
+
from decord import VideoReader, cpu
|
| 27 |
+
except Exception as e:
|
| 28 |
+
eval_logger.warning(f"Error importing claude: {e}")
|
| 29 |
+
|
| 30 |
+
API_URL = os.getenv("ANTHROPIC_API_URL", "https://api.anthropic.com/v1/complete")
|
| 31 |
+
API_KEY = os.getenv("ANTHROPIC_API_KEY", "YOUR_API_KEY")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@register_model("claude")
|
| 35 |
+
class Claude(lmms):
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
model_version: str = "claude-3-opus-20240229",
|
| 39 |
+
image_token: str = "<image>", # Use to separate interleaved image and text
|
| 40 |
+
system_prompt: str = "", # Whether you want some special system prompt here
|
| 41 |
+
modality: str = "image",
|
| 42 |
+
max_frames_num: int = 10,
|
| 43 |
+
continual_mode: bool = False,
|
| 44 |
+
response_persistent_folder: str = None,
|
| 45 |
+
**kwargs,
|
| 46 |
+
) -> None:
|
| 47 |
+
super().__init__()
|
| 48 |
+
self.model_version = model_version
|
| 49 |
+
self.image_token = image_token
|
| 50 |
+
self.system_prompt = system_prompt
|
| 51 |
+
self.modality = modality
|
| 52 |
+
self.max_frames_num = max_frames_num
|
| 53 |
+
|
| 54 |
+
self.continual_mode = continual_mode
|
| 55 |
+
if self.continual_mode:
|
| 56 |
+
if response_persistent_folder is None:
|
| 57 |
+
raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
|
| 58 |
+
|
| 59 |
+
os.makedirs(response_persistent_folder, exist_ok=True)
|
| 60 |
+
self.response_persistent_folder = response_persistent_folder
|
| 61 |
+
self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
|
| 62 |
+
|
| 63 |
+
if os.path.exists(self.response_persistent_file):
|
| 64 |
+
with open(self.response_persistent_file, "r") as f:
|
| 65 |
+
self.response_cache = json.load(f)
|
| 66 |
+
self.cache_mode = "resume"
|
| 67 |
+
else:
|
| 68 |
+
self.response_cache = {}
|
| 69 |
+
self.cache_mode = "start"
|
| 70 |
+
|
| 71 |
+
accelerator = Accelerator()
|
| 72 |
+
if accelerator.num_processes > 1:
|
| 73 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 74 |
+
self.accelerator = accelerator
|
| 75 |
+
if self.accelerator.is_local_main_process:
|
| 76 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 77 |
+
self._rank = self.accelerator.local_process_index
|
| 78 |
+
self._world_size = self.accelerator.num_processes
|
| 79 |
+
else:
|
| 80 |
+
self.accelerator = accelerator
|
| 81 |
+
self._rank = self.accelerator.local_process_index
|
| 82 |
+
self._world_size = self.accelerator.num_processes
|
| 83 |
+
|
| 84 |
+
self.device = self.accelerator.device
|
| 85 |
+
|
| 86 |
+
def encode_image(self, image):
|
| 87 |
+
output_buffer = BytesIO()
|
| 88 |
+
image.save(output_buffer, format="JPEG")
|
| 89 |
+
byte_data = output_buffer.getvalue()
|
| 90 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 91 |
+
return base64_str
|
| 92 |
+
|
| 93 |
+
def flatten(self, input):
|
| 94 |
+
new_list = []
|
| 95 |
+
for i in input:
|
| 96 |
+
for j in i:
|
| 97 |
+
new_list.append(j)
|
| 98 |
+
return new_list
|
| 99 |
+
|
| 100 |
+
def get_image_size(self, image):
|
| 101 |
+
# Create a BytesIO object to store the image bytes
|
| 102 |
+
img_byte_array = BytesIO()
|
| 103 |
+
|
| 104 |
+
# Save the image to the BytesIO object
|
| 105 |
+
image.save(img_byte_array, format="PNG")
|
| 106 |
+
|
| 107 |
+
# Get the size of the BytesIO object
|
| 108 |
+
img_size = img_byte_array.tell()
|
| 109 |
+
|
| 110 |
+
return img_size
|
| 111 |
+
|
| 112 |
+
# The max file size is 5MB for claude
|
| 113 |
+
def shrink_image_to_file_size(self, img: Image, max_file_size=4838990) -> Image:
|
| 114 |
+
# Get the current size of the image
|
| 115 |
+
original_size = self.get_image_size(img)
|
| 116 |
+
|
| 117 |
+
# If the image size is already smaller than the desired size, return
|
| 118 |
+
if original_size <= max_file_size:
|
| 119 |
+
return img
|
| 120 |
+
|
| 121 |
+
# Calculate the ratio to shrink the image
|
| 122 |
+
# Somehow I found out sqrt ratio is not enough to shrink the image
|
| 123 |
+
# below threshold, so I guess we do more
|
| 124 |
+
shrink_ratio = min(0.9, max_file_size / original_size)
|
| 125 |
+
|
| 126 |
+
# Resize the image with the calculated ratio
|
| 127 |
+
new_width = int(img.width * shrink_ratio)
|
| 128 |
+
new_height = int(img.height * shrink_ratio)
|
| 129 |
+
img = img.resize((new_width, new_height), Image.LANCZOS)
|
| 130 |
+
|
| 131 |
+
return self.shrink_image_to_file_size(img, max_file_size)
|
| 132 |
+
|
| 133 |
+
def encode_video(self, video_path):
|
| 134 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 135 |
+
total_frame_num = len(vr)
|
| 136 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, self.max_frames_num, dtype=int)
|
| 137 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 138 |
+
frames = vr.get_batch(frame_idx).asnumpy()
|
| 139 |
+
|
| 140 |
+
base64_frames = []
|
| 141 |
+
for frame in frames:
|
| 142 |
+
img = Image.fromarray(frame)
|
| 143 |
+
output_buffer = BytesIO()
|
| 144 |
+
img.save(output_buffer, format="JPEG")
|
| 145 |
+
byte_data = output_buffer.getvalue()
|
| 146 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 147 |
+
base64_frames.append(f"{base64_str}")
|
| 148 |
+
|
| 149 |
+
return base64_frames
|
| 150 |
+
|
| 151 |
+
def generate_until(self, requests) -> List[str]:
|
| 152 |
+
client = anthropic.Anthropic()
|
| 153 |
+
|
| 154 |
+
res = []
|
| 155 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 156 |
+
|
| 157 |
+
empty_image_block = {
|
| 158 |
+
"type": "image",
|
| 159 |
+
"source": {
|
| 160 |
+
"type": "base64",
|
| 161 |
+
"media_type": "image/jpeg",
|
| 162 |
+
},
|
| 163 |
+
}
|
| 164 |
+
empty_text_block = {"type": "text"}
|
| 165 |
+
empty_messages = [
|
| 166 |
+
{
|
| 167 |
+
"role": "user",
|
| 168 |
+
"content": [],
|
| 169 |
+
}
|
| 170 |
+
]
|
| 171 |
+
|
| 172 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 173 |
+
###################### CONTINUAL MODE ######################
|
| 174 |
+
if self.continual_mode is True and self.cache_mode == "resume":
|
| 175 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 176 |
+
if doc_uuid in self.response_cache:
|
| 177 |
+
response_text = self.response_cache[doc_uuid]
|
| 178 |
+
if response_text:
|
| 179 |
+
res.append(response_text)
|
| 180 |
+
pbar.update(1)
|
| 181 |
+
continue
|
| 182 |
+
|
| 183 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 184 |
+
visuals = self.flatten(visuals)
|
| 185 |
+
imgs = []
|
| 186 |
+
for visual in visuals:
|
| 187 |
+
if isinstance(visual, str) and os.path.exists(visual): # Assuming visual is a path to a video
|
| 188 |
+
visual = self.encode_video(visual)
|
| 189 |
+
for img in visual:
|
| 190 |
+
imgs.append(img)
|
| 191 |
+
else:
|
| 192 |
+
visual = self.shrink_image_to_file_size(visual)
|
| 193 |
+
img = self.encode_image(visual)
|
| 194 |
+
imgs.append(img)
|
| 195 |
+
|
| 196 |
+
messages = deepcopy(empty_messages)
|
| 197 |
+
|
| 198 |
+
if self.image_token not in contexts:
|
| 199 |
+
for img in imgs:
|
| 200 |
+
image_block = deepcopy(empty_image_block)
|
| 201 |
+
image_block["source"]["data"] = img
|
| 202 |
+
messages[0]["content"].append(image_block)
|
| 203 |
+
text_block = deepcopy(empty_text_block)
|
| 204 |
+
text_block["text"] = contexts
|
| 205 |
+
messages[0]["content"].append(text_block)
|
| 206 |
+
else:
|
| 207 |
+
contexts = contexts.split(self.image_token)
|
| 208 |
+
for idx, img in enumerate(imgs):
|
| 209 |
+
text_block = deepcopy(empty_text_block)
|
| 210 |
+
image_block = deepcopy(empty_image_block)
|
| 211 |
+
text_block["text"] = contexts
|
| 212 |
+
messages[0]["content"].append(text_block)
|
| 213 |
+
image_block["source"]["data"] = img
|
| 214 |
+
messages[0]["content"].append(image_block)
|
| 215 |
+
|
| 216 |
+
# If n image tokens are in the contexts
|
| 217 |
+
# contexts will be splitted into n+1 chunks
|
| 218 |
+
# Manually add it into the messages
|
| 219 |
+
text_block = deepcopy(empty_text_block)
|
| 220 |
+
text_block["text"] = contexts
|
| 221 |
+
messages["content"].append(text_block)
|
| 222 |
+
|
| 223 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 224 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 225 |
+
if gen_kwargs["max_new_tokens"] > 4096:
|
| 226 |
+
gen_kwargs["max_new_tokens"] = 4096
|
| 227 |
+
if "temperature" not in gen_kwargs:
|
| 228 |
+
gen_kwargs["temperature"] = 0
|
| 229 |
+
if "top_p" not in gen_kwargs or gen_kwargs["top_p"] is None:
|
| 230 |
+
gen_kwargs["top_p"] = 1
|
| 231 |
+
if "num_beams" not in gen_kwargs:
|
| 232 |
+
gen_kwargs["num_beams"] = 1
|
| 233 |
+
|
| 234 |
+
for attempt in range(5):
|
| 235 |
+
retry_flag = True
|
| 236 |
+
try:
|
| 237 |
+
message = client.messages.create(model=self.model_version, max_tokens=gen_kwargs["max_new_tokens"], system=self.system_prompt, temperature=gen_kwargs["temperature"], top_p=gen_kwargs["top_p"], messages=messages)
|
| 238 |
+
retry_flag = False
|
| 239 |
+
except Exception as e:
|
| 240 |
+
eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
|
| 241 |
+
if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt
|
| 242 |
+
time.sleep(NUM_SECONDS_TO_SLEEP)
|
| 243 |
+
else: # If this was the last attempt, log and return empty
|
| 244 |
+
eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
|
| 245 |
+
res.append("")
|
| 246 |
+
pbar.update(1)
|
| 247 |
+
continue
|
| 248 |
+
if not retry_flag:
|
| 249 |
+
break
|
| 250 |
+
eval_logger.info("Retrying...")
|
| 251 |
+
|
| 252 |
+
response_text = message.content[0].text
|
| 253 |
+
res.append(message.content[0].text)
|
| 254 |
+
pbar.update(1)
|
| 255 |
+
|
| 256 |
+
###################### CONTINUAL MODE ######################
|
| 257 |
+
if self.continual_mode is True: # Cache the response
|
| 258 |
+
response_text = message.content[0].text
|
| 259 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 260 |
+
self.response_cache[doc_uuid] = response_text
|
| 261 |
+
with open(self.response_persistent_file, "w") as f:
|
| 262 |
+
json.dump(self.response_cache, f, indent=4)
|
| 263 |
+
|
| 264 |
+
pbar.close()
|
| 265 |
+
|
| 266 |
+
return res
|
| 267 |
+
|
| 268 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 269 |
+
assert False, "Not supported for claude"
|
| 270 |
+
|
| 271 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 272 |
+
raise NotImplementedError("TODO: Implement multi-round generation for Claude")
|
LaViDa/eval/lmms_eval/models/cogvlm2.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from accelerate import Accelerator, DistributedType
|
| 6 |
+
from accelerate.state import AcceleratorState
|
| 7 |
+
from tqdm import tqdm
|
| 8 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 9 |
+
|
| 10 |
+
from lmms_eval import utils
|
| 11 |
+
from lmms_eval.api.instance import Instance
|
| 12 |
+
from lmms_eval.api.model import lmms
|
| 13 |
+
from lmms_eval.api.registry import register_model
|
| 14 |
+
|
| 15 |
+
warnings.filterwarnings("ignore")
|
| 16 |
+
|
| 17 |
+
from loguru import logger as eval_logger
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@register_model("cogvlm2")
|
| 21 |
+
class CogVLM2(lmms):
|
| 22 |
+
"""
|
| 23 |
+
CogVLM2 Model
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
def __init__(
|
| 27 |
+
self,
|
| 28 |
+
pretrained: str = "THUDM/cogvlm2-llama3-chinese-chat-19B",
|
| 29 |
+
device: Optional[str] = "cuda",
|
| 30 |
+
dtype: Optional[Union[str, torch.dtype]] = torch.bfloat16,
|
| 31 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 32 |
+
trust_remote_code: Optional[bool] = True,
|
| 33 |
+
**kwargs,
|
| 34 |
+
) -> None:
|
| 35 |
+
super().__init__()
|
| 36 |
+
# Do not use kwargs for now
|
| 37 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 38 |
+
|
| 39 |
+
accelerator = Accelerator()
|
| 40 |
+
if accelerator.num_processes > 1:
|
| 41 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 42 |
+
else:
|
| 43 |
+
self._device = device
|
| 44 |
+
self.dtype = dtype
|
| 45 |
+
self._model = AutoModelForCausalLM.from_pretrained(pretrained, trust_remote_code=trust_remote_code, torch_dtype=dtype, device_map=self._device)
|
| 46 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code)
|
| 47 |
+
self._config = self._model.config
|
| 48 |
+
self.model.eval()
|
| 49 |
+
self.model.tie_weights()
|
| 50 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 51 |
+
if accelerator.num_processes > 1:
|
| 52 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 53 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 54 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 55 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 56 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 57 |
+
kwargs = {
|
| 58 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 59 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 60 |
+
}
|
| 61 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 62 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 63 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 64 |
+
self._model = accelerator.prepare(self.model)
|
| 65 |
+
else:
|
| 66 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 67 |
+
self.accelerator = accelerator
|
| 68 |
+
if self.accelerator.is_local_main_process:
|
| 69 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 70 |
+
self._rank = self.accelerator.local_process_index
|
| 71 |
+
self._world_size = self.accelerator.num_processes
|
| 72 |
+
else:
|
| 73 |
+
self._rank = 0
|
| 74 |
+
self._world_size = 1
|
| 75 |
+
|
| 76 |
+
@property
|
| 77 |
+
def config(self):
|
| 78 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 79 |
+
return self._config
|
| 80 |
+
|
| 81 |
+
@property
|
| 82 |
+
def tokenizer(self):
|
| 83 |
+
return self._tokenizer
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def model(self):
|
| 87 |
+
# returns the model, unwrapping it if using Accelerate
|
| 88 |
+
if hasattr(self, "accelerator"):
|
| 89 |
+
return self.accelerator.unwrap_model(self._model)
|
| 90 |
+
else:
|
| 91 |
+
return self._model
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def eot_token_id(self):
|
| 95 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 96 |
+
return self.tokenizer.eos_token_id
|
| 97 |
+
|
| 98 |
+
@property
|
| 99 |
+
def max_length(self):
|
| 100 |
+
return self._max_length
|
| 101 |
+
|
| 102 |
+
@property
|
| 103 |
+
def batch_size(self):
|
| 104 |
+
return self.batch_size_per_gpu
|
| 105 |
+
|
| 106 |
+
@property
|
| 107 |
+
def device(self):
|
| 108 |
+
return self._device
|
| 109 |
+
|
| 110 |
+
@property
|
| 111 |
+
def rank(self):
|
| 112 |
+
return self._rank
|
| 113 |
+
|
| 114 |
+
@property
|
| 115 |
+
def world_size(self):
|
| 116 |
+
return self._world_size
|
| 117 |
+
|
| 118 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 119 |
+
""" """
|
| 120 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 121 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 122 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 123 |
+
if left_truncate_len:
|
| 124 |
+
encoding = encoding[-left_truncate_len:]
|
| 125 |
+
return encoding
|
| 126 |
+
|
| 127 |
+
def tok_decode(self, tokens):
|
| 128 |
+
return self.tokenizer.decode(tokens)
|
| 129 |
+
|
| 130 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 131 |
+
# TODO
|
| 132 |
+
assert False, "We have not implemented this function for CogVLM2 yet"
|
| 133 |
+
|
| 134 |
+
def flatten(self, input):
|
| 135 |
+
new_list = []
|
| 136 |
+
for i in input:
|
| 137 |
+
for j in i:
|
| 138 |
+
new_list.append(j)
|
| 139 |
+
return new_list
|
| 140 |
+
|
| 141 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 142 |
+
res = []
|
| 143 |
+
|
| 144 |
+
def _collate(x):
|
| 145 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 146 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 147 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 148 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 149 |
+
# automatic adaptive batches much much easier to implement
|
| 150 |
+
# - any OOMs will happen right away rather than near the end
|
| 151 |
+
toks = self.tok_encode(x[0])
|
| 152 |
+
return -len(toks), x[0]
|
| 153 |
+
|
| 154 |
+
# we group requests by their generation_kwargs,
|
| 155 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 156 |
+
# in the same batch.
|
| 157 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 158 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 159 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 160 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 161 |
+
for chunk in chunks:
|
| 162 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 163 |
+
task = task[0]
|
| 164 |
+
split = split[0]
|
| 165 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 166 |
+
visuals = self.flatten(visuals)
|
| 167 |
+
# we assume all gen kwargs in the batch are the same
|
| 168 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 169 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 170 |
+
|
| 171 |
+
# Set default values for until and max_new_tokens
|
| 172 |
+
until = [self.tok_decode(self.eot_token_id)]
|
| 173 |
+
|
| 174 |
+
# Update values from gen_kwargs if present
|
| 175 |
+
if "until" in gen_kwargs:
|
| 176 |
+
until = gen_kwargs.pop("until")
|
| 177 |
+
if isinstance(until, str):
|
| 178 |
+
until = [until]
|
| 179 |
+
elif not isinstance(until, list):
|
| 180 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 181 |
+
assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
|
| 182 |
+
assert len(visuals) == 1, "CogVLM2 interface does not support bn_image > 1 for now"
|
| 183 |
+
context = contexts[0]
|
| 184 |
+
if "<image>" in context:
|
| 185 |
+
context = context.replace("<image>", "")
|
| 186 |
+
|
| 187 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 188 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 189 |
+
if "temperature" not in gen_kwargs:
|
| 190 |
+
gen_kwargs["temperature"] = 0
|
| 191 |
+
if "top_p" not in gen_kwargs:
|
| 192 |
+
gen_kwargs["top_p"] = None
|
| 193 |
+
if "num_beams" not in gen_kwargs:
|
| 194 |
+
gen_kwargs["num_beams"] = 1
|
| 195 |
+
|
| 196 |
+
image = visuals[0]
|
| 197 |
+
input_by_model = self.model.build_conversation_input_ids(self.tokenizer, query=context, history=[], images=[image])
|
| 198 |
+
|
| 199 |
+
inputs = {
|
| 200 |
+
"input_ids": input_by_model["input_ids"].unsqueeze(0).to(self.device),
|
| 201 |
+
"token_type_ids": input_by_model["token_type_ids"].unsqueeze(0).to(self.device),
|
| 202 |
+
"attention_mask": input_by_model["attention_mask"].unsqueeze(0).to(self.device),
|
| 203 |
+
"images": [[input_by_model["images"][0].to(self.device).to(self.dtype)]],
|
| 204 |
+
}
|
| 205 |
+
if "cross_images" in input_by_model and input_by_model["cross_images"]:
|
| 206 |
+
inputs["cross_images"] = [[input_by_model["cross_images"][0].to(self.device).to(self.dtype)]]
|
| 207 |
+
|
| 208 |
+
try:
|
| 209 |
+
outputs = self.model.generate(**inputs, **gen_kwargs)
|
| 210 |
+
outputs = outputs[:, inputs["input_ids"].shape[1] :]
|
| 211 |
+
response = self.tokenizer.decode(outputs[0])
|
| 212 |
+
response = response.split("</s>")[0]
|
| 213 |
+
response = response.split("<|end_of_text|>")[0]
|
| 214 |
+
|
| 215 |
+
context = [{"role": "user", "content": context}, {"role": "assistant", "content": response}]
|
| 216 |
+
except Exception as e:
|
| 217 |
+
eval_logger.error(f"Error {e} in generating")
|
| 218 |
+
cont = ""
|
| 219 |
+
res.append(response)
|
| 220 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response)
|
| 221 |
+
pbar.update(1)
|
| 222 |
+
# reorder this group of results back to original unsorted form
|
| 223 |
+
res = re_ords.get_original(res)
|
| 224 |
+
|
| 225 |
+
pbar.close()
|
| 226 |
+
return res
|
| 227 |
+
|
| 228 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 229 |
+
raise NotImplementedError("TODO: Implement multi-round generation for CogVLM2")
|
LaViDa/eval/lmms_eval/models/from_log.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
from typing import List, Tuple
|
| 6 |
+
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from loguru import logger as eval_logger
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
|
| 11 |
+
from lmms_eval.api.instance import Instance
|
| 12 |
+
from lmms_eval.api.model import lmms
|
| 13 |
+
from lmms_eval.api.registry import register_model
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@register_model("from_log")
|
| 17 |
+
class FromLog(lmms):
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
logs: str = "logs",
|
| 21 |
+
model_name: str = None,
|
| 22 |
+
model_args: str = None,
|
| 23 |
+
have_limits: bool = False,
|
| 24 |
+
**kwargs,
|
| 25 |
+
) -> None:
|
| 26 |
+
super().__init__()
|
| 27 |
+
|
| 28 |
+
self.logs = {}
|
| 29 |
+
|
| 30 |
+
log_folders = logs.split(",")
|
| 31 |
+
|
| 32 |
+
def matched_model(_model_args):
|
| 33 |
+
if model_name and model_name != _model_args["model"]:
|
| 34 |
+
return False
|
| 35 |
+
|
| 36 |
+
if model_args:
|
| 37 |
+
_model_args_list = model_args.split(",")
|
| 38 |
+
|
| 39 |
+
for _model_arg in _model_args_list:
|
| 40 |
+
if _model_arg not in _model_args["model_args"]:
|
| 41 |
+
return False
|
| 42 |
+
|
| 43 |
+
if not have_limits and _model_args["limit"] is not None:
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
return True
|
| 47 |
+
|
| 48 |
+
for log_folder in log_folders:
|
| 49 |
+
for root, dirs, files in os.walk(log_folder):
|
| 50 |
+
for file in files:
|
| 51 |
+
if file.endswith(".json"):
|
| 52 |
+
try:
|
| 53 |
+
log_file = os.path.join(root, file)
|
| 54 |
+
|
| 55 |
+
with open(log_file, "r") as f:
|
| 56 |
+
log_data = json.load(f)
|
| 57 |
+
|
| 58 |
+
# check if model is matched
|
| 59 |
+
_model_args = log_data["args"]
|
| 60 |
+
if not matched_model(_model_args):
|
| 61 |
+
raise Exception("Model not matched")
|
| 62 |
+
|
| 63 |
+
# load logs
|
| 64 |
+
logs = {}
|
| 65 |
+
for data in log_data["logs"]:
|
| 66 |
+
id = data["doc_id"]
|
| 67 |
+
response = data["resps"][0]
|
| 68 |
+
logs[id] = response
|
| 69 |
+
|
| 70 |
+
task = log_data["model_configs"]["task"]
|
| 71 |
+
|
| 72 |
+
pattern = re.compile(r"\d{4}_\d{4}")
|
| 73 |
+
|
| 74 |
+
if "time" in log_data:
|
| 75 |
+
log_time = log_data["time"]
|
| 76 |
+
elif pattern.search(os.path.abspath(log_file)):
|
| 77 |
+
log_time = pattern.findall(os.path.abspath(log_file))[-1]
|
| 78 |
+
else:
|
| 79 |
+
log_time = "unknown"
|
| 80 |
+
|
| 81 |
+
if task not in self.logs or (self.logs[task]["time"] == "unknown" or datetime.strptime(log_time, "%m%d_%H%M") > datetime.strptime(self.logs[task]["time"], "%m%d_%H%M")):
|
| 82 |
+
self.logs[task] = {"time": log_time, "logs": logs}
|
| 83 |
+
|
| 84 |
+
except Exception as e:
|
| 85 |
+
pass
|
| 86 |
+
|
| 87 |
+
accelerator = Accelerator()
|
| 88 |
+
if accelerator.num_processes > 1:
|
| 89 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 90 |
+
self.accelerator = accelerator
|
| 91 |
+
if self.accelerator.is_local_main_process:
|
| 92 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 93 |
+
self._rank = self.accelerator.local_process_index
|
| 94 |
+
self._world_size = self.accelerator.num_processes
|
| 95 |
+
else:
|
| 96 |
+
self.accelerator = accelerator
|
| 97 |
+
self._rank = self.accelerator.local_process_index
|
| 98 |
+
self._world_size = self.accelerator.num_processes
|
| 99 |
+
|
| 100 |
+
self.device = self.accelerator.device
|
| 101 |
+
|
| 102 |
+
def generate_until(self, requests) -> List[str]:
|
| 103 |
+
res = []
|
| 104 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 105 |
+
|
| 106 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 107 |
+
response = self.logs[task]["logs"][doc_id]
|
| 108 |
+
res.append(response[0])
|
| 109 |
+
pbar.update(1)
|
| 110 |
+
|
| 111 |
+
pbar.close()
|
| 112 |
+
return res
|
| 113 |
+
|
| 114 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 115 |
+
# TODO
|
| 116 |
+
assert False, "not support"
|
| 117 |
+
|
| 118 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 119 |
+
return generate_until(self, requests)
|
LaViDa/eval/lmms_eval/models/fuyu.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
|
| 3 |
+
warnings.simplefilter("ignore", category=DeprecationWarning)
|
| 4 |
+
warnings.filterwarnings("ignore")
|
| 5 |
+
|
| 6 |
+
from typing import List, Optional, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from accelerate import Accelerator, DistributedType
|
| 10 |
+
from accelerate.state import AcceleratorState
|
| 11 |
+
from loguru import logger as eval_logger
|
| 12 |
+
from PIL import Image
|
| 13 |
+
from tqdm import tqdm
|
| 14 |
+
from transformers import (
|
| 15 |
+
AutoTokenizer,
|
| 16 |
+
FuyuForCausalLM,
|
| 17 |
+
FuyuImageProcessor,
|
| 18 |
+
FuyuProcessor,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
from lmms_eval import utils
|
| 22 |
+
from lmms_eval.api.instance import Instance
|
| 23 |
+
from lmms_eval.api.model import lmms
|
| 24 |
+
from lmms_eval.api.registry import register_model
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@register_model("fuyu")
|
| 28 |
+
class Fuyu(lmms):
|
| 29 |
+
"""
|
| 30 |
+
Fuyu Model
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
def __init__(
|
| 34 |
+
self,
|
| 35 |
+
pretrained: str = "adept/fuyu-8b",
|
| 36 |
+
device: Optional[str] = "cuda",
|
| 37 |
+
max_new_tokens: int = 256,
|
| 38 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 39 |
+
**kwargs,
|
| 40 |
+
) -> None:
|
| 41 |
+
super().__init__()
|
| 42 |
+
# Do not use kwargs for now
|
| 43 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 44 |
+
|
| 45 |
+
accelerator = Accelerator()
|
| 46 |
+
if accelerator.num_processes > 1:
|
| 47 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 48 |
+
else:
|
| 49 |
+
self._device = device
|
| 50 |
+
|
| 51 |
+
self._model = FuyuForCausalLM.from_pretrained(pretrained, torch_dtype=torch.bfloat16, device_map=self.device)
|
| 52 |
+
self.model.eval()
|
| 53 |
+
self.model.tie_weights()
|
| 54 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
|
| 55 |
+
self._config = self.model.config
|
| 56 |
+
|
| 57 |
+
self.image_processor = FuyuImageProcessor()
|
| 58 |
+
self.processor = FuyuProcessor(image_processor=self.image_processor, tokenizer=self.tokenizer)
|
| 59 |
+
self.max_new_tokens = max_new_tokens
|
| 60 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 61 |
+
accelerator = Accelerator()
|
| 62 |
+
if accelerator.num_processes > 1:
|
| 63 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 64 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 65 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 66 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 67 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 68 |
+
kwargs = {
|
| 69 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 70 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 71 |
+
}
|
| 72 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 73 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 74 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 75 |
+
self._model = accelerator.prepare(self.model)
|
| 76 |
+
else:
|
| 77 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 78 |
+
self.accelerator = accelerator
|
| 79 |
+
if self.accelerator.is_local_main_process:
|
| 80 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 81 |
+
self._rank = self.accelerator.local_process_index
|
| 82 |
+
self._world_size = self.accelerator.num_processes
|
| 83 |
+
else:
|
| 84 |
+
self.model.to(self._device)
|
| 85 |
+
self._rank = 0
|
| 86 |
+
self._world_size = 1
|
| 87 |
+
|
| 88 |
+
"""if accelerator.num_processes > 1:
|
| 89 |
+
assert accelerator.distributed_type in [
|
| 90 |
+
DistributedType.FSDP,
|
| 91 |
+
DistributedType.MULTI_GPU,
|
| 92 |
+
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 93 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 94 |
+
self._model = accelerator.prepare(self.model)
|
| 95 |
+
else:
|
| 96 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 97 |
+
self.accelerator = accelerator
|
| 98 |
+
if self.accelerator.is_local_main_process:
|
| 99 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 100 |
+
self._rank = self.accelerator.local_process_index
|
| 101 |
+
self._world_size = self.accelerator.num_processes"""
|
| 102 |
+
|
| 103 |
+
@property
|
| 104 |
+
def config(self):
|
| 105 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 106 |
+
return self._config
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def tokenizer(self):
|
| 110 |
+
return self._tokenizer
|
| 111 |
+
|
| 112 |
+
@property
|
| 113 |
+
def model(self):
|
| 114 |
+
# returns the model, unwrapping it if using Accelerate
|
| 115 |
+
if hasattr(self, "accelerator"):
|
| 116 |
+
return self.accelerator.unwrap_model(self._model)
|
| 117 |
+
else:
|
| 118 |
+
return self._model
|
| 119 |
+
|
| 120 |
+
@property
|
| 121 |
+
def eot_token_id(self):
|
| 122 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 123 |
+
return self.tokenizer.eos_token_id
|
| 124 |
+
|
| 125 |
+
@property
|
| 126 |
+
def max_length(self):
|
| 127 |
+
# Assuming max_length is the sum of max context tokens and max new tokens
|
| 128 |
+
return self.tokenizer.model_max_length
|
| 129 |
+
|
| 130 |
+
@property
|
| 131 |
+
def batch_size(self):
|
| 132 |
+
return self.batch_size_per_gpu
|
| 133 |
+
|
| 134 |
+
@property
|
| 135 |
+
def device(self):
|
| 136 |
+
return self._device
|
| 137 |
+
|
| 138 |
+
@property
|
| 139 |
+
def rank(self):
|
| 140 |
+
return self._rank
|
| 141 |
+
|
| 142 |
+
@property
|
| 143 |
+
def world_size(self):
|
| 144 |
+
return self._world_size
|
| 145 |
+
|
| 146 |
+
def flatten(self, input, only_get_first=False):
|
| 147 |
+
new_list = []
|
| 148 |
+
for i in input:
|
| 149 |
+
for j in i:
|
| 150 |
+
new_list.append(j)
|
| 151 |
+
if only_get_first:
|
| 152 |
+
break
|
| 153 |
+
return new_list
|
| 154 |
+
|
| 155 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 156 |
+
res = []
|
| 157 |
+
|
| 158 |
+
def _collate(x):
|
| 159 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 160 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 161 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 162 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 163 |
+
# automatic adaptive batches much much easier to implement
|
| 164 |
+
# - any OOMs will happen right away rather than near the end
|
| 165 |
+
toks = self.tok_encode(x[0])
|
| 166 |
+
return -len(toks), x[0]
|
| 167 |
+
|
| 168 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 169 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 170 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 171 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 172 |
+
|
| 173 |
+
for chunk in chunks:
|
| 174 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 175 |
+
task = task[0]
|
| 176 |
+
split = split[0]
|
| 177 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 178 |
+
visuals = self.flatten(visuals, only_get_first=True)
|
| 179 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 180 |
+
|
| 181 |
+
# if isinstance(visuals[0], list):
|
| 182 |
+
# visuals = [visuals[idx][0] for idx in range(len(visuals))] # get the first image in multi-image scenarios.
|
| 183 |
+
|
| 184 |
+
# assert len(contexts) == self.batch_size_per_gpu, f"Expected contexts batch size {self.batch_size_per_gpu}, got {len(contexts)}"
|
| 185 |
+
# assert len(visuals) == self.batch_size_per_gpu, f"Expected visuals batch size {self.batch_size_per_gpu}, got {len(visuals)}"
|
| 186 |
+
formatted_contexts = [f"{context}\n" for context in contexts]
|
| 187 |
+
model_inputs = self.processor(text=formatted_contexts, images=visuals, device=self.device)
|
| 188 |
+
for k, v in model_inputs.items():
|
| 189 |
+
model_inputs[k] = v.to(self.device, non_blocking=True) if isinstance(v, torch.Tensor) else [vv.to(self.device, non_blocking=True) for vv in v]
|
| 190 |
+
|
| 191 |
+
for index in range(len(model_inputs["image_patches"])):
|
| 192 |
+
model_inputs["image_patches"][index] = model_inputs["image_patches"][index].to(dtype=next(self.model.parameters()).dtype)
|
| 193 |
+
|
| 194 |
+
# preconfigure gen_kwargs with defaults
|
| 195 |
+
gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
|
| 196 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 197 |
+
gen_kwargs["max_new_tokens"] = 256
|
| 198 |
+
if "temperature" not in gen_kwargs:
|
| 199 |
+
gen_kwargs["temperature"] = 0
|
| 200 |
+
if "top_p" not in gen_kwargs:
|
| 201 |
+
gen_kwargs["top_p"] = None
|
| 202 |
+
if "num_beams" not in gen_kwargs:
|
| 203 |
+
gen_kwargs["num_beams"] = 1
|
| 204 |
+
# generation_output = self.model.generate(
|
| 205 |
+
# **model_inputs, temperature=gen_kwargs["temperature"], max_new_tokens=gen_kwargs["max_new_tokens"], top_p=gen_kwargs["top_p"], num_beams=gen_kwargs["num_beams"], pad_token_id=self.tokenizer.eos_token_id
|
| 206 |
+
# )
|
| 207 |
+
generation_output = self.model.generate(**model_inputs, max_new_tokens=gen_kwargs["max_new_tokens"], pad_token_id=self.tokenizer.eos_token_id)
|
| 208 |
+
generation_texts = self.processor.batch_decode(generation_output, skip_special_tokens=True)
|
| 209 |
+
response = [gen_text.split("\x04")[1].strip(" ").strip("\n") for gen_text in generation_texts]
|
| 210 |
+
res.extend(response)
|
| 211 |
+
pbar.update(1)
|
| 212 |
+
|
| 213 |
+
pbar.close()
|
| 214 |
+
return res
|
| 215 |
+
|
| 216 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 217 |
+
# TODO
|
| 218 |
+
res = []
|
| 219 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 220 |
+
|
| 221 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 222 |
+
# encode, pad, and truncate contexts for this batch
|
| 223 |
+
if type(doc_to_target) == str:
|
| 224 |
+
continuation = doc_to_target
|
| 225 |
+
else:
|
| 226 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 227 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 228 |
+
visuals = self.flatten(visuals)
|
| 229 |
+
formatted_contexts = [f"{contexts}\n"]
|
| 230 |
+
formatted_continuation = [f"{contexts}\n{continuation}"]
|
| 231 |
+
model_inputs = self.processor(text=formatted_continuation, images=visuals, device=self.device)
|
| 232 |
+
for k, v in model_inputs.items():
|
| 233 |
+
model_inputs[k] = v.to(self.device, non_blocking=True) if isinstance(v, torch.Tensor) else [vv.to(self.device, non_blocking=True) for vv in v]
|
| 234 |
+
|
| 235 |
+
for index in range(len(model_inputs["image_patches"])):
|
| 236 |
+
model_inputs["image_patches"][index] = model_inputs["image_patches"][index].to(dtype=next(self.model.parameters()).dtype)
|
| 237 |
+
|
| 238 |
+
labels = model_inputs["input_ids"].clone()
|
| 239 |
+
contxt_id = self.processor(text=formatted_contexts, return_tensors="pt")["input_ids"]
|
| 240 |
+
labels[: len(contxt_id)] = -100
|
| 241 |
+
with torch.inference_mode():
|
| 242 |
+
outputs = self.model(**model_inputs, labels=labels)
|
| 243 |
+
loss = outputs["loss"]
|
| 244 |
+
# loss = torch.exp(loss)
|
| 245 |
+
logits = outputs["logits"]
|
| 246 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 247 |
+
cont_toks = model_inputs["input_ids"][:, contxt_id.shape[1] :] # [1, seq]
|
| 248 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : model_inputs["input_ids"].shape[1]] # [1, seq]
|
| 249 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 250 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 251 |
+
pbar.update(1)
|
| 252 |
+
|
| 253 |
+
pbar.close()
|
| 254 |
+
return res
|
| 255 |
+
|
| 256 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 257 |
+
""" """
|
| 258 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 259 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 260 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 261 |
+
if left_truncate_len:
|
| 262 |
+
encoding = encoding[-left_truncate_len:]
|
| 263 |
+
return encoding
|
| 264 |
+
|
| 265 |
+
def tok_decode(self, tokens):
|
| 266 |
+
return self.tokenizer.decode(tokens)
|
| 267 |
+
|
| 268 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 269 |
+
raise NotImplementedError("TODO: Implement multi-round generation for Fuyu")
|
LaViDa/eval/lmms_eval/models/gemini_api.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import io
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import pathlib
|
| 5 |
+
import re
|
| 6 |
+
import time
|
| 7 |
+
from typing import List, Tuple
|
| 8 |
+
|
| 9 |
+
import datasets
|
| 10 |
+
from accelerate import Accelerator, DistributedType
|
| 11 |
+
from loguru import logger as eval_logger
|
| 12 |
+
from PIL import Image
|
| 13 |
+
from tqdm import tqdm
|
| 14 |
+
|
| 15 |
+
from lmms_eval.api.instance import Instance
|
| 16 |
+
from lmms_eval.api.model import lmms
|
| 17 |
+
from lmms_eval.api.registry import register_model
|
| 18 |
+
|
| 19 |
+
try:
|
| 20 |
+
import google.generativeai as genai
|
| 21 |
+
from google.generativeai.types import HarmBlockThreshold, HarmCategory
|
| 22 |
+
|
| 23 |
+
NUM_SECONDS_TO_SLEEP = 30
|
| 24 |
+
GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
|
| 25 |
+
genai.configure(api_key=GOOGLE_API_KEY)
|
| 26 |
+
|
| 27 |
+
except Exception as e:
|
| 28 |
+
eval_logger.error(f"Error importing generativeai: {str(e)}")
|
| 29 |
+
genai = None
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
import soundfile as sf
|
| 33 |
+
except Exception as e:
|
| 34 |
+
eval_logger.warning(f"Error importing soundfile, audio generation will not work: {str(e)}")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@register_model("gemini_api")
|
| 38 |
+
class GeminiAPI(lmms):
|
| 39 |
+
def __init__(
|
| 40 |
+
self,
|
| 41 |
+
model_version: str = "gemini-1.5-pro",
|
| 42 |
+
# modality: str = "image",
|
| 43 |
+
timeout: int = 120,
|
| 44 |
+
continual_mode: bool = True,
|
| 45 |
+
response_persistent_folder: str = "./logs/gemini_persistent_folder",
|
| 46 |
+
interleave: bool = False,
|
| 47 |
+
# We will cache the Gemini API response in this path and use it for future requests
|
| 48 |
+
**kwargs,
|
| 49 |
+
) -> None:
|
| 50 |
+
super().__init__()
|
| 51 |
+
self.model_version = model_version
|
| 52 |
+
self.timeout = timeout
|
| 53 |
+
self.model = genai.GenerativeModel(model_version)
|
| 54 |
+
self.continual_mode = continual_mode
|
| 55 |
+
self.response_persistent_file = ""
|
| 56 |
+
self.interleave = interleave
|
| 57 |
+
# if self.continual_mode and response_persistent_folder is None:
|
| 58 |
+
# raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.")
|
| 59 |
+
if self.continual_mode:
|
| 60 |
+
self.response_persistent_folder = response_persistent_folder
|
| 61 |
+
if not os.path.exists(self.response_persistent_folder):
|
| 62 |
+
os.makedirs(self.response_persistent_folder)
|
| 63 |
+
self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
|
| 64 |
+
|
| 65 |
+
if os.path.exists(self.response_persistent_file):
|
| 66 |
+
with open(self.response_persistent_file, "r") as f:
|
| 67 |
+
self.response_cache = json.load(f)
|
| 68 |
+
self.cache_mode = "resume"
|
| 69 |
+
else:
|
| 70 |
+
self.response_cache = {}
|
| 71 |
+
self.cache_mode = "start"
|
| 72 |
+
|
| 73 |
+
accelerator = Accelerator()
|
| 74 |
+
if accelerator.num_processes > 1:
|
| 75 |
+
assert self.continual_mode is False, "Continual mode is not supported with distributed inference."
|
| 76 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 77 |
+
self.accelerator = accelerator
|
| 78 |
+
if self.accelerator.is_local_main_process:
|
| 79 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 80 |
+
self._rank = self.accelerator.local_process_index
|
| 81 |
+
self._world_size = self.accelerator.num_processes
|
| 82 |
+
else:
|
| 83 |
+
self.accelerator = accelerator
|
| 84 |
+
self._rank = self.accelerator.local_process_index
|
| 85 |
+
self._world_size = self.accelerator.num_processes
|
| 86 |
+
|
| 87 |
+
self.device = self.accelerator.device
|
| 88 |
+
|
| 89 |
+
# self.modality = modality
|
| 90 |
+
|
| 91 |
+
self.video_pool = []
|
| 92 |
+
|
| 93 |
+
def free_video(self):
|
| 94 |
+
for video in self.video_pool:
|
| 95 |
+
video.delete()
|
| 96 |
+
self.video_pool = []
|
| 97 |
+
|
| 98 |
+
def flatten(self, input):
|
| 99 |
+
new_list = []
|
| 100 |
+
for i in input:
|
| 101 |
+
for j in i:
|
| 102 |
+
new_list.append(j)
|
| 103 |
+
return new_list
|
| 104 |
+
|
| 105 |
+
def get_image_size(self, image):
|
| 106 |
+
# Create a BytesIO object to store the image bytes
|
| 107 |
+
img_byte_array = io.BytesIO()
|
| 108 |
+
|
| 109 |
+
# Save the image to the BytesIO object
|
| 110 |
+
image.save(img_byte_array, format="PNG")
|
| 111 |
+
|
| 112 |
+
# Get the size of the BytesIO object
|
| 113 |
+
img_size = img_byte_array.tell()
|
| 114 |
+
|
| 115 |
+
return img_size
|
| 116 |
+
|
| 117 |
+
def encode_video(self, video_path):
|
| 118 |
+
uploaded_obj = genai.upload_file(path=video_path)
|
| 119 |
+
time.sleep(5)
|
| 120 |
+
self.video_pool.append(uploaded_obj)
|
| 121 |
+
return uploaded_obj
|
| 122 |
+
|
| 123 |
+
def encode_audio(self, audio):
|
| 124 |
+
audio_io = io.BytesIO()
|
| 125 |
+
sf.write(audio_io, audio["array"], audio["sampling_rate"], format="WAV")
|
| 126 |
+
return genai.upload_file(audio_io, mime_type="audio/wav")
|
| 127 |
+
|
| 128 |
+
def convert_modality(self, images):
|
| 129 |
+
for idx, img in enumerate(images):
|
| 130 |
+
if isinstance(img, dict) and "sampling_rate" in img: # audio
|
| 131 |
+
audio = self.encode_audio(img)
|
| 132 |
+
images[idx] = audio
|
| 133 |
+
elif isinstance(img, str): # video
|
| 134 |
+
try:
|
| 135 |
+
images[idx] = self.encode_video(img)
|
| 136 |
+
except Exception as e:
|
| 137 |
+
eval_logger.error(f"Error converting video: {str(e)}")
|
| 138 |
+
return images
|
| 139 |
+
|
| 140 |
+
def construct_interleaved_input(self, content, media):
|
| 141 |
+
pattern = r"<media_(\d+)>"
|
| 142 |
+
parts = re.split(pattern, content)
|
| 143 |
+
result = []
|
| 144 |
+
for i, part in enumerate(parts):
|
| 145 |
+
if i % 2 == 0:
|
| 146 |
+
if part == "":
|
| 147 |
+
continue
|
| 148 |
+
result.append(part)
|
| 149 |
+
else:
|
| 150 |
+
result.append(media[int(part)])
|
| 151 |
+
|
| 152 |
+
return result
|
| 153 |
+
|
| 154 |
+
def generate_until(self, requests) -> List[str]:
|
| 155 |
+
res = []
|
| 156 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 157 |
+
|
| 158 |
+
def get_uuid(task, split, doc_id):
|
| 159 |
+
return f"{task}___{split}___{doc_id}"
|
| 160 |
+
|
| 161 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 162 |
+
if self.continual_mode and self.cache_mode == "resume":
|
| 163 |
+
doc_uuid = get_uuid(task, split, doc_id)
|
| 164 |
+
if doc_uuid in self.response_cache:
|
| 165 |
+
content = self.response_cache[doc_uuid]
|
| 166 |
+
if content:
|
| 167 |
+
res.append(content)
|
| 168 |
+
pbar.update(1)
|
| 169 |
+
continue
|
| 170 |
+
|
| 171 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 172 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 173 |
+
if "temperature" not in gen_kwargs:
|
| 174 |
+
gen_kwargs["temperature"] = 0
|
| 175 |
+
|
| 176 |
+
config = genai.GenerationConfig(
|
| 177 |
+
max_output_tokens=gen_kwargs["max_new_tokens"],
|
| 178 |
+
temperature=gen_kwargs["temperature"],
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 182 |
+
visuals = self.flatten(visuals)
|
| 183 |
+
visuals = self.convert_modality(visuals)
|
| 184 |
+
|
| 185 |
+
if self.interleave:
|
| 186 |
+
message = self.construct_interleaved_input(contexts, visuals)
|
| 187 |
+
else:
|
| 188 |
+
message = [contexts] + visuals
|
| 189 |
+
|
| 190 |
+
for attempt in range(5):
|
| 191 |
+
try:
|
| 192 |
+
content = self.model.generate_content(
|
| 193 |
+
message,
|
| 194 |
+
generation_config=config,
|
| 195 |
+
safety_settings={
|
| 196 |
+
HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
|
| 197 |
+
HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
|
| 198 |
+
HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
|
| 199 |
+
HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
|
| 200 |
+
},
|
| 201 |
+
)
|
| 202 |
+
content = content.text
|
| 203 |
+
break
|
| 204 |
+
except Exception as e:
|
| 205 |
+
eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
|
| 206 |
+
if isinstance(e, ValueError):
|
| 207 |
+
try:
|
| 208 |
+
eval_logger.info(f"Prompt feed_back: {content.prompt_feedback}")
|
| 209 |
+
content = ""
|
| 210 |
+
break
|
| 211 |
+
except Exception:
|
| 212 |
+
pass
|
| 213 |
+
if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt
|
| 214 |
+
time.sleep(NUM_SECONDS_TO_SLEEP)
|
| 215 |
+
else: # If this was the last attempt, log and return empty
|
| 216 |
+
eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
|
| 217 |
+
content = ""
|
| 218 |
+
res.append(content)
|
| 219 |
+
pbar.update(1)
|
| 220 |
+
|
| 221 |
+
self.free_video()
|
| 222 |
+
|
| 223 |
+
if self.continual_mode is True: # Cache the response
|
| 224 |
+
doc_uuid = get_uuid(task, split, doc_id)
|
| 225 |
+
self.response_cache[doc_uuid] = content
|
| 226 |
+
with open(self.response_persistent_file, "w") as f:
|
| 227 |
+
json.dump(self.response_cache, f)
|
| 228 |
+
|
| 229 |
+
pbar.close()
|
| 230 |
+
return res
|
| 231 |
+
|
| 232 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 233 |
+
raise NotImplementedError("TODO: Implement multi-round generation for Gemini API")
|
| 234 |
+
|
| 235 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 236 |
+
# TODO
|
| 237 |
+
assert False, "Gemini API not support"
|
| 238 |
+
|
| 239 |
+
def get_image_audio_text_interleaved_messsage(self, image_path, audio_path, question):
|
| 240 |
+
# image_path for list of image path
|
| 241 |
+
# audio_path for list of audio path
|
| 242 |
+
# question for question
|
| 243 |
+
|
| 244 |
+
# fixed image token and no audio in text
|
| 245 |
+
for index in range(1, 1 + len(image_path)):
|
| 246 |
+
question = question.replace(f"[img{index}]", "<image>")
|
| 247 |
+
for index in range(1, 1 + len(audio_path)):
|
| 248 |
+
question = question.replace(f"[audio{index}]", "<audio>")
|
| 249 |
+
|
| 250 |
+
text = question
|
| 251 |
+
|
| 252 |
+
info_list = []
|
| 253 |
+
image_counter = 0
|
| 254 |
+
audio_counter = 0
|
| 255 |
+
for part in re.split(r"(<image>|<audio>)", text):
|
| 256 |
+
if part == "<image>":
|
| 257 |
+
info_list.append(Image.open(image_path[image_counter]))
|
| 258 |
+
image_counter += 1
|
| 259 |
+
elif part == "<audio>":
|
| 260 |
+
info_list.append({"mime_type": "audio/wav", "data": pathlib.Path(audio_path[audio_counter]).read_bytes()})
|
| 261 |
+
audio_counter += 1
|
| 262 |
+
else:
|
| 263 |
+
if part == " ":
|
| 264 |
+
continue
|
| 265 |
+
info_list.append(part)
|
| 266 |
+
|
| 267 |
+
return info_list
|
| 268 |
+
|
| 269 |
+
def get_video_audio_text_interleaved_message(self, video_path, audio_path, question):
|
| 270 |
+
# image_path for list of image path
|
| 271 |
+
# audio_path for list of audio path
|
| 272 |
+
# question for question
|
| 273 |
+
|
| 274 |
+
# fixed video token and no audio in text
|
| 275 |
+
for index in range(1, 1 + len(video_path)):
|
| 276 |
+
question = question.replace(f"[video{index}]", "<video>")
|
| 277 |
+
for index in range(1, 1 + len(audio_path)):
|
| 278 |
+
question = question.replace(f"[audio{index}]", "<audio>")
|
| 279 |
+
|
| 280 |
+
text = question
|
| 281 |
+
|
| 282 |
+
info_list = []
|
| 283 |
+
video_counter = 0
|
| 284 |
+
audio_counter = 0
|
| 285 |
+
for part in re.split(r"(<video>|<audio>)", text):
|
| 286 |
+
if part == "<video>":
|
| 287 |
+
current_video_file_name = video_path[video_counter]
|
| 288 |
+
current_video_file = genai.upload_file(path=current_video_file_name)
|
| 289 |
+
while current_video_file.state.name == "processing":
|
| 290 |
+
print("uploading file")
|
| 291 |
+
time.sleep(5)
|
| 292 |
+
current_video_file = genai.get_file(current_video_file.name)
|
| 293 |
+
if current_video_file.state.name == "FAILED":
|
| 294 |
+
print("uploading file failed, next question")
|
| 295 |
+
return 0
|
| 296 |
+
info_list.append(current_video_file)
|
| 297 |
+
video_counter += 1
|
| 298 |
+
elif part == "<audio>":
|
| 299 |
+
info_list.append({"mime_type": "audio/wav", "data": pathlib.Path(audio_path[audio_counter]).read_bytes()})
|
| 300 |
+
audio_counter += 1
|
| 301 |
+
else:
|
| 302 |
+
if part == " ":
|
| 303 |
+
continue
|
| 304 |
+
info_list.append(part)
|
| 305 |
+
|
| 306 |
+
return info_list
|
LaViDa/eval/lmms_eval/models/gpt4v.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
from copy import deepcopy
|
| 6 |
+
from io import BytesIO
|
| 7 |
+
from typing import List, Tuple, Union
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import requests as url_requests
|
| 11 |
+
from accelerate import Accelerator, DistributedType
|
| 12 |
+
from openai import AzureOpenAI, OpenAI
|
| 13 |
+
from tqdm import tqdm
|
| 14 |
+
|
| 15 |
+
from lmms_eval.api.instance import Instance
|
| 16 |
+
from lmms_eval.api.model import lmms
|
| 17 |
+
from lmms_eval.api.registry import register_model
|
| 18 |
+
|
| 19 |
+
try:
|
| 20 |
+
from decord import VideoReader, cpu
|
| 21 |
+
except ImportError:
|
| 22 |
+
pass
|
| 23 |
+
|
| 24 |
+
from loguru import logger as eval_logger
|
| 25 |
+
from PIL import Image
|
| 26 |
+
|
| 27 |
+
API_TYPE = os.getenv("API_TYPE", "openai")
|
| 28 |
+
NUM_SECONDS_TO_SLEEP = 10
|
| 29 |
+
if API_TYPE == "openai":
|
| 30 |
+
API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
|
| 31 |
+
API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
|
| 32 |
+
|
| 33 |
+
elif API_TYPE == "azure":
|
| 34 |
+
API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
|
| 35 |
+
API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
|
| 36 |
+
API_VERSION = os.getenv("AZURE_API_VERSION", "2023-07-01-preview")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@register_model("gpt4v")
|
| 40 |
+
class GPT4V(lmms):
|
| 41 |
+
def __init__(
|
| 42 |
+
self,
|
| 43 |
+
model_version: str = "gpt-4-vision-preview",
|
| 44 |
+
modality: str = "video",
|
| 45 |
+
max_frames_num: int = 10,
|
| 46 |
+
timeout: int = 120,
|
| 47 |
+
continual_mode: bool = False,
|
| 48 |
+
response_persistent_folder: str = None,
|
| 49 |
+
max_size_in_mb: int = 20,
|
| 50 |
+
**kwargs,
|
| 51 |
+
) -> None:
|
| 52 |
+
super().__init__()
|
| 53 |
+
# Manually set a image token for GPT4V so that we can search for it
|
| 54 |
+
# and split the text and image
|
| 55 |
+
# Here we just use the same token as llava for convenient
|
| 56 |
+
self.model_version = model_version
|
| 57 |
+
self.modality = modality
|
| 58 |
+
self.max_frames_num = max_frames_num
|
| 59 |
+
self.image_token = "<image>"
|
| 60 |
+
self.timeout = timeout
|
| 61 |
+
self.continual_mode = continual_mode
|
| 62 |
+
if self.continual_mode:
|
| 63 |
+
if response_persistent_folder is None:
|
| 64 |
+
raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
|
| 65 |
+
|
| 66 |
+
os.makedirs(response_persistent_folder, exist_ok=True)
|
| 67 |
+
self.response_persistent_folder = response_persistent_folder
|
| 68 |
+
self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
|
| 69 |
+
|
| 70 |
+
if os.path.exists(self.response_persistent_file):
|
| 71 |
+
with open(self.response_persistent_file, "r") as f:
|
| 72 |
+
self.response_cache = json.load(f)
|
| 73 |
+
self.cache_mode = "resume"
|
| 74 |
+
else:
|
| 75 |
+
self.response_cache = {}
|
| 76 |
+
self.cache_mode = "start"
|
| 77 |
+
|
| 78 |
+
if API_TYPE == "openai":
|
| 79 |
+
self.client = OpenAI(api_key=API_KEY)
|
| 80 |
+
elif API_TYPE == "azure":
|
| 81 |
+
self.client = AzureOpenAI(api_key=API_KEY, azure_endpoint=API_URL, api_version=API_VERSION)
|
| 82 |
+
|
| 83 |
+
accelerator = Accelerator()
|
| 84 |
+
# assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 85 |
+
if accelerator.num_processes > 1:
|
| 86 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 87 |
+
self.accelerator = accelerator
|
| 88 |
+
if self.accelerator.is_local_main_process:
|
| 89 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 90 |
+
self._rank = self.accelerator.local_process_index
|
| 91 |
+
self._world_size = self.accelerator.num_processes
|
| 92 |
+
else:
|
| 93 |
+
self.accelerator = accelerator
|
| 94 |
+
self._rank = self.accelerator.local_process_index
|
| 95 |
+
self._world_size = self.accelerator.num_processes
|
| 96 |
+
|
| 97 |
+
self.max_size_in_mb = max_size_in_mb
|
| 98 |
+
self.device = self.accelerator.device
|
| 99 |
+
|
| 100 |
+
# Function to encode the image
|
| 101 |
+
def encode_image(self, image: Union[Image.Image, str]):
|
| 102 |
+
max_size = self.max_size_in_mb * 1024 * 1024 # 20MB in bytes
|
| 103 |
+
if isinstance(image, str):
|
| 104 |
+
img = Image.open(image).convert("RGB")
|
| 105 |
+
else:
|
| 106 |
+
img = image.copy()
|
| 107 |
+
|
| 108 |
+
output_buffer = BytesIO()
|
| 109 |
+
img.save(output_buffer, format="PNG")
|
| 110 |
+
byte_data = output_buffer.getvalue()
|
| 111 |
+
|
| 112 |
+
# If image is too large, resize it while maintaining aspect ratio
|
| 113 |
+
while len(byte_data) > max_size and img.size[0] > 100 and img.size[1] > 100:
|
| 114 |
+
new_size = (int(img.size[0] * 0.75), int(img.size[1] * 0.75))
|
| 115 |
+
img = img.resize(new_size, Image.Resampling.LANCZOS)
|
| 116 |
+
|
| 117 |
+
output_buffer = BytesIO()
|
| 118 |
+
img.save(output_buffer, format="PNG")
|
| 119 |
+
byte_data = output_buffer.getvalue()
|
| 120 |
+
|
| 121 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 122 |
+
return base64_str
|
| 123 |
+
|
| 124 |
+
# Function to encode the video
|
| 125 |
+
def encode_video(self, video_path, for_get_frames_num):
|
| 126 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 127 |
+
total_frame_num = len(vr)
|
| 128 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int)
|
| 129 |
+
|
| 130 |
+
# Ensure the last frame is included
|
| 131 |
+
if total_frame_num - 1 not in uniform_sampled_frames:
|
| 132 |
+
uniform_sampled_frames = np.append(uniform_sampled_frames, total_frame_num - 1)
|
| 133 |
+
|
| 134 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 135 |
+
frames = vr.get_batch(frame_idx).asnumpy()
|
| 136 |
+
|
| 137 |
+
base64_frames = []
|
| 138 |
+
for frame in frames:
|
| 139 |
+
img = Image.fromarray(frame)
|
| 140 |
+
output_buffer = BytesIO()
|
| 141 |
+
img.save(output_buffer, format="PNG")
|
| 142 |
+
byte_data = output_buffer.getvalue()
|
| 143 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 144 |
+
base64_frames.append(base64_str)
|
| 145 |
+
|
| 146 |
+
return base64_frames
|
| 147 |
+
|
| 148 |
+
def flatten(self, input):
|
| 149 |
+
new_list = []
|
| 150 |
+
for i in input:
|
| 151 |
+
for j in i:
|
| 152 |
+
new_list.append(j)
|
| 153 |
+
return new_list
|
| 154 |
+
|
| 155 |
+
def generate_until(self, requests) -> List[str]:
|
| 156 |
+
res = []
|
| 157 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 158 |
+
|
| 159 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 160 |
+
if self.continual_mode is True and self.cache_mode == "resume":
|
| 161 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 162 |
+
if doc_uuid in self.response_cache:
|
| 163 |
+
response_text = self.response_cache[doc_uuid]
|
| 164 |
+
if response_text:
|
| 165 |
+
res.append(response_text)
|
| 166 |
+
pbar.update(1)
|
| 167 |
+
continue
|
| 168 |
+
|
| 169 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 170 |
+
if None in visuals:
|
| 171 |
+
visuals = []
|
| 172 |
+
imgs = []
|
| 173 |
+
else:
|
| 174 |
+
visuals = self.flatten(visuals)
|
| 175 |
+
imgs = [] # multiple images or frames for video
|
| 176 |
+
for visual in visuals:
|
| 177 |
+
if isinstance(visual, str) and (".mp4" in visual or ".avi" in visual or ".mov" in visual or ".flv" in visual or ".wmv" in visual):
|
| 178 |
+
frames = self.encode_video(visual, self.max_frames_num)
|
| 179 |
+
imgs.extend(frames)
|
| 180 |
+
elif isinstance(visual, str) and (".jpg" in visual or ".jpeg" in visual or ".png" in visual or ".gif" in visual or ".bmp" in visual or ".tiff" in visual or ".webp" in visual):
|
| 181 |
+
img = self.encode_image(visual)
|
| 182 |
+
imgs.append(img)
|
| 183 |
+
elif isinstance(visual, Image.Image):
|
| 184 |
+
img = self.encode_image(visual)
|
| 185 |
+
imgs.append(img)
|
| 186 |
+
|
| 187 |
+
payload = {"messages": []}
|
| 188 |
+
payload["model"] = self.model_version
|
| 189 |
+
|
| 190 |
+
payload["messages"].append({"role": "user", "content": []})
|
| 191 |
+
payload["messages"][0]["content"].append({"type": "text", "text": contexts})
|
| 192 |
+
for img in imgs:
|
| 193 |
+
payload["messages"][0]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}})
|
| 194 |
+
|
| 195 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 196 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 197 |
+
if gen_kwargs["max_new_tokens"] > 4096:
|
| 198 |
+
gen_kwargs["max_new_tokens"] = 4096
|
| 199 |
+
if "temperature" not in gen_kwargs:
|
| 200 |
+
gen_kwargs["temperature"] = 0
|
| 201 |
+
if "top_p" not in gen_kwargs:
|
| 202 |
+
gen_kwargs["top_p"] = None
|
| 203 |
+
if "num_beams" not in gen_kwargs:
|
| 204 |
+
gen_kwargs["num_beams"] = 1
|
| 205 |
+
|
| 206 |
+
payload["max_tokens"] = gen_kwargs["max_new_tokens"]
|
| 207 |
+
payload["temperature"] = gen_kwargs["temperature"]
|
| 208 |
+
|
| 209 |
+
MAX_RETRIES = 5
|
| 210 |
+
for attempt in range(MAX_RETRIES):
|
| 211 |
+
try:
|
| 212 |
+
response = self.client.chat.completions.create(**payload)
|
| 213 |
+
response_text = response.choices[0].message.content
|
| 214 |
+
break # If successful, break out of the loop
|
| 215 |
+
|
| 216 |
+
except Exception as e:
|
| 217 |
+
error_msg = str(e)
|
| 218 |
+
eval_logger.info(f"Attempt {attempt + 1}/{MAX_RETRIES} failed with error: {error_msg}")
|
| 219 |
+
|
| 220 |
+
# On last attempt, log error and set empty response
|
| 221 |
+
if attempt == MAX_RETRIES - 1:
|
| 222 |
+
eval_logger.error(f"All {MAX_RETRIES} attempts failed. Last error: {error_msg}")
|
| 223 |
+
response_text = ""
|
| 224 |
+
else:
|
| 225 |
+
time.sleep(NUM_SECONDS_TO_SLEEP)
|
| 226 |
+
|
| 227 |
+
res.append(response_text)
|
| 228 |
+
pbar.update(1)
|
| 229 |
+
|
| 230 |
+
if self.continual_mode is True: # Cache the response
|
| 231 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 232 |
+
self.response_cache[doc_uuid] = response_text
|
| 233 |
+
with open(self.response_persistent_file, "w") as f:
|
| 234 |
+
json.dump(self.response_cache, f)
|
| 235 |
+
|
| 236 |
+
pbar.close()
|
| 237 |
+
return res
|
| 238 |
+
|
| 239 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 240 |
+
raise NotImplementedError("TODO: Implement multi-round generation for GPT4V")
|
| 241 |
+
|
| 242 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 243 |
+
# TODO
|
| 244 |
+
assert False, "GPT4V not support"
|
LaViDa/eval/lmms_eval/models/idefics2.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from accelerate import Accelerator, DistributedType
|
| 6 |
+
from accelerate.state import AcceleratorState
|
| 7 |
+
from torchvision.transforms.functional import to_pil_image
|
| 8 |
+
from tqdm import tqdm
|
| 9 |
+
from transformers import AutoProcessor, Idefics2ForConditionalGeneration
|
| 10 |
+
|
| 11 |
+
from lmms_eval import utils
|
| 12 |
+
from lmms_eval.api.instance import Instance
|
| 13 |
+
from lmms_eval.api.model import lmms
|
| 14 |
+
from lmms_eval.api.registry import register_model
|
| 15 |
+
from lmms_eval.models.model_utils.load_video import load_video_decord
|
| 16 |
+
|
| 17 |
+
warnings.filterwarnings("ignore")
|
| 18 |
+
|
| 19 |
+
from loguru import logger as eval_logger
|
| 20 |
+
|
| 21 |
+
DEFAULT_IMAGE_TOKEN = "<image>"
|
| 22 |
+
try:
|
| 23 |
+
import flash_attn
|
| 24 |
+
|
| 25 |
+
best_fit_attn_implementation = "flash_attention_2"
|
| 26 |
+
except ImportError:
|
| 27 |
+
best_fit_attn_implementation = "eager"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@register_model("idefics2")
|
| 31 |
+
class Idefics2(lmms):
|
| 32 |
+
"""
|
| 33 |
+
Idefics2 Model for Hugging Face Transformers: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py
|
| 34 |
+
|
| 35 |
+
Example usage:
|
| 36 |
+
|
| 37 |
+
accelerate launch --num_processes=8 -m lmms_eval \
|
| 38 |
+
--model idefics2 \
|
| 39 |
+
--model_args pretrained=HuggingFaceM4/idefics2-8b \
|
| 40 |
+
--tasks mme \
|
| 41 |
+
--batch_size 1 \
|
| 42 |
+
--output_path ./logs/ \
|
| 43 |
+
--log_samples
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
def __init__(
|
| 47 |
+
self,
|
| 48 |
+
pretrained: str = "HuggingFaceM4/idefics2-8b",
|
| 49 |
+
revision: str = "main",
|
| 50 |
+
device: str = "cuda",
|
| 51 |
+
dtype: Optional[Union[str, torch.dtype]] = "float16",
|
| 52 |
+
batch_size: int = 1,
|
| 53 |
+
trust_remote_code: Optional[bool] = False,
|
| 54 |
+
attn_implementation: Optional[str] = best_fit_attn_implementation,
|
| 55 |
+
device_map: str = "",
|
| 56 |
+
use_cache: bool = True,
|
| 57 |
+
do_image_splitting: bool = False,
|
| 58 |
+
max_frames_num: int = 16,
|
| 59 |
+
**kwargs,
|
| 60 |
+
) -> None:
|
| 61 |
+
super().__init__()
|
| 62 |
+
# Do not use kwargs for now
|
| 63 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 64 |
+
|
| 65 |
+
accelerator = Accelerator()
|
| 66 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 67 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 68 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 69 |
+
else:
|
| 70 |
+
self._device = torch.device(device)
|
| 71 |
+
self.device_map = device_map
|
| 72 |
+
if isinstance(dtype, str) and dtype != "auto":
|
| 73 |
+
dtype = getattr(torch, dtype)
|
| 74 |
+
self._model = Idefics2ForConditionalGeneration.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
|
| 75 |
+
self._processor = AutoProcessor.from_pretrained(pretrained, do_image_splitting=do_image_splitting, revision=revision, trust_remote_code=trust_remote_code)
|
| 76 |
+
self.max_frames_num = max_frames_num
|
| 77 |
+
|
| 78 |
+
self._tokenizer = self._processor.tokenizer
|
| 79 |
+
self._config = self._model.config
|
| 80 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 81 |
+
self.use_cache = use_cache
|
| 82 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 83 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 84 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 85 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 86 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 87 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 88 |
+
kwargs = {
|
| 89 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 90 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 91 |
+
}
|
| 92 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 93 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 94 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 95 |
+
self._model = accelerator.prepare(self.model)
|
| 96 |
+
else:
|
| 97 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 98 |
+
self.accelerator = accelerator
|
| 99 |
+
if self.accelerator.is_local_main_process:
|
| 100 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 101 |
+
self._rank = self.accelerator.local_process_index
|
| 102 |
+
self._world_size = self.accelerator.num_processes
|
| 103 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 104 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
|
| 105 |
+
self._rank = 0
|
| 106 |
+
self._world_size = 1
|
| 107 |
+
else:
|
| 108 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 109 |
+
self.model.to(self._device)
|
| 110 |
+
self._rank = 0
|
| 111 |
+
self._world_size = 1
|
| 112 |
+
|
| 113 |
+
@property
|
| 114 |
+
def config(self):
|
| 115 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 116 |
+
return self._config
|
| 117 |
+
|
| 118 |
+
@property
|
| 119 |
+
def tokenizer(self):
|
| 120 |
+
return self._tokenizer
|
| 121 |
+
|
| 122 |
+
@property
|
| 123 |
+
def model(self):
|
| 124 |
+
# returns the model, unwrapping it if using Accelerate
|
| 125 |
+
if hasattr(self, "accelerator"):
|
| 126 |
+
return self.accelerator.unwrap_model(self._model)
|
| 127 |
+
else:
|
| 128 |
+
return self._model
|
| 129 |
+
|
| 130 |
+
@property
|
| 131 |
+
def eot_token_id(self):
|
| 132 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 133 |
+
return self.tokenizer.eos_token_id
|
| 134 |
+
|
| 135 |
+
@property
|
| 136 |
+
def max_length(self):
|
| 137 |
+
return self._max_length
|
| 138 |
+
|
| 139 |
+
@property
|
| 140 |
+
def batch_size(self):
|
| 141 |
+
return self.batch_size_per_gpu
|
| 142 |
+
|
| 143 |
+
@property
|
| 144 |
+
def device(self):
|
| 145 |
+
return self._device
|
| 146 |
+
|
| 147 |
+
@property
|
| 148 |
+
def rank(self):
|
| 149 |
+
return self._rank
|
| 150 |
+
|
| 151 |
+
@property
|
| 152 |
+
def world_size(self):
|
| 153 |
+
return self._world_size
|
| 154 |
+
|
| 155 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 156 |
+
""" """
|
| 157 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 158 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 159 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 160 |
+
if left_truncate_len:
|
| 161 |
+
encoding = encoding[-left_truncate_len:]
|
| 162 |
+
return encoding
|
| 163 |
+
|
| 164 |
+
def tok_decode(self, tokens):
|
| 165 |
+
return self.tokenizer.decode(tokens)
|
| 166 |
+
|
| 167 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 168 |
+
raise NotImplementedError("Loglikelihood is not implemented for Idefics2 model")
|
| 169 |
+
|
| 170 |
+
def flatten(self, input):
|
| 171 |
+
new_list = []
|
| 172 |
+
for i in input:
|
| 173 |
+
for j in i:
|
| 174 |
+
new_list.append(j)
|
| 175 |
+
return new_list
|
| 176 |
+
|
| 177 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 178 |
+
res = []
|
| 179 |
+
|
| 180 |
+
def _collate(x):
|
| 181 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 182 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 183 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 184 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 185 |
+
# automatic adaptive batches much much easier to implement
|
| 186 |
+
# - any OOMs will happen right away rather than near the end
|
| 187 |
+
toks = self.tok_encode(x[0])
|
| 188 |
+
return -len(toks), x[0]
|
| 189 |
+
|
| 190 |
+
# we group requests by their generation_kwargs,
|
| 191 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 192 |
+
# in the same batch.
|
| 193 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 194 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 195 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 196 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 197 |
+
for chunk in chunks:
|
| 198 |
+
contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
|
| 199 |
+
visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
|
| 200 |
+
# we assume all gen kwargs in the batch are the same
|
| 201 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 202 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 203 |
+
#
|
| 204 |
+
until = gen_kwargs.pop("until", None)
|
| 205 |
+
image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio", None)
|
| 206 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 207 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 208 |
+
if "temperature" not in gen_kwargs:
|
| 209 |
+
gen_kwargs["temperature"] = 0
|
| 210 |
+
|
| 211 |
+
prompts = []
|
| 212 |
+
videos = None
|
| 213 |
+
for context, visual in zip(contexts, visuals):
|
| 214 |
+
content = []
|
| 215 |
+
if isinstance(visual[0], str):
|
| 216 |
+
videos = load_video_decord(visual[0], max_frames_num=self.max_frames_num)
|
| 217 |
+
for _ in range(videos.shape[0]):
|
| 218 |
+
content.append({"type": "image"})
|
| 219 |
+
elif DEFAULT_IMAGE_TOKEN not in context:
|
| 220 |
+
for image in visual:
|
| 221 |
+
content.append({"type": "image"})
|
| 222 |
+
content.append({"type": "text", "text": context})
|
| 223 |
+
message = [{"role": "user", "content": content}]
|
| 224 |
+
prompt = self._processor.apply_chat_template(message, add_generation_prompt=True)
|
| 225 |
+
prompts.append(prompt)
|
| 226 |
+
if videos is not None:
|
| 227 |
+
images = []
|
| 228 |
+
for frame in videos:
|
| 229 |
+
images.append(to_pil_image(frame))
|
| 230 |
+
inputs = self._processor(text=prompts, images=images, padding=True, return_tensors="pt")
|
| 231 |
+
else:
|
| 232 |
+
inputs = self._processor(text=prompts, images=visuals, padding=True, return_tensors="pt")
|
| 233 |
+
inputs = {k: v.to(self.device) for k, v in inputs.items()}
|
| 234 |
+
output_ids = self.model.generate(**inputs, **gen_kwargs)
|
| 235 |
+
# only retain the generated text
|
| 236 |
+
for output_id, input_id in zip(output_ids, inputs["input_ids"]):
|
| 237 |
+
generated_id = output_id[len(input_id) :]
|
| 238 |
+
generated_text = self.tokenizer.decode(generated_id, skip_special_tokens=True)
|
| 239 |
+
|
| 240 |
+
res.append(generated_text)
|
| 241 |
+
pbar.update(1)
|
| 242 |
+
# reorder this group of results back to original unsorted form
|
| 243 |
+
res = re_ords.get_original(res)
|
| 244 |
+
|
| 245 |
+
pbar.close()
|
| 246 |
+
return res
|
| 247 |
+
|
| 248 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 249 |
+
raise NotImplementedError("TODO: Implement multi-round generation for Idefics2")
|
LaViDa/eval/lmms_eval/models/instructblip.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import warnings
|
| 3 |
+
from typing import List, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import transformers
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from accelerate.state import AcceleratorState
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
from transformers import InstructBlipForConditionalGeneration, InstructBlipProcessor
|
| 11 |
+
|
| 12 |
+
from lmms_eval import utils
|
| 13 |
+
from lmms_eval.api.instance import Instance
|
| 14 |
+
from lmms_eval.api.model import lmms
|
| 15 |
+
from lmms_eval.api.registry import register_model
|
| 16 |
+
from lmms_eval.tasks.mmmu.utils_group_img import process_images
|
| 17 |
+
from lmms_eval.utils import stop_sequences_criteria
|
| 18 |
+
|
| 19 |
+
warnings.filterwarnings("ignore")
|
| 20 |
+
|
| 21 |
+
from loguru import logger as eval_logger
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@register_model("instructblip")
|
| 25 |
+
class InstructBLIP(lmms):
|
| 26 |
+
"""
|
| 27 |
+
InstructBLIP Model
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(
|
| 31 |
+
self,
|
| 32 |
+
pretrained: str = "Salesforce/instructblip-vicuna-7b",
|
| 33 |
+
device: Optional[str] = "cuda",
|
| 34 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 35 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 36 |
+
**kwargs,
|
| 37 |
+
) -> None:
|
| 38 |
+
super().__init__()
|
| 39 |
+
# Do not use kwargs for now
|
| 40 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 41 |
+
|
| 42 |
+
accelerator = Accelerator()
|
| 43 |
+
if accelerator.num_processes > 1:
|
| 44 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 45 |
+
else:
|
| 46 |
+
self._device = device
|
| 47 |
+
self._model = InstructBlipForConditionalGeneration.from_pretrained(pretrained, device_map=self._device)
|
| 48 |
+
self._image_processor = InstructBlipProcessor.from_pretrained(pretrained)
|
| 49 |
+
self._tokenizer = self._image_processor.tokenizer
|
| 50 |
+
self._config = self._model.config
|
| 51 |
+
self.model.eval()
|
| 52 |
+
self.model.tie_weights()
|
| 53 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 54 |
+
if accelerator.num_processes > 1:
|
| 55 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 56 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 57 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 58 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 59 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 60 |
+
kwargs = {
|
| 61 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 62 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 63 |
+
}
|
| 64 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 65 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 66 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 67 |
+
self._model = accelerator.prepare(self.model)
|
| 68 |
+
else:
|
| 69 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 70 |
+
self.accelerator = accelerator
|
| 71 |
+
if self.accelerator.is_local_main_process:
|
| 72 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 73 |
+
self._rank = self.accelerator.local_process_index
|
| 74 |
+
self._world_size = self.accelerator.num_processes
|
| 75 |
+
else:
|
| 76 |
+
self.model.to(self._device)
|
| 77 |
+
self._rank = 0
|
| 78 |
+
self._world_size = 1
|
| 79 |
+
|
| 80 |
+
@property
|
| 81 |
+
def config(self):
|
| 82 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 83 |
+
return self._config
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def tokenizer(self):
|
| 87 |
+
return self._tokenizer
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def model(self):
|
| 91 |
+
# returns the model, unwrapping it if using Accelerate
|
| 92 |
+
if hasattr(self, "accelerator"):
|
| 93 |
+
return self.accelerator.unwrap_model(self._model)
|
| 94 |
+
else:
|
| 95 |
+
return self._model
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def eot_token_id(self):
|
| 99 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 100 |
+
return self.tokenizer.eos_token_id
|
| 101 |
+
|
| 102 |
+
@property
|
| 103 |
+
def max_length(self):
|
| 104 |
+
return self._max_length
|
| 105 |
+
|
| 106 |
+
@property
|
| 107 |
+
def batch_size(self):
|
| 108 |
+
return self.batch_size_per_gpu
|
| 109 |
+
|
| 110 |
+
@property
|
| 111 |
+
def device(self):
|
| 112 |
+
return self._device
|
| 113 |
+
|
| 114 |
+
@property
|
| 115 |
+
def rank(self):
|
| 116 |
+
return self._rank
|
| 117 |
+
|
| 118 |
+
@property
|
| 119 |
+
def world_size(self):
|
| 120 |
+
return self._world_size
|
| 121 |
+
|
| 122 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 123 |
+
""" """
|
| 124 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 125 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 126 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 127 |
+
if left_truncate_len:
|
| 128 |
+
encoding = encoding[-left_truncate_len:]
|
| 129 |
+
return encoding
|
| 130 |
+
|
| 131 |
+
def tok_decode(self, tokens):
|
| 132 |
+
return self.tokenizer.decode(tokens)
|
| 133 |
+
|
| 134 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 135 |
+
# TODO
|
| 136 |
+
assert False, "We have not implemented this function for InstructBLIP yet"
|
| 137 |
+
|
| 138 |
+
def flatten(self, input):
|
| 139 |
+
new_list = []
|
| 140 |
+
for i in input:
|
| 141 |
+
for j in i:
|
| 142 |
+
new_list.append(j)
|
| 143 |
+
return new_list
|
| 144 |
+
|
| 145 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 146 |
+
res = []
|
| 147 |
+
|
| 148 |
+
def _collate(x):
|
| 149 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 150 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 151 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 152 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 153 |
+
# automatic adaptive batches much much easier to implement
|
| 154 |
+
# - any OOMs will happen right away rather than near the end
|
| 155 |
+
toks = self.tok_encode(x[0])
|
| 156 |
+
return -len(toks), x[0]
|
| 157 |
+
|
| 158 |
+
# we group requests by their generation_kwargs,
|
| 159 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 160 |
+
# in the same batch.
|
| 161 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 162 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 163 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 164 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 165 |
+
for chunk in chunks:
|
| 166 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 167 |
+
task = task[0]
|
| 168 |
+
split = split[0]
|
| 169 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 170 |
+
visuals = self.flatten(visuals)
|
| 171 |
+
# we assume all gen kwargs in the batch are the same
|
| 172 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 173 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 174 |
+
|
| 175 |
+
# Set default values for until and max_new_tokens
|
| 176 |
+
until = [self.tok_decode(self.eot_token_id)]
|
| 177 |
+
|
| 178 |
+
# Update values from gen_kwargs if present
|
| 179 |
+
if "until" in gen_kwargs:
|
| 180 |
+
until = gen_kwargs.pop("until")
|
| 181 |
+
if isinstance(until, str):
|
| 182 |
+
until = [until]
|
| 183 |
+
elif not isinstance(until, list):
|
| 184 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 185 |
+
assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
|
| 186 |
+
context = contexts[0]
|
| 187 |
+
if "<image>" in context:
|
| 188 |
+
# instruct blip does not expect the <image> tag
|
| 189 |
+
context = context.replace("<image>", "")
|
| 190 |
+
# Set trunction equals true here, the max length for qformer tokenizer is 512
|
| 191 |
+
# if not truncate, some questions will cause size mismatch
|
| 192 |
+
# The transformer implementation can't handle multi images for blip
|
| 193 |
+
# Concat it into one image
|
| 194 |
+
if len(visuals) > 1:
|
| 195 |
+
visuals = [process_images(visuals)]
|
| 196 |
+
inputs = self._image_processor(images=visuals, text=context, return_tensors="pt", truncation=True).to(self.device)
|
| 197 |
+
|
| 198 |
+
gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
|
| 199 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 200 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 201 |
+
if "temperature" not in gen_kwargs:
|
| 202 |
+
gen_kwargs["temperature"] = 0
|
| 203 |
+
if "top_p" not in gen_kwargs:
|
| 204 |
+
gen_kwargs["top_p"] = None
|
| 205 |
+
if "num_beams" not in gen_kwargs:
|
| 206 |
+
gen_kwargs["num_beams"] = 1
|
| 207 |
+
try:
|
| 208 |
+
cont = self.model.generate(
|
| 209 |
+
**inputs,
|
| 210 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 211 |
+
temperature=gen_kwargs["temperature"],
|
| 212 |
+
top_p=gen_kwargs["top_p"],
|
| 213 |
+
num_beams=gen_kwargs["num_beams"],
|
| 214 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 215 |
+
)
|
| 216 |
+
except Exception as e:
|
| 217 |
+
eval_logger.error(f"Error {e} in generating")
|
| 218 |
+
cont = ""
|
| 219 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0].strip()
|
| 220 |
+
res.append(text_outputs)
|
| 221 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 222 |
+
pbar.update(1)
|
| 223 |
+
# reorder this group of results back to original unsorted form
|
| 224 |
+
res = re_ords.get_original(res)
|
| 225 |
+
|
| 226 |
+
pbar.close()
|
| 227 |
+
return res
|
| 228 |
+
|
| 229 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 230 |
+
raise NotImplementedError("TODO: Implement multi-round generation for InstructBlip")
|
LaViDa/eval/lmms_eval/models/internvideo2.py
ADDED
|
@@ -0,0 +1,366 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import os
|
| 3 |
+
from typing import List, Tuple
|
| 4 |
+
|
| 5 |
+
import decord
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import torchvision.transforms as T
|
| 9 |
+
from accelerate import Accelerator, DistributedType
|
| 10 |
+
from decord import VideoReader, cpu
|
| 11 |
+
|
| 12 |
+
decord.bridge.set_bridge("torch")
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
from PIL import Image
|
| 15 |
+
from tqdm import tqdm
|
| 16 |
+
from transformers import AutoModel, AutoTokenizer
|
| 17 |
+
|
| 18 |
+
from lmms_eval.api.instance import Instance
|
| 19 |
+
from lmms_eval.api.model import lmms
|
| 20 |
+
from lmms_eval.api.registry import register_model
|
| 21 |
+
|
| 22 |
+
eval_logger = logging.getLogger("eval_logger")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
from datetime import timedelta
|
| 26 |
+
|
| 27 |
+
from accelerate.state import AcceleratorState
|
| 28 |
+
from accelerate.utils import InitProcessGroupKwargs
|
| 29 |
+
|
| 30 |
+
DEFAULT_GEN_KWARGS = dict(
|
| 31 |
+
num_beams=1,
|
| 32 |
+
max_new_tokens=1024,
|
| 33 |
+
do_sample=False,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
# def get_index(num_frames, num_segments):
|
| 37 |
+
# seg_size = float(num_frames - 1) / num_segments
|
| 38 |
+
# start = int(seg_size / 2)
|
| 39 |
+
# offsets = np.array([
|
| 40 |
+
# start + int(np.round(seg_size * idx)) for idx in range(num_segments)
|
| 41 |
+
# ])
|
| 42 |
+
# return offsets
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def get_index(max_frame, num_segments, fps, first_idx=0, bound=None):
|
| 46 |
+
if bound:
|
| 47 |
+
start, end = bound[0], bound[1]
|
| 48 |
+
if start is None:
|
| 49 |
+
start, end = -100000, 100000
|
| 50 |
+
else:
|
| 51 |
+
start, end = -100000, 100000
|
| 52 |
+
start_idx = max(first_idx, round(start * fps))
|
| 53 |
+
end_idx = min(round(end * fps), max_frame)
|
| 54 |
+
seg_size = float(end_idx - start_idx) / num_segments
|
| 55 |
+
frame_indices = np.array([int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)])
|
| 56 |
+
return frame_indices
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def load_image(image_path, resolution=224, hd_num=6):
|
| 60 |
+
image = Image.open(image_path).convert("RGB")
|
| 61 |
+
image_tensor = T.PILToTensor()(image).unsqueeze(0)
|
| 62 |
+
image_tensor = HD_transform_no_padding(image_tensor.float(), image_size=resolution, hd_num=hd_num)
|
| 63 |
+
T_, C, H, W = image_tensor.shape
|
| 64 |
+
|
| 65 |
+
mean = (0.485, 0.456, 0.406)
|
| 66 |
+
std = (0.229, 0.224, 0.225)
|
| 67 |
+
|
| 68 |
+
transform = T.Compose([T.Lambda(lambda x: x.float().div(255.0)), T.Normalize(mean, std)])
|
| 69 |
+
image_tensor = transform(image_tensor).cuda()
|
| 70 |
+
|
| 71 |
+
sub_img = image_tensor.reshape(1, T_, 3, H // resolution, resolution, W // resolution, resolution).permute(0, 3, 5, 1, 2, 4, 6).reshape(-1, T_, 3, resolution, resolution).contiguous()
|
| 72 |
+
|
| 73 |
+
glb_img = F.interpolate(image_tensor.float(), size=(resolution, resolution), mode="bicubic", align_corners=False).to(sub_img.dtype).unsqueeze(0)
|
| 74 |
+
|
| 75 |
+
image_tensor = torch.cat([sub_img, glb_img]) # .unsqueeze(0)
|
| 76 |
+
return image_tensor
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def load_video(video_path, num_segments=16, return_msg=False, resolution=224, hd_num=6, padding=False):
|
| 80 |
+
vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
|
| 81 |
+
num_frames = len(vr) - 1
|
| 82 |
+
|
| 83 |
+
frame_indices = get_index(max_frame=num_frames, num_segments=num_segments, fps=float(vr.get_avg_fps()), first_idx=0, bound=None)
|
| 84 |
+
mean = (0.485, 0.456, 0.406)
|
| 85 |
+
std = (0.229, 0.224, 0.225)
|
| 86 |
+
|
| 87 |
+
transform = T.Compose([T.Lambda(lambda x: x.float().div(255.0)), T.Normalize(mean, std)])
|
| 88 |
+
|
| 89 |
+
frames = vr.get_batch(frame_indices)
|
| 90 |
+
frames = frames.permute(0, 3, 1, 2)
|
| 91 |
+
|
| 92 |
+
if padding:
|
| 93 |
+
frames = HD_transform_padding(frames.float(), image_size=resolution, hd_num=hd_num)
|
| 94 |
+
else:
|
| 95 |
+
frames = HD_transform_no_padding(frames.float(), image_size=resolution, hd_num=hd_num)
|
| 96 |
+
|
| 97 |
+
frames = transform(frames)
|
| 98 |
+
T_, C, H, W = frames.shape
|
| 99 |
+
|
| 100 |
+
sub_img = frames.reshape(1, T_, 3, H // resolution, resolution, W // resolution, resolution).permute(0, 3, 5, 1, 2, 4, 6).reshape(-1, T_, 3, resolution, resolution).contiguous()
|
| 101 |
+
|
| 102 |
+
glb_img = F.interpolate(frames.float(), size=(resolution, resolution), mode="bicubic", align_corners=False).to(sub_img.dtype).unsqueeze(0)
|
| 103 |
+
|
| 104 |
+
frames = torch.cat([sub_img, glb_img]).unsqueeze(0)
|
| 105 |
+
|
| 106 |
+
if return_msg:
|
| 107 |
+
fps = float(vr.get_avg_fps())
|
| 108 |
+
sec = ", ".join([str(round(f / fps, 1)) for f in frame_indices])
|
| 109 |
+
# " " should be added in the start and end
|
| 110 |
+
msg = f"The video contains {len(frame_indices)} frames sampled at {sec} seconds."
|
| 111 |
+
return frames, msg
|
| 112 |
+
else:
|
| 113 |
+
return frames
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def HD_transform_padding(frames, image_size=224, hd_num=6):
|
| 117 |
+
def _padding_224(frames):
|
| 118 |
+
_, _, H, W = frames.shape
|
| 119 |
+
tar = int(np.ceil(H / 224) * 224)
|
| 120 |
+
top_padding = (tar - H) // 2
|
| 121 |
+
bottom_padding = tar - H - top_padding
|
| 122 |
+
left_padding = 0
|
| 123 |
+
right_padding = 0
|
| 124 |
+
|
| 125 |
+
padded_frames = F.pad(frames, pad=[left_padding, right_padding, top_padding, bottom_padding], mode="constant", value=255)
|
| 126 |
+
return padded_frames
|
| 127 |
+
|
| 128 |
+
_, _, H, W = frames.shape
|
| 129 |
+
trans = False
|
| 130 |
+
if W < H:
|
| 131 |
+
frames = frames.flip(-2, -1)
|
| 132 |
+
trans = True
|
| 133 |
+
width, height = H, W
|
| 134 |
+
else:
|
| 135 |
+
width, height = W, H
|
| 136 |
+
|
| 137 |
+
ratio = width / height
|
| 138 |
+
scale = 1
|
| 139 |
+
while scale * np.ceil(scale / ratio) <= hd_num:
|
| 140 |
+
scale += 1
|
| 141 |
+
scale -= 1
|
| 142 |
+
new_w = int(scale * image_size)
|
| 143 |
+
new_h = int(new_w / ratio)
|
| 144 |
+
|
| 145 |
+
resized_frames = F.interpolate(frames, size=(new_h, new_w), mode="bicubic", align_corners=False)
|
| 146 |
+
padded_frames = _padding_224(resized_frames)
|
| 147 |
+
|
| 148 |
+
if trans:
|
| 149 |
+
padded_frames = padded_frames.flip(-2, -1)
|
| 150 |
+
|
| 151 |
+
return padded_frames
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
| 155 |
+
best_ratio_diff = float("inf")
|
| 156 |
+
best_ratio = (1, 1)
|
| 157 |
+
area = width * height
|
| 158 |
+
for ratio in target_ratios:
|
| 159 |
+
target_aspect_ratio = ratio[0] / ratio[1]
|
| 160 |
+
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
| 161 |
+
if ratio_diff < best_ratio_diff:
|
| 162 |
+
best_ratio_diff = ratio_diff
|
| 163 |
+
best_ratio = ratio
|
| 164 |
+
elif ratio_diff == best_ratio_diff:
|
| 165 |
+
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
| 166 |
+
best_ratio = ratio
|
| 167 |
+
return best_ratio
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def HD_transform_no_padding(frames, image_size=224, hd_num=6, fix_ratio=(2, 1)):
|
| 171 |
+
min_num = 1
|
| 172 |
+
max_num = hd_num
|
| 173 |
+
_, _, orig_height, orig_width = frames.shape
|
| 174 |
+
aspect_ratio = orig_width / orig_height
|
| 175 |
+
|
| 176 |
+
# calculate the existing video aspect ratio
|
| 177 |
+
target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
|
| 178 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 179 |
+
|
| 180 |
+
# find the closest aspect ratio to the target
|
| 181 |
+
if fix_ratio:
|
| 182 |
+
target_aspect_ratio = fix_ratio
|
| 183 |
+
else:
|
| 184 |
+
target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
| 185 |
+
|
| 186 |
+
# calculate the target width and height
|
| 187 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 188 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 189 |
+
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
| 190 |
+
|
| 191 |
+
# resize the frames
|
| 192 |
+
resized_frame = F.interpolate(frames, size=(target_height, target_width), mode="bicubic", align_corners=False)
|
| 193 |
+
return resized_frame
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@register_model("InternVideo2")
|
| 197 |
+
class InternVideo2(lmms):
|
| 198 |
+
def __init__(
|
| 199 |
+
self,
|
| 200 |
+
pretrained: str = "OpenGVLab/InternVideo2_chat_8B_HD",
|
| 201 |
+
modality: str = "video",
|
| 202 |
+
device: str = "cuda:0",
|
| 203 |
+
device_map: str = "cuda:0",
|
| 204 |
+
batch_size: str = "1",
|
| 205 |
+
num_segments: str = "8",
|
| 206 |
+
hd_num: str = "6",
|
| 207 |
+
**kwargs,
|
| 208 |
+
):
|
| 209 |
+
super().__init__()
|
| 210 |
+
self.path = pretrained
|
| 211 |
+
self.instruction = "Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons.\n"
|
| 212 |
+
|
| 213 |
+
self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True, use_fast=False)
|
| 214 |
+
self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, trust_remote_code=True).eval().cuda()
|
| 215 |
+
batch_size = int(batch_size)
|
| 216 |
+
self.num_segments = int(num_segments)
|
| 217 |
+
self.hd_num = int(hd_num)
|
| 218 |
+
assert batch_size == 1, f"Batch size should be 1 for InternVideo2, but got {batch_size}."
|
| 219 |
+
self.batch_size_per_gpu = batch_size
|
| 220 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 221 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 222 |
+
self.accelerator = accelerator
|
| 223 |
+
if accelerator.num_processes > 1:
|
| 224 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 225 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 226 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 227 |
+
self._device = torch.device(device)
|
| 228 |
+
self.device_map = device_map
|
| 229 |
+
else:
|
| 230 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 231 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 232 |
+
|
| 233 |
+
if accelerator.num_processes > 1:
|
| 234 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 235 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 236 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 237 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 238 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 239 |
+
kwargs = {
|
| 240 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 241 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 242 |
+
}
|
| 243 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 244 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 245 |
+
|
| 246 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 247 |
+
self._model = accelerator.prepare(self.model)
|
| 248 |
+
else:
|
| 249 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 250 |
+
self.accelerator = accelerator
|
| 251 |
+
if self.accelerator.is_local_main_process:
|
| 252 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 253 |
+
self._rank = self.accelerator.local_process_index
|
| 254 |
+
self._world_size = self.accelerator.num_processes
|
| 255 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 256 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 257 |
+
self._rank = 0
|
| 258 |
+
self._world_size = 1
|
| 259 |
+
else:
|
| 260 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 261 |
+
self.model.to(self._device)
|
| 262 |
+
self._rank = 0
|
| 263 |
+
self._world_size = 1
|
| 264 |
+
|
| 265 |
+
self.modality = modality
|
| 266 |
+
|
| 267 |
+
@property
|
| 268 |
+
def config(self):
|
| 269 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 270 |
+
return self._config
|
| 271 |
+
|
| 272 |
+
@property
|
| 273 |
+
def tokenizer(self):
|
| 274 |
+
return self._tokenizer
|
| 275 |
+
|
| 276 |
+
@property
|
| 277 |
+
def model(self):
|
| 278 |
+
# returns the model, unwrapping it if using Accelerate
|
| 279 |
+
if hasattr(self, "accelerator"):
|
| 280 |
+
return self.accelerator.unwrap_model(self._model)
|
| 281 |
+
else:
|
| 282 |
+
return self._model
|
| 283 |
+
|
| 284 |
+
@property
|
| 285 |
+
def batch_size(self):
|
| 286 |
+
return self.batch_size_per_gpu
|
| 287 |
+
|
| 288 |
+
@property
|
| 289 |
+
def device(self):
|
| 290 |
+
return self._device
|
| 291 |
+
|
| 292 |
+
@property
|
| 293 |
+
def rank(self):
|
| 294 |
+
return self._rank
|
| 295 |
+
|
| 296 |
+
@property
|
| 297 |
+
def world_size(self):
|
| 298 |
+
return self._world_size
|
| 299 |
+
|
| 300 |
+
def flatten(self, input):
|
| 301 |
+
new_list = []
|
| 302 |
+
for i in input:
|
| 303 |
+
for j in i:
|
| 304 |
+
new_list.append(j)
|
| 305 |
+
return new_list
|
| 306 |
+
|
| 307 |
+
def generate_until(self, requests) -> List[str]:
|
| 308 |
+
res = []
|
| 309 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 310 |
+
|
| 311 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 312 |
+
if "until" in gen_kwargs:
|
| 313 |
+
gen_kwargs.pop("until")
|
| 314 |
+
for k, v in DEFAULT_GEN_KWARGS.items():
|
| 315 |
+
if k not in gen_kwargs:
|
| 316 |
+
gen_kwargs[k] = v
|
| 317 |
+
|
| 318 |
+
pop_keys = []
|
| 319 |
+
for k, v in gen_kwargs.items():
|
| 320 |
+
if k not in DEFAULT_GEN_KWARGS:
|
| 321 |
+
pop_keys.append(k)
|
| 322 |
+
|
| 323 |
+
for k in pop_keys:
|
| 324 |
+
gen_kwargs.pop(k)
|
| 325 |
+
|
| 326 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 327 |
+
visuals = self.flatten(visuals)
|
| 328 |
+
if self.modality == "image":
|
| 329 |
+
image_path = visuals[0]
|
| 330 |
+
pixel_values = load_image(image_path, resolution=224, hd_num=self.hd_num)
|
| 331 |
+
pixel_values = pixel_values.to(torch.bfloat16).cuda()
|
| 332 |
+
question = contexts
|
| 333 |
+
response, history = self.model.chat(self.tokenizer, msg="", user_prompt=question, media_type="image", media_tensor=pixel_values, instruction=None, chat_history=[], return_history=True, **gen_kwargs)
|
| 334 |
+
elif self.modality == "video":
|
| 335 |
+
assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos. [META-INFO]{visuals}"
|
| 336 |
+
video_path = visuals[0]
|
| 337 |
+
if "mvbench" in task:
|
| 338 |
+
answer_prompt = "Best Option:("
|
| 339 |
+
else:
|
| 340 |
+
answer_prompt = None
|
| 341 |
+
pixel_values = load_video(video_path, num_segments=self.num_segments, return_msg=False, resolution=224, hd_num=self.hd_num)
|
| 342 |
+
pixel_values = pixel_values.to(torch.bfloat16).cuda()
|
| 343 |
+
question = self.instruction + contexts
|
| 344 |
+
response, history = self.model.chat(
|
| 345 |
+
self.tokenizer,
|
| 346 |
+
msg="",
|
| 347 |
+
user_prompt=question,
|
| 348 |
+
media_type="video",
|
| 349 |
+
media_tensor=pixel_values,
|
| 350 |
+
instruction=self.instruction,
|
| 351 |
+
chat_history=[],
|
| 352 |
+
return_history=True,
|
| 353 |
+
generation_config=gen_kwargs,
|
| 354 |
+
answer_prompt=answer_prompt,
|
| 355 |
+
debug_conv=False,
|
| 356 |
+
)
|
| 357 |
+
res.append(response)
|
| 358 |
+
pbar.update(1)
|
| 359 |
+
pbar.close()
|
| 360 |
+
return res
|
| 361 |
+
|
| 362 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 363 |
+
assert False, "Not implemented yet."
|
| 364 |
+
|
| 365 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 366 |
+
raise NotImplementedError("TODO: Implement multi-round generation for InternVideo2")
|
LaViDa/eval/lmms_eval/models/internvideo2_5.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
from datetime import timedelta
|
| 3 |
+
from typing import List, Tuple
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
import torchvision.transforms as T
|
| 8 |
+
from accelerate import Accelerator, DistributedType
|
| 9 |
+
from accelerate.state import AcceleratorState
|
| 10 |
+
from accelerate.utils import InitProcessGroupKwargs
|
| 11 |
+
from decord import VideoReader, cpu
|
| 12 |
+
from PIL import Image
|
| 13 |
+
from torchvision.transforms.functional import InterpolationMode
|
| 14 |
+
from tqdm import tqdm
|
| 15 |
+
from transformers import AutoModel, AutoTokenizer
|
| 16 |
+
|
| 17 |
+
from lmms_eval.api.instance import Instance
|
| 18 |
+
from lmms_eval.api.model import lmms
|
| 19 |
+
from lmms_eval.api.registry import register_model
|
| 20 |
+
|
| 21 |
+
eval_logger = logging.getLogger("eval_logger")
|
| 22 |
+
|
| 23 |
+
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
| 24 |
+
IMAGENET_STD = (0.229, 0.224, 0.225)
|
| 25 |
+
|
| 26 |
+
DEFAULT_GEN_KWARGS = dict(
|
| 27 |
+
num_beams=1,
|
| 28 |
+
max_new_tokens=1024,
|
| 29 |
+
do_sample=False,
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def build_transform(input_size):
|
| 34 |
+
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
|
| 35 |
+
transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
|
| 36 |
+
return transform
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
| 40 |
+
best_ratio_diff = float("inf")
|
| 41 |
+
best_ratio = (1, 1)
|
| 42 |
+
area = width * height
|
| 43 |
+
for ratio in target_ratios:
|
| 44 |
+
target_aspect_ratio = ratio[0] / ratio[1]
|
| 45 |
+
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
| 46 |
+
if ratio_diff < best_ratio_diff:
|
| 47 |
+
best_ratio_diff = ratio_diff
|
| 48 |
+
best_ratio = ratio
|
| 49 |
+
elif ratio_diff == best_ratio_diff:
|
| 50 |
+
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
| 51 |
+
best_ratio = ratio
|
| 52 |
+
return best_ratio
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
|
| 56 |
+
orig_width, orig_height = image.size
|
| 57 |
+
aspect_ratio = orig_width / orig_height
|
| 58 |
+
|
| 59 |
+
# calculate the existing image aspect ratio
|
| 60 |
+
target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
|
| 61 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 62 |
+
|
| 63 |
+
# find the closest aspect ratio to the target
|
| 64 |
+
target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
| 65 |
+
|
| 66 |
+
# calculate the target width and height
|
| 67 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 68 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 69 |
+
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
| 70 |
+
|
| 71 |
+
# resize the image
|
| 72 |
+
resized_img = image.resize((target_width, target_height))
|
| 73 |
+
processed_images = []
|
| 74 |
+
for i in range(blocks):
|
| 75 |
+
box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
|
| 76 |
+
# split the image
|
| 77 |
+
split_img = resized_img.crop(box)
|
| 78 |
+
processed_images.append(split_img)
|
| 79 |
+
assert len(processed_images) == blocks
|
| 80 |
+
if use_thumbnail and len(processed_images) != 1:
|
| 81 |
+
thumbnail_img = image.resize((image_size, image_size))
|
| 82 |
+
processed_images.append(thumbnail_img)
|
| 83 |
+
return processed_images
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def load_image(image, input_size=448, max_num=6):
|
| 87 |
+
transform = build_transform(input_size=input_size)
|
| 88 |
+
images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
|
| 89 |
+
pixel_values = [transform(image) for image in images]
|
| 90 |
+
pixel_values = torch.stack(pixel_values)
|
| 91 |
+
return pixel_values
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
|
| 95 |
+
if bound:
|
| 96 |
+
start, end = bound[0], bound[1]
|
| 97 |
+
else:
|
| 98 |
+
start, end = -100000, 100000
|
| 99 |
+
start_idx = max(first_idx, round(start * fps))
|
| 100 |
+
end_idx = min(round(end * fps), max_frame)
|
| 101 |
+
seg_size = float(end_idx - start_idx) / num_segments
|
| 102 |
+
frame_indices = [int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)]
|
| 103 |
+
return frame_indices
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32, media_dict=None):
|
| 107 |
+
if type(video_path) == str:
|
| 108 |
+
vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
|
| 109 |
+
else:
|
| 110 |
+
vr = VideoReader(video_path[0], ctx=cpu(0), num_threads=1)
|
| 111 |
+
|
| 112 |
+
max_frame = len(vr) - 1
|
| 113 |
+
fps = float(vr.get_avg_fps())
|
| 114 |
+
|
| 115 |
+
pixel_values_list, num_patches_list = [], []
|
| 116 |
+
transform = build_transform(input_size=input_size)
|
| 117 |
+
frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
|
| 118 |
+
if media_dict is not None and media_dict["video_read_type"] == "decord_last":
|
| 119 |
+
frame_indices = frame_indices + [max_frame for _ in range(4)] # add last 4 frames
|
| 120 |
+
|
| 121 |
+
frame_indices = np.array(frame_indices)
|
| 122 |
+
|
| 123 |
+
for frame_index in frame_indices:
|
| 124 |
+
img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
|
| 125 |
+
img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=False, max_num=max_num)
|
| 126 |
+
pixel_values = [transform(tile) for tile in img]
|
| 127 |
+
pixel_values = torch.stack(pixel_values)
|
| 128 |
+
num_patches_list.append(pixel_values.shape[0])
|
| 129 |
+
pixel_values_list.append(pixel_values)
|
| 130 |
+
pixel_values = torch.cat(pixel_values_list)
|
| 131 |
+
return pixel_values, num_patches_list
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
@register_model("internvideo2_5")
|
| 135 |
+
class InternVideo2_5(lmms):
|
| 136 |
+
def __init__(
|
| 137 |
+
self,
|
| 138 |
+
pretrained: str = "OpenGVLab/InternVideo2_5_Chat_8B",
|
| 139 |
+
modality: str = "video",
|
| 140 |
+
device: str = "cuda:0",
|
| 141 |
+
device_map: str = "cuda:0",
|
| 142 |
+
batch_size: str = "1",
|
| 143 |
+
max_frames_num: int = 32,
|
| 144 |
+
**kwargs,
|
| 145 |
+
):
|
| 146 |
+
super().__init__()
|
| 147 |
+
self.max_frames_num = max_frames_num
|
| 148 |
+
self.path = pretrained
|
| 149 |
+
self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, use_flash_attn=True, trust_remote_code=True).eval().cuda()
|
| 150 |
+
self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True)
|
| 151 |
+
|
| 152 |
+
batch_size = int(batch_size)
|
| 153 |
+
assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}."
|
| 154 |
+
self.batch_size_per_gpu = batch_size
|
| 155 |
+
|
| 156 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 157 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 158 |
+
self.accelerator = accelerator
|
| 159 |
+
if accelerator.num_processes > 1:
|
| 160 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 161 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 162 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 163 |
+
self._device = torch.device(device)
|
| 164 |
+
self.device_map = device_map
|
| 165 |
+
else:
|
| 166 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 167 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 168 |
+
|
| 169 |
+
if accelerator.num_processes > 1:
|
| 170 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 171 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 172 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 173 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 174 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 175 |
+
kwargs = {
|
| 176 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 177 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 178 |
+
}
|
| 179 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 180 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 181 |
+
|
| 182 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 183 |
+
self._model = accelerator.prepare(self.model)
|
| 184 |
+
else:
|
| 185 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 186 |
+
self.accelerator = accelerator
|
| 187 |
+
if self.accelerator.is_local_main_process:
|
| 188 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 189 |
+
self._rank = self.accelerator.local_process_index
|
| 190 |
+
self._world_size = self.accelerator.num_processes
|
| 191 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 192 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 193 |
+
self._rank = 0
|
| 194 |
+
self._word_size = 1
|
| 195 |
+
else:
|
| 196 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 197 |
+
self.model.to(self._device)
|
| 198 |
+
self._rank = 0
|
| 199 |
+
self._world_size = 1
|
| 200 |
+
|
| 201 |
+
self.modality = modality
|
| 202 |
+
|
| 203 |
+
@property
|
| 204 |
+
def config(self):
|
| 205 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 206 |
+
return self._config
|
| 207 |
+
|
| 208 |
+
@property
|
| 209 |
+
def tokenizer(self):
|
| 210 |
+
return self._tokenizer
|
| 211 |
+
|
| 212 |
+
@property
|
| 213 |
+
def model(self):
|
| 214 |
+
# returns the model, unwrapping it if using Accelerate
|
| 215 |
+
if hasattr(self, "accelerator"):
|
| 216 |
+
return self.accelerator.unwrap_model(self._model)
|
| 217 |
+
else:
|
| 218 |
+
return self._model
|
| 219 |
+
|
| 220 |
+
@property
|
| 221 |
+
def batch_size(self):
|
| 222 |
+
return self.batch_size_per_gpu
|
| 223 |
+
|
| 224 |
+
@property
|
| 225 |
+
def device(self):
|
| 226 |
+
return self._device
|
| 227 |
+
|
| 228 |
+
@property
|
| 229 |
+
def rank(self):
|
| 230 |
+
return self._rank
|
| 231 |
+
|
| 232 |
+
@property
|
| 233 |
+
def world_size(self):
|
| 234 |
+
return self._world_size
|
| 235 |
+
|
| 236 |
+
def flatten(self, input):
|
| 237 |
+
new_list = []
|
| 238 |
+
for i in input:
|
| 239 |
+
for j in i:
|
| 240 |
+
new_list.append(j)
|
| 241 |
+
return new_list
|
| 242 |
+
|
| 243 |
+
def generate_until(self, requests) -> List[str]:
|
| 244 |
+
res = []
|
| 245 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 246 |
+
|
| 247 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 248 |
+
if "until" in gen_kwargs:
|
| 249 |
+
gen_kwargs.pop("until")
|
| 250 |
+
for k, v in DEFAULT_GEN_KWARGS.items():
|
| 251 |
+
if k not in gen_kwargs:
|
| 252 |
+
gen_kwargs[k] = v
|
| 253 |
+
|
| 254 |
+
pop_keys = []
|
| 255 |
+
for k, v in gen_kwargs.items():
|
| 256 |
+
if k not in DEFAULT_GEN_KWARGS:
|
| 257 |
+
pop_keys.append(k)
|
| 258 |
+
|
| 259 |
+
for k in pop_keys:
|
| 260 |
+
gen_kwargs.pop(k)
|
| 261 |
+
|
| 262 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 263 |
+
visuals = self.flatten(visuals)
|
| 264 |
+
if self.modality == "image":
|
| 265 |
+
if visuals:
|
| 266 |
+
visuals = [load_image(visual).to(torch.bfloat16).cuda() for visual in visuals]
|
| 267 |
+
pixel_values = torch.cat(visuals, dim=0)
|
| 268 |
+
num_patches_list = [visual.size(0) for visual in visuals]
|
| 269 |
+
image_tokens = ["<image>"] * len(visuals)
|
| 270 |
+
image_tokens = " ".join(image_tokens)
|
| 271 |
+
contexts = image_tokens + "\n" + contexts
|
| 272 |
+
else:
|
| 273 |
+
pixel_values = None
|
| 274 |
+
num_patch_list = None
|
| 275 |
+
response, history = self.model.chat(self.tokenizer, pixel_values, contexts, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
|
| 276 |
+
elif self.modality == "video":
|
| 277 |
+
# assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos. {visuals}"
|
| 278 |
+
video_path = visuals[0]
|
| 279 |
+
if len(visuals) > 1:
|
| 280 |
+
assert len(visuals) == 2, visuals
|
| 281 |
+
media_dict = visuals[1]
|
| 282 |
+
else:
|
| 283 |
+
media_dict = {"video_read_type": "decord"}
|
| 284 |
+
pixel_values, num_patches_list = load_video(video_path, num_segments=self.max_frames_num, max_num=1, media_dict=media_dict)
|
| 285 |
+
pixel_values = pixel_values.to(torch.bfloat16).cuda()
|
| 286 |
+
video_prefix = "".join([f"Frame{i+1}: <image>\n" for i in range(len(num_patches_list))])
|
| 287 |
+
question = video_prefix + contexts
|
| 288 |
+
response, history = self.model.chat(self.tokenizer, pixel_values, question, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
|
| 289 |
+
res.append(response)
|
| 290 |
+
pbar.update(1)
|
| 291 |
+
pbar.close()
|
| 292 |
+
return res
|
| 293 |
+
|
| 294 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 295 |
+
assert False, "Not implemented yet."
|
LaViDa/eval/lmms_eval/models/internvl.py
ADDED
|
@@ -0,0 +1,496 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import os
|
| 3 |
+
import subprocess
|
| 4 |
+
from datetime import timedelta
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import List, Optional, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import requests
|
| 10 |
+
import torch
|
| 11 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 12 |
+
from accelerate.state import AcceleratorState
|
| 13 |
+
from huggingface_hub import snapshot_download
|
| 14 |
+
from PIL import Image
|
| 15 |
+
from tqdm import tqdm
|
| 16 |
+
from transformers import AutoConfig
|
| 17 |
+
|
| 18 |
+
from lmms_eval import utils
|
| 19 |
+
from lmms_eval.api.instance import Instance
|
| 20 |
+
from lmms_eval.api.model import lmms
|
| 21 |
+
from lmms_eval.api.registry import register_model
|
| 22 |
+
from lmms_eval.utils import stop_sequences_criteria
|
| 23 |
+
|
| 24 |
+
wd = Path(__file__).parent.parent.parent.resolve()
|
| 25 |
+
import sys
|
| 26 |
+
|
| 27 |
+
sys.path.append(os.path.join(str(wd), "InternVL", "internvl_chat"))
|
| 28 |
+
from loguru import logger as eval_logger
|
| 29 |
+
|
| 30 |
+
if not hasattr(eval_logger, "internvl_warning_logged"):
|
| 31 |
+
eval_logger.internvl_warning_logged = False
|
| 32 |
+
|
| 33 |
+
try:
|
| 34 |
+
from internvl.model.internlm2.modeling_internlm2 import InternLM2ForCausalLM
|
| 35 |
+
from internvl.model.internvl_chat import InternVLChatModel
|
| 36 |
+
from internvl.model.internvl_chat.configuration_internvl_chat import (
|
| 37 |
+
InternVLChatConfig,
|
| 38 |
+
)
|
| 39 |
+
from internvl.model.internvl_chat.modeling_intern_vit import InternVisionModel
|
| 40 |
+
from internvl.train.dataset import build_transform, dynamic_preprocess
|
| 41 |
+
except ImportError:
|
| 42 |
+
eval_logger.debug("InternVL is not installed. Please install InternVL to use this model.")
|
| 43 |
+
if not eval_logger.internvl_warning_logged:
|
| 44 |
+
eval_logger.debug("InternVL is not installed. Please install InternVL to use this model.")
|
| 45 |
+
eval_logger.internvl_warning_logged = True
|
| 46 |
+
|
| 47 |
+
import re
|
| 48 |
+
import warnings
|
| 49 |
+
from typing import Any, List, Optional, Tuple, Union
|
| 50 |
+
|
| 51 |
+
import torch.utils.checkpoint
|
| 52 |
+
from huggingface_hub import snapshot_download
|
| 53 |
+
from peft import LoraConfig, get_peft_model
|
| 54 |
+
from torch import nn
|
| 55 |
+
from torch.nn import CrossEntropyLoss
|
| 56 |
+
from transformers import (
|
| 57 |
+
AutoModel,
|
| 58 |
+
AutoTokenizer,
|
| 59 |
+
GenerationConfig,
|
| 60 |
+
LlamaForCausalLM,
|
| 61 |
+
LlamaTokenizer,
|
| 62 |
+
)
|
| 63 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 64 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@register_model("internvl")
|
| 68 |
+
class InternVLChat(lmms):
|
| 69 |
+
# config_class = InternVLChatConfig
|
| 70 |
+
main_input_name = "pixel_values"
|
| 71 |
+
_no_split_modules = ["InternVisionEncoderLayer", "LlamaDecoderLayer"]
|
| 72 |
+
|
| 73 |
+
"""
|
| 74 |
+
0. Install lmms-eval
|
| 75 |
+
cd lmms-eval
|
| 76 |
+
pip install -e .
|
| 77 |
+
|
| 78 |
+
How to Install InternVL:
|
| 79 |
+
1. Clone the InternVL repository:
|
| 80 |
+
git clone https://github.com/OpenGVLab/InternVL.git
|
| 81 |
+
|
| 82 |
+
2. Install the requirements:
|
| 83 |
+
pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu118
|
| 84 |
+
|
| 85 |
+
3. Install flash-attn==2.3.6:
|
| 86 |
+
pip install flash-attn==2.3.6 --no-build-isolation
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
"""
|
| 90 |
+
How to download the pretrained model:
|
| 91 |
+
1. Download the pretrained model from hugginface:
|
| 92 |
+
cd pretrained/
|
| 93 |
+
# pip install -U huggingface_hub
|
| 94 |
+
huggingface-cli download --resume-download --local-dir-use-symlinks False OpenGVLab/InternVL-Chat-V1-5 --local-dir InternVL-Chat-V1-5
|
| 95 |
+
|
| 96 |
+
2. the pretrained model should be in the following directory:
|
| 97 |
+
pretrained
|
| 98 |
+
└── InternVL-Chat-V1-5
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
#
|
| 102 |
+
# The above steps can be optional, I add snapshot download, so now can just use hf repo_id
|
| 103 |
+
# model_args pretrained=OpenGVLab/InternVL-Chat-V1-5
|
| 104 |
+
#
|
| 105 |
+
|
| 106 |
+
"""
|
| 107 |
+
InternVL-Chat-V1-5 Model for OpenGVLab https://github.com/OpenGVLab/InternVL/blob/main/internvl_chat/internvl/model/internvl_chat/modeling_internvl_chat.py
|
| 108 |
+
Example usage:
|
| 109 |
+
|
| 110 |
+
accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
|
| 111 |
+
--model internvl \
|
| 112 |
+
--model_args pretrained=OpenGVLab/InternVL-Chat-V1-5 \
|
| 113 |
+
--tasks llava_wilder_small \
|
| 114 |
+
--batch_size 1 \
|
| 115 |
+
--output_path ./logs/ \
|
| 116 |
+
--log_samples
|
| 117 |
+
"""
|
| 118 |
+
|
| 119 |
+
def __init__(
|
| 120 |
+
self,
|
| 121 |
+
config=None,
|
| 122 |
+
pretrained: str = "OpenGVLab/InternVL-Chat-V1-5",
|
| 123 |
+
truncation: Optional[bool] = True,
|
| 124 |
+
device: Optional[str] = "cuda:0",
|
| 125 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 126 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 127 |
+
trust_remote_code: Optional[bool] = False,
|
| 128 |
+
revision=None,
|
| 129 |
+
device_map="cuda:0",
|
| 130 |
+
conv_template="vicuna_v1",
|
| 131 |
+
use_cache=True,
|
| 132 |
+
truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 133 |
+
customized_config=None, # ends in json
|
| 134 |
+
dynamic=True,
|
| 135 |
+
load_in_8bit=False,
|
| 136 |
+
vision_model=None,
|
| 137 |
+
language_model=None,
|
| 138 |
+
max_num=12,
|
| 139 |
+
**kwargs,
|
| 140 |
+
) -> None:
|
| 141 |
+
super().__init__()
|
| 142 |
+
|
| 143 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 144 |
+
|
| 145 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 146 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 147 |
+
if accelerator.num_processes > 1:
|
| 148 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 149 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 150 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 151 |
+
self._device = torch.device(device)
|
| 152 |
+
self.device_map = device_map
|
| 153 |
+
else:
|
| 154 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 155 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 156 |
+
|
| 157 |
+
self.dynamic = dynamic # dynamic image_size
|
| 158 |
+
self.max_num = max_num
|
| 159 |
+
if accelerator.is_main_process:
|
| 160 |
+
cache_dir = snapshot_download(repo_id=pretrained, cache_dir="cache_dir", local_dir="cache_dir", local_dir_use_symlinks=False)
|
| 161 |
+
accelerator.wait_for_everyone()
|
| 162 |
+
# So what I did is that I let main process to download the repo, and then
|
| 163 |
+
# other process can just simply read from this repo
|
| 164 |
+
cache_dir = snapshot_download(repo_id=pretrained, cache_dir="cache_dir", local_dir="cache_dir", local_dir_use_symlinks=False)
|
| 165 |
+
config = InternVLChatConfig.from_pretrained(cache_dir)
|
| 166 |
+
tokenizer = AutoTokenizer.from_pretrained(cache_dir, trust_remote_code=True, use_fast=False)
|
| 167 |
+
model = InternVLChatModel.from_pretrained(cache_dir, low_cpu_mem_usage=True, config=config, torch_dtype=torch.bfloat16, load_in_8bit=load_in_8bit).eval()
|
| 168 |
+
if not load_in_8bit:
|
| 169 |
+
model = model.cuda()
|
| 170 |
+
# self.model=model
|
| 171 |
+
# self.device=self._device
|
| 172 |
+
self._tokenizer = tokenizer
|
| 173 |
+
# self.tokenizer=tokenizer
|
| 174 |
+
self._model = model
|
| 175 |
+
self._config = self._model.config
|
| 176 |
+
self.use_thumbnail = self.model.config.use_thumbnail
|
| 177 |
+
self.model.eval()
|
| 178 |
+
self.model.tie_weights()
|
| 179 |
+
self.truncation = truncation
|
| 180 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 181 |
+
self.conv_template = conv_template
|
| 182 |
+
self.use_cache = use_cache
|
| 183 |
+
self.truncate_context = truncate_context
|
| 184 |
+
if accelerator.num_processes > 1:
|
| 185 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 186 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 187 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 188 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 189 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 190 |
+
kwargs = {
|
| 191 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 192 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 193 |
+
}
|
| 194 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 195 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 196 |
+
|
| 197 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 198 |
+
self._model = accelerator.prepare(self.model)
|
| 199 |
+
else:
|
| 200 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 201 |
+
self.accelerator = accelerator
|
| 202 |
+
if self.accelerator.is_local_main_process:
|
| 203 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 204 |
+
self._rank = self.accelerator.local_process_index
|
| 205 |
+
self._world_size = self.accelerator.num_processes
|
| 206 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 207 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 208 |
+
self._rank = 0
|
| 209 |
+
self._world_size = 1
|
| 210 |
+
else:
|
| 211 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 212 |
+
self.model.to(self._device)
|
| 213 |
+
self._rank = 0
|
| 214 |
+
self._world_size = 1
|
| 215 |
+
|
| 216 |
+
# from internvl model
|
| 217 |
+
|
| 218 |
+
self.image_size = config.force_image_size or config.vision_config.image_size
|
| 219 |
+
|
| 220 |
+
def wrap_backbone_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):
|
| 221 |
+
lora_config = LoraConfig(
|
| 222 |
+
r=r,
|
| 223 |
+
target_modules=["attn.qkv", "attn.proj", "mlp.fc1", "mlp.fc2"],
|
| 224 |
+
lora_alpha=lora_alpha,
|
| 225 |
+
lora_dropout=lora_dropout,
|
| 226 |
+
)
|
| 227 |
+
self.vision_model = get_peft_model(self.vision_model, lora_config)
|
| 228 |
+
self.vision_model.print_trainable_parameters()
|
| 229 |
+
|
| 230 |
+
def wrap_llm_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):
|
| 231 |
+
lora_config = LoraConfig(
|
| 232 |
+
r=r, target_modules=["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.down_proj", "mlp.up_proj"], lora_alpha=lora_alpha, lora_dropout=lora_dropout, task_type="CAUSAL_LM"
|
| 233 |
+
)
|
| 234 |
+
self.language_model = get_peft_model(self.language_model, lora_config)
|
| 235 |
+
self.language_model.enable_input_require_grads()
|
| 236 |
+
self.language_model.print_trainable_parameters()
|
| 237 |
+
|
| 238 |
+
def pixel_shuffle(self, x, scale_factor=0.5):
|
| 239 |
+
n, w, h, c = x.size()
|
| 240 |
+
# N, W, H, C --> N, W, H * scale, C // scale
|
| 241 |
+
x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
|
| 242 |
+
# N, W, H * scale, C // scale --> N, H * scale, W, C // scale
|
| 243 |
+
x = x.permute(0, 2, 1, 3).contiguous()
|
| 244 |
+
# N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
|
| 245 |
+
x = x.view(n, int(h * scale_factor), int(w * scale_factor), int(c / (scale_factor * scale_factor)))
|
| 246 |
+
if self.ps_version == "v1":
|
| 247 |
+
warnings.warn("In ps_version 'v1', the height and width have not been swapped back, " "which results in a transposed image.")
|
| 248 |
+
else:
|
| 249 |
+
x = x.permute(0, 2, 1, 3).contiguous()
|
| 250 |
+
return x
|
| 251 |
+
|
| 252 |
+
def noised_embed(self, vit_embeds, noise_alpha=5):
|
| 253 |
+
dims = torch.tensor(vit_embeds.size(1) * vit_embeds.size(2))
|
| 254 |
+
mag_norm = noise_alpha / torch.sqrt(dims)
|
| 255 |
+
noise = torch.zeros_like(vit_embeds).uniform_(-mag_norm, mag_norm)
|
| 256 |
+
return vit_embeds + noise
|
| 257 |
+
|
| 258 |
+
def extract_feature(self, pixel_values):
|
| 259 |
+
if self.select_layer == -1:
|
| 260 |
+
vit_embeds = self.vision_model(pixel_values=pixel_values, output_hidden_states=False, return_dict=True).last_hidden_state
|
| 261 |
+
else:
|
| 262 |
+
vit_embeds = self.vision_model(pixel_values=pixel_values, output_hidden_states=True, return_dict=True).hidden_states[self.select_layer]
|
| 263 |
+
vit_embeds = vit_embeds[:, 1:, :]
|
| 264 |
+
|
| 265 |
+
if self.training and self.neftune_alpha is not None:
|
| 266 |
+
vit_embeds = self.noised_embed(vit_embeds, self.neftune_alpha)
|
| 267 |
+
|
| 268 |
+
h = w = int(vit_embeds.shape[1] ** 0.5)
|
| 269 |
+
vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
|
| 270 |
+
vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
|
| 271 |
+
vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
|
| 272 |
+
vit_embeds = self.mlp1(vit_embeds) # .to(pixel_values.device)
|
| 273 |
+
return vit_embeds
|
| 274 |
+
|
| 275 |
+
def multi_image_chat(self, tokenizer, pixel_values, image_counts, question, generation_config, history=None, return_history=False, IMG_START_TOKEN="<img>", IMG_END_TOKEN="</img>", IMG_CONTEXT_TOKEN="<IMG_CONTEXT>"):
|
| 276 |
+
img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
| 277 |
+
self.img_context_token_id = img_context_token_id
|
| 278 |
+
if tokenizer.convert_tokens_to_ids("<|im_end|>") != 0:
|
| 279 |
+
eos_token_id = tokenizer.convert_tokens_to_ids("<|im_end|>") # 92542, InternLM2
|
| 280 |
+
else:
|
| 281 |
+
eos_token_id = tokenizer.eos_token_id
|
| 282 |
+
|
| 283 |
+
from internvl.conversation import get_conv_template
|
| 284 |
+
|
| 285 |
+
template = get_conv_template(self.template)
|
| 286 |
+
|
| 287 |
+
if history is None:
|
| 288 |
+
history = []
|
| 289 |
+
image_tokens = ""
|
| 290 |
+
image_bs = pixel_values.shape[0]
|
| 291 |
+
# print(f"dynamic ViT batch size: {image_bs}, image_counts: {image_counts}")
|
| 292 |
+
for idx, image_count in enumerate(image_counts):
|
| 293 |
+
image_tokens += f"<image {idx+1}> (图{idx+1}):" + IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * image_count + IMG_END_TOKEN
|
| 294 |
+
question = image_tokens + "\n" + question
|
| 295 |
+
else:
|
| 296 |
+
for old_question, old_answer in history:
|
| 297 |
+
template.append_message(template.roles[0], old_question)
|
| 298 |
+
template.append_message(template.roles[1], old_answer)
|
| 299 |
+
template.append_message(template.roles[0], question)
|
| 300 |
+
template.append_message(template.roles[1], None)
|
| 301 |
+
query = template.get_prompt()
|
| 302 |
+
model_inputs = tokenizer(query, return_tensors="pt")
|
| 303 |
+
input_ids = model_inputs["input_ids"].cuda()
|
| 304 |
+
attention_mask = model_inputs["attention_mask"].cuda()
|
| 305 |
+
generation_config["eos_token_id"] = eos_token_id
|
| 306 |
+
|
| 307 |
+
generation_output = self.generate(pixel_values=pixel_values, input_ids=input_ids, attention_mask=attention_mask, **generation_config)
|
| 308 |
+
response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
|
| 309 |
+
response = response.split("<|im_end|>")[0].strip() # for InternLM2
|
| 310 |
+
history.append((question, response))
|
| 311 |
+
if return_history:
|
| 312 |
+
return response, history
|
| 313 |
+
else:
|
| 314 |
+
query_to_print = query.replace(image_tokens, "<image>")
|
| 315 |
+
# print(query_to_print, response)
|
| 316 |
+
return response
|
| 317 |
+
return response
|
| 318 |
+
|
| 319 |
+
@property
|
| 320 |
+
def tokenizer(self):
|
| 321 |
+
return self._tokenizer
|
| 322 |
+
|
| 323 |
+
@property
|
| 324 |
+
def model(self):
|
| 325 |
+
# returns the model, unwrapping it if using Accelerate
|
| 326 |
+
if hasattr(self, "accelerator"):
|
| 327 |
+
return self.accelerator.unwrap_model(self._model)
|
| 328 |
+
else:
|
| 329 |
+
return self._model
|
| 330 |
+
|
| 331 |
+
@property
|
| 332 |
+
def batch_size(self):
|
| 333 |
+
return self.batch_size_per_gpu
|
| 334 |
+
|
| 335 |
+
@property
|
| 336 |
+
def device(self):
|
| 337 |
+
return self._device
|
| 338 |
+
|
| 339 |
+
@property
|
| 340 |
+
def rank(self):
|
| 341 |
+
return self._rank
|
| 342 |
+
|
| 343 |
+
@property
|
| 344 |
+
def world_size(self):
|
| 345 |
+
return self._world_size
|
| 346 |
+
|
| 347 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 348 |
+
""" """
|
| 349 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 350 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 351 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 352 |
+
if left_truncate_len:
|
| 353 |
+
encoding = encoding[-left_truncate_len:]
|
| 354 |
+
return encoding
|
| 355 |
+
|
| 356 |
+
def tok_decode(self, tokens):
|
| 357 |
+
try:
|
| 358 |
+
return self.tokenizer.decode(tokens)
|
| 359 |
+
except:
|
| 360 |
+
return self.tokenizer.decode([tokens])
|
| 361 |
+
|
| 362 |
+
def post_processing(self, response):
|
| 363 |
+
response = response.replace("\n", "").replace("不是", "No").replace("是", "Yes").replace("否", "No")
|
| 364 |
+
response = response.lower().replace("true", "yes").replace("false", "no")
|
| 365 |
+
pattern = re.compile(r"[\u4e00-\u9fa5]")
|
| 366 |
+
response = re.sub(pattern, "", response)
|
| 367 |
+
return response
|
| 368 |
+
|
| 369 |
+
@torch.no_grad()
|
| 370 |
+
def generate(
|
| 371 |
+
self,
|
| 372 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 373 |
+
input_ids: Optional[torch.FloatTensor] = None,
|
| 374 |
+
attention_mask: Optional[torch.LongTensor] = None,
|
| 375 |
+
visual_features: Optional[torch.FloatTensor] = None,
|
| 376 |
+
generation_config: Optional[GenerationConfig] = None,
|
| 377 |
+
output_hidden_states: Optional[bool] = None,
|
| 378 |
+
return_dict: Optional[bool] = None,
|
| 379 |
+
**generate_kwargs,
|
| 380 |
+
) -> torch.LongTensor:
|
| 381 |
+
assert self.img_context_token_id is not None
|
| 382 |
+
if pixel_values is not None:
|
| 383 |
+
if visual_features is not None:
|
| 384 |
+
vit_embeds = visual_features
|
| 385 |
+
else:
|
| 386 |
+
vit_embeds = self.extract_feature(pixel_values)
|
| 387 |
+
|
| 388 |
+
input_embeds = self.language_model.get_input_embeddings()(input_ids)
|
| 389 |
+
B, N, C = input_embeds.shape
|
| 390 |
+
input_embeds = input_embeds.reshape(B * N, C)
|
| 391 |
+
|
| 392 |
+
input_ids = input_ids.reshape(B * N)
|
| 393 |
+
selected = input_ids == self.img_context_token_id
|
| 394 |
+
assert selected.sum() != 0
|
| 395 |
+
input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
|
| 396 |
+
|
| 397 |
+
input_embeds = input_embeds.reshape(B, N, C)
|
| 398 |
+
else:
|
| 399 |
+
input_embeds = self.language_model.get_input_embeddings()(input_ids)
|
| 400 |
+
|
| 401 |
+
outputs = self.language_model.generate(
|
| 402 |
+
inputs_embeds=input_embeds,
|
| 403 |
+
attention_mask=attention_mask,
|
| 404 |
+
generation_config=generation_config,
|
| 405 |
+
output_hidden_states=output_hidden_states,
|
| 406 |
+
return_dict=return_dict,
|
| 407 |
+
use_cache=True,
|
| 408 |
+
**generate_kwargs,
|
| 409 |
+
)
|
| 410 |
+
|
| 411 |
+
return outputs
|
| 412 |
+
|
| 413 |
+
def flatten(self, input):
|
| 414 |
+
new_list = []
|
| 415 |
+
for i in input:
|
| 416 |
+
for j in i:
|
| 417 |
+
new_list.append(j)
|
| 418 |
+
return new_list
|
| 419 |
+
|
| 420 |
+
def load_image(self, flattened_visuals, input_size=224):
|
| 421 |
+
assert flattened_visuals[0].mode == "RGB"
|
| 422 |
+
image = flattened_visuals[0].convert("RGB")
|
| 423 |
+
transform = build_transform(is_train=False, input_size=input_size)
|
| 424 |
+
if self.dynamic:
|
| 425 |
+
images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=self.use_thumbnail, max_num=self.max_num)
|
| 426 |
+
else:
|
| 427 |
+
images = [image]
|
| 428 |
+
pixel_values = [transform(image) for image in images]
|
| 429 |
+
pixel_values = torch.stack(pixel_values)
|
| 430 |
+
return pixel_values
|
| 431 |
+
|
| 432 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 433 |
+
res = []
|
| 434 |
+
|
| 435 |
+
def _collate(x):
|
| 436 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 437 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 438 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 439 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 440 |
+
# automatic adaptive batches much much easier to implement
|
| 441 |
+
# - any OOMs will happen right away rather than near the end
|
| 442 |
+
toks = self.tok_encode(x[0])
|
| 443 |
+
return -len(toks), x[0]
|
| 444 |
+
|
| 445 |
+
# we group requests by their generation_kwargs,
|
| 446 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 447 |
+
# in the same batch.
|
| 448 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 449 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 450 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 451 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 452 |
+
for chunk in chunks:
|
| 453 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 454 |
+
task = task[0]
|
| 455 |
+
split = split[0]
|
| 456 |
+
batched_visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # [B, N]
|
| 457 |
+
flattened_visuals = self.flatten(batched_visuals)
|
| 458 |
+
try:
|
| 459 |
+
pixel_values = self.load_image(flattened_visuals, self.image_size).cuda().to(torch.bfloat16)
|
| 460 |
+
except IndexError:
|
| 461 |
+
pixel_values = None
|
| 462 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 463 |
+
|
| 464 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 465 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 466 |
+
if "temperature" not in gen_kwargs:
|
| 467 |
+
gen_kwargs["temperature"] = 0
|
| 468 |
+
if "top_p" not in gen_kwargs:
|
| 469 |
+
gen_kwargs["top_p"] = None
|
| 470 |
+
if "num_beams" not in gen_kwargs:
|
| 471 |
+
gen_kwargs["num_beams"] = 1
|
| 472 |
+
|
| 473 |
+
generation_config = dict(
|
| 474 |
+
do_sample=False,
|
| 475 |
+
top_k=50,
|
| 476 |
+
top_p=gen_kwargs["top_p"],
|
| 477 |
+
num_beams=gen_kwargs["num_beams"],
|
| 478 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 479 |
+
eos_token_id=self.tokenizer.eos_token_id,
|
| 480 |
+
)
|
| 481 |
+
question = contexts[0]
|
| 482 |
+
response = self.model.chat(tokenizer=self.tokenizer, pixel_values=pixel_values, question=question, generation_config=generation_config)
|
| 483 |
+
# TODO(choiszt) try batch_chat for multiple inputs
|
| 484 |
+
response = self.post_processing(response)
|
| 485 |
+
res.append(response)
|
| 486 |
+
self.cache_hook.add_partial("generate_until", (question, gen_kwargs), response)
|
| 487 |
+
pbar.update(1)
|
| 488 |
+
res = re_ords.get_original(res)
|
| 489 |
+
return res
|
| 490 |
+
# print(chunk)
|
| 491 |
+
|
| 492 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 493 |
+
pass
|
| 494 |
+
|
| 495 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 496 |
+
raise NotImplementedError("TODO: Implement multi-round generation for InternVL")
|
LaViDa/eval/lmms_eval/models/internvl2.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
from typing import List, Tuple
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import torch
|
| 6 |
+
import torchvision.transforms as T
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from decord import VideoReader, cpu
|
| 9 |
+
from PIL import Image
|
| 10 |
+
from torchvision.transforms.functional import InterpolationMode
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from transformers import AutoModel, AutoTokenizer
|
| 13 |
+
|
| 14 |
+
from lmms_eval.api.instance import Instance
|
| 15 |
+
from lmms_eval.api.model import lmms
|
| 16 |
+
from lmms_eval.api.registry import register_model
|
| 17 |
+
|
| 18 |
+
eval_logger = logging.getLogger("eval_logger")
|
| 19 |
+
|
| 20 |
+
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
| 21 |
+
IMAGENET_STD = (0.229, 0.224, 0.225)
|
| 22 |
+
|
| 23 |
+
DEFAULT_GEN_KWARGS = dict(
|
| 24 |
+
num_beams=1,
|
| 25 |
+
max_new_tokens=1024,
|
| 26 |
+
do_sample=False,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def build_transform(input_size):
|
| 31 |
+
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
|
| 32 |
+
transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
|
| 33 |
+
return transform
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
| 37 |
+
best_ratio_diff = float("inf")
|
| 38 |
+
best_ratio = (1, 1)
|
| 39 |
+
area = width * height
|
| 40 |
+
for ratio in target_ratios:
|
| 41 |
+
target_aspect_ratio = ratio[0] / ratio[1]
|
| 42 |
+
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
| 43 |
+
if ratio_diff < best_ratio_diff:
|
| 44 |
+
best_ratio_diff = ratio_diff
|
| 45 |
+
best_ratio = ratio
|
| 46 |
+
elif ratio_diff == best_ratio_diff:
|
| 47 |
+
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
| 48 |
+
best_ratio = ratio
|
| 49 |
+
return best_ratio
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
|
| 53 |
+
orig_width, orig_height = image.size
|
| 54 |
+
aspect_ratio = orig_width / orig_height
|
| 55 |
+
|
| 56 |
+
# calculate the existing image aspect ratio
|
| 57 |
+
target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
|
| 58 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 59 |
+
|
| 60 |
+
# find the closest aspect ratio to the target
|
| 61 |
+
target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
| 62 |
+
|
| 63 |
+
# calculate the target width and height
|
| 64 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 65 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 66 |
+
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
| 67 |
+
|
| 68 |
+
# resize the image
|
| 69 |
+
resized_img = image.resize((target_width, target_height))
|
| 70 |
+
processed_images = []
|
| 71 |
+
for i in range(blocks):
|
| 72 |
+
box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
|
| 73 |
+
# split the image
|
| 74 |
+
split_img = resized_img.crop(box)
|
| 75 |
+
processed_images.append(split_img)
|
| 76 |
+
assert len(processed_images) == blocks
|
| 77 |
+
if use_thumbnail and len(processed_images) != 1:
|
| 78 |
+
thumbnail_img = image.resize((image_size, image_size))
|
| 79 |
+
processed_images.append(thumbnail_img)
|
| 80 |
+
return processed_images
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def load_image(image, input_size=448, max_num=6):
|
| 84 |
+
transform = build_transform(input_size=input_size)
|
| 85 |
+
images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
|
| 86 |
+
pixel_values = [transform(image) for image in images]
|
| 87 |
+
pixel_values = torch.stack(pixel_values)
|
| 88 |
+
return pixel_values
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
|
| 92 |
+
if bound:
|
| 93 |
+
start, end = bound[0], bound[1]
|
| 94 |
+
else:
|
| 95 |
+
start, end = -100000, 100000
|
| 96 |
+
start_idx = max(first_idx, round(start * fps))
|
| 97 |
+
end_idx = min(round(end * fps), max_frame)
|
| 98 |
+
seg_size = float(end_idx - start_idx) / num_segments
|
| 99 |
+
frame_indices = np.array([int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)])
|
| 100 |
+
return frame_indices
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32):
|
| 104 |
+
vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
|
| 105 |
+
max_frame = len(vr) - 1
|
| 106 |
+
fps = float(vr.get_avg_fps())
|
| 107 |
+
|
| 108 |
+
pixel_values_list, num_patches_list = [], []
|
| 109 |
+
transform = build_transform(input_size=input_size)
|
| 110 |
+
frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
|
| 111 |
+
for frame_index in frame_indices:
|
| 112 |
+
img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
|
| 113 |
+
img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=True, max_num=max_num)
|
| 114 |
+
pixel_values = [transform(tile) for tile in img]
|
| 115 |
+
pixel_values = torch.stack(pixel_values)
|
| 116 |
+
num_patches_list.append(pixel_values.shape[0])
|
| 117 |
+
pixel_values_list.append(pixel_values)
|
| 118 |
+
pixel_values = torch.cat(pixel_values_list)
|
| 119 |
+
return pixel_values, num_patches_list
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
import math
|
| 123 |
+
from datetime import timedelta
|
| 124 |
+
|
| 125 |
+
from accelerate.state import AcceleratorState
|
| 126 |
+
from accelerate.utils import InitProcessGroupKwargs
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# The reason for writing the code this way is to avoid errors that occur during multi-GPU inference due to tensors not being on the same device. By ensuring that the first and last layers of the large language model (LLM) are on the same device, we prevent such errors.
|
| 130 |
+
def split_model(model_name, num_layers=None):
|
| 131 |
+
device_map = {}
|
| 132 |
+
world_size = torch.cuda.device_count()
|
| 133 |
+
if num_layers is None:
|
| 134 |
+
num_layers = {
|
| 135 |
+
"InternVL2_5-1B": 24,
|
| 136 |
+
"InternVL2_5-2B": 24,
|
| 137 |
+
"InternVL2_5-4B": 36,
|
| 138 |
+
"InternVL2_5-8B": 32,
|
| 139 |
+
"InternVL2_5-26B": 48,
|
| 140 |
+
"InternVL2_5-38B": 64,
|
| 141 |
+
"InternVL2_5-78B": 80,
|
| 142 |
+
"InternVL2-1B": 24,
|
| 143 |
+
"InternVL2-2B": 24,
|
| 144 |
+
"InternVL2-4B": 32,
|
| 145 |
+
"InternVL2-8B": 32,
|
| 146 |
+
"InternVL2-26B": 48,
|
| 147 |
+
"InternVL2-40B": 60,
|
| 148 |
+
"InternVL2-Llama3-76B": 80,
|
| 149 |
+
}[model_name]
|
| 150 |
+
# Since the first GPU will be used for ViT, treat it as half a GPU.
|
| 151 |
+
num_layers_per_gpu = math.ceil(num_layers / (world_size - 0.5))
|
| 152 |
+
num_layers_per_gpu = [num_layers_per_gpu] * world_size
|
| 153 |
+
num_layers_per_gpu[0] = math.ceil(num_layers_per_gpu[0] * 0.5)
|
| 154 |
+
layer_cnt = 0
|
| 155 |
+
for i, num_layer in enumerate(num_layers_per_gpu):
|
| 156 |
+
for j in range(num_layer):
|
| 157 |
+
device_map[f"language_model.model.layers.{layer_cnt}"] = i
|
| 158 |
+
layer_cnt += 1
|
| 159 |
+
device_map["vision_model"] = 0
|
| 160 |
+
device_map["mlp1"] = 0
|
| 161 |
+
device_map["language_model.model.tok_embeddings"] = 0
|
| 162 |
+
device_map["language_model.model.embed_tokens"] = 0
|
| 163 |
+
device_map["language_model.output"] = 0
|
| 164 |
+
device_map["language_model.model.norm"] = 0
|
| 165 |
+
device_map["language_model.lm_head"] = 0
|
| 166 |
+
device_map[f"language_model.model.layers.{num_layers - 1}"] = 0
|
| 167 |
+
|
| 168 |
+
return device_map
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
@register_model("internvl2")
|
| 172 |
+
class InternVL2(lmms):
|
| 173 |
+
def __init__(
|
| 174 |
+
self,
|
| 175 |
+
pretrained: str = "OpenGVLab/InternVL2-2B",
|
| 176 |
+
modality: str = "image",
|
| 177 |
+
device: str = "cuda:0",
|
| 178 |
+
device_map: str = "cuda:0",
|
| 179 |
+
batch_size: str = "1",
|
| 180 |
+
num_frame: int = 32,
|
| 181 |
+
num_layers=None,
|
| 182 |
+
**kwargs,
|
| 183 |
+
):
|
| 184 |
+
super().__init__()
|
| 185 |
+
|
| 186 |
+
self.path = pretrained
|
| 187 |
+
self.num_frame = num_frame
|
| 188 |
+
|
| 189 |
+
batch_size = int(batch_size)
|
| 190 |
+
assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}."
|
| 191 |
+
self.batch_size_per_gpu = batch_size
|
| 192 |
+
|
| 193 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 194 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 195 |
+
self.accelerator = accelerator
|
| 196 |
+
if accelerator.num_processes > 1:
|
| 197 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 198 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 199 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 200 |
+
self._device = torch.device(device)
|
| 201 |
+
device_map = split_model(pretrained.split("/")[-1], num_layers=num_layers)
|
| 202 |
+
self.device_map = device_map
|
| 203 |
+
else:
|
| 204 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 205 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 206 |
+
|
| 207 |
+
self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True, device_map=self.device_map).eval()
|
| 208 |
+
self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True, device_map=self.device_map)
|
| 209 |
+
|
| 210 |
+
if accelerator.num_processes > 1:
|
| 211 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 212 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 213 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 214 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 215 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 216 |
+
kwargs = {
|
| 217 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 218 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 219 |
+
}
|
| 220 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 221 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 222 |
+
|
| 223 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 224 |
+
self._model = accelerator.prepare(self.model)
|
| 225 |
+
else:
|
| 226 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 227 |
+
self.accelerator = accelerator
|
| 228 |
+
if self.accelerator.is_local_main_process:
|
| 229 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 230 |
+
self._rank = self.accelerator.local_process_index
|
| 231 |
+
self._world_size = self.accelerator.num_processes
|
| 232 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 233 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 234 |
+
self._rank = 0
|
| 235 |
+
self._world_size = 1
|
| 236 |
+
else:
|
| 237 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 238 |
+
self.model.to(self._device)
|
| 239 |
+
self._rank = 0
|
| 240 |
+
self._world_size = 1
|
| 241 |
+
|
| 242 |
+
self.modality = modality
|
| 243 |
+
|
| 244 |
+
@property
|
| 245 |
+
def config(self):
|
| 246 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 247 |
+
return self._config
|
| 248 |
+
|
| 249 |
+
@property
|
| 250 |
+
def tokenizer(self):
|
| 251 |
+
return self._tokenizer
|
| 252 |
+
|
| 253 |
+
@property
|
| 254 |
+
def model(self):
|
| 255 |
+
# returns the model, unwrapping it if using Accelerate
|
| 256 |
+
if hasattr(self, "accelerator"):
|
| 257 |
+
return self.accelerator.unwrap_model(self._model)
|
| 258 |
+
else:
|
| 259 |
+
return self._model
|
| 260 |
+
|
| 261 |
+
@property
|
| 262 |
+
def batch_size(self):
|
| 263 |
+
return self.batch_size_per_gpu
|
| 264 |
+
|
| 265 |
+
@property
|
| 266 |
+
def device(self):
|
| 267 |
+
return self._device
|
| 268 |
+
|
| 269 |
+
@property
|
| 270 |
+
def rank(self):
|
| 271 |
+
return self._rank
|
| 272 |
+
|
| 273 |
+
@property
|
| 274 |
+
def world_size(self):
|
| 275 |
+
return self._world_size
|
| 276 |
+
|
| 277 |
+
def flatten(self, input):
|
| 278 |
+
new_list = []
|
| 279 |
+
for i in input:
|
| 280 |
+
for j in i:
|
| 281 |
+
new_list.append(j)
|
| 282 |
+
return new_list
|
| 283 |
+
|
| 284 |
+
def generate_until(self, requests) -> List[str]:
|
| 285 |
+
res = []
|
| 286 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 287 |
+
|
| 288 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 289 |
+
if "until" in gen_kwargs:
|
| 290 |
+
gen_kwargs.pop("until")
|
| 291 |
+
for k, v in DEFAULT_GEN_KWARGS.items():
|
| 292 |
+
if k not in gen_kwargs:
|
| 293 |
+
gen_kwargs[k] = v
|
| 294 |
+
|
| 295 |
+
pop_keys = []
|
| 296 |
+
for k, v in gen_kwargs.items():
|
| 297 |
+
if k not in DEFAULT_GEN_KWARGS:
|
| 298 |
+
pop_keys.append(k)
|
| 299 |
+
|
| 300 |
+
for k in pop_keys:
|
| 301 |
+
gen_kwargs.pop(k)
|
| 302 |
+
|
| 303 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 304 |
+
visuals = self.flatten(visuals)
|
| 305 |
+
if self.modality == "image":
|
| 306 |
+
if visuals:
|
| 307 |
+
visuals = [load_image(visual).to(torch.bfloat16).cuda() for visual in visuals]
|
| 308 |
+
pixel_values = torch.cat(visuals, dim=0)
|
| 309 |
+
num_patches_list = [visual.size(0) for visual in visuals]
|
| 310 |
+
image_tokens = ["<image>"] * len(visuals)
|
| 311 |
+
image_tokens = " ".join(image_tokens)
|
| 312 |
+
contexts = image_tokens + "\n" + contexts
|
| 313 |
+
else:
|
| 314 |
+
pixel_values = None
|
| 315 |
+
num_patches_list = None
|
| 316 |
+
response, history = self.model.chat(self.tokenizer, pixel_values, contexts, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
|
| 317 |
+
elif self.modality == "video":
|
| 318 |
+
assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos."
|
| 319 |
+
video_path = visuals[0]
|
| 320 |
+
pixel_values, num_patches_list = load_video(video_path, num_segments=self.num_frame)
|
| 321 |
+
pixel_values = pixel_values.to(torch.bfloat16).cuda()
|
| 322 |
+
video_prefix = "".join([f"Frame{i+1}: <image>\n" for i in range(len(num_patches_list))])
|
| 323 |
+
question = video_prefix + contexts
|
| 324 |
+
response, history = self.model.chat(self.tokenizer, pixel_values, question, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
|
| 325 |
+
res.append(response)
|
| 326 |
+
pbar.update(1)
|
| 327 |
+
pbar.close()
|
| 328 |
+
return res
|
| 329 |
+
|
| 330 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 331 |
+
assert False, "Not implemented yet."
|
| 332 |
+
|
| 333 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 334 |
+
raise NotImplementedError("TODO: Implement multi-round generation for InternVL2")
|
LaViDa/eval/lmms_eval/models/llama_vid.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import os
|
| 3 |
+
import subprocess
|
| 4 |
+
from datetime import timedelta
|
| 5 |
+
from typing import List, Optional, Tuple, Union
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import requests
|
| 9 |
+
import torch
|
| 10 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 11 |
+
from accelerate.state import AcceleratorState
|
| 12 |
+
from decord import VideoReader, cpu
|
| 13 |
+
from huggingface_hub import snapshot_download
|
| 14 |
+
from loguru import logger as eval_logger
|
| 15 |
+
from tqdm import tqdm
|
| 16 |
+
from transformers import AutoConfig
|
| 17 |
+
|
| 18 |
+
from lmms_eval import utils
|
| 19 |
+
from lmms_eval.api.instance import Instance
|
| 20 |
+
from lmms_eval.api.model import lmms
|
| 21 |
+
from lmms_eval.api.registry import register_model
|
| 22 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 23 |
+
from lmms_eval.utils import stop_sequences_criteria
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
from llamavid.constants import (
|
| 27 |
+
DEFAULT_IM_END_TOKEN,
|
| 28 |
+
DEFAULT_IM_START_TOKEN,
|
| 29 |
+
DEFAULT_IMAGE_TOKEN,
|
| 30 |
+
IMAGE_TOKEN_INDEX,
|
| 31 |
+
)
|
| 32 |
+
from llamavid.conversation import SeparatorStyle, conv_templates
|
| 33 |
+
from llamavid.model.builder import load_pretrained_model
|
| 34 |
+
from llava.mm_utils import (
|
| 35 |
+
KeywordsStoppingCriteria,
|
| 36 |
+
get_model_name_from_path,
|
| 37 |
+
tokenizer_image_token,
|
| 38 |
+
)
|
| 39 |
+
except ImportError:
|
| 40 |
+
eval_logger.debug("LLaMA-Video is not installed. Please install LLaMA-Video to use this model.")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@register_model("llama_vid")
|
| 44 |
+
class LLaMAVid(lmms):
|
| 45 |
+
def __init__(
|
| 46 |
+
self,
|
| 47 |
+
pretrained: str = "YanweiLi/llama-vid-7b-full-224-video-fps-1",
|
| 48 |
+
truncation: Optional[bool] = True,
|
| 49 |
+
device: Optional[str] = "cuda:0",
|
| 50 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 51 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 52 |
+
trust_remote_code: Optional[bool] = False,
|
| 53 |
+
revision=None,
|
| 54 |
+
attn_implementation=(
|
| 55 |
+
"sdpa" if torch.__version__ > "2.1.2" else "eager"
|
| 56 |
+
), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 57 |
+
device_map="cuda:0",
|
| 58 |
+
conv_template="vicuna_v1",
|
| 59 |
+
use_cache=True,
|
| 60 |
+
truncate_context=False,
|
| 61 |
+
num_frames: int = 100,
|
| 62 |
+
**kwargs,
|
| 63 |
+
) -> None:
|
| 64 |
+
super().__init__()
|
| 65 |
+
|
| 66 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 67 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 68 |
+
if accelerator.num_processes > 1:
|
| 69 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 70 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 71 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 72 |
+
self._device = torch.device(device)
|
| 73 |
+
self.device_map = device_map
|
| 74 |
+
else:
|
| 75 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 76 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 77 |
+
|
| 78 |
+
self.pretrained = pretrained
|
| 79 |
+
self.model_path = snapshot_download(self.pretrained)
|
| 80 |
+
self.model_name = get_model_name_from_path(pretrained)
|
| 81 |
+
self.num_frames = num_frames
|
| 82 |
+
if not os.path.exists("./model_zoo/LAVIS/eva_vit_g.pth") and accelerator.is_main_process:
|
| 83 |
+
eval_logger.info("\n\n Eva Encoder is not found for LLaMA-VID. Download automatically to the folder ./model_zoo/LAVIS")
|
| 84 |
+
cache_path = "model_zoo/LAVIS"
|
| 85 |
+
os.makedirs(cache_path, exist_ok=True)
|
| 86 |
+
subprocess.run(["wget https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/BLIP2/eva_vit_g.pth -O ./model_zoo/LAVIS/eva_vit_g.pth"], shell=True)
|
| 87 |
+
|
| 88 |
+
accelerator.wait_for_everyone()
|
| 89 |
+
self._tokenizer, self._model, self.image_processor, self._max_length = load_pretrained_model(
|
| 90 |
+
self.model_path,
|
| 91 |
+
None,
|
| 92 |
+
self.model_name,
|
| 93 |
+
device_map=self.device_map,
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
self._config = self._model.config
|
| 97 |
+
self.model.eval()
|
| 98 |
+
self.model.tie_weights()
|
| 99 |
+
self.truncation = truncation
|
| 100 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 101 |
+
self.conv_template = conv_template
|
| 102 |
+
self.use_cache = use_cache
|
| 103 |
+
self.truncate_context = truncate_context
|
| 104 |
+
# assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 105 |
+
if accelerator.num_processes > 1:
|
| 106 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 107 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 108 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 109 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 110 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 111 |
+
kwargs = {
|
| 112 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 113 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 114 |
+
}
|
| 115 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 116 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 117 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 118 |
+
self._model = accelerator.prepare(self.model)
|
| 119 |
+
else:
|
| 120 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 121 |
+
self.accelerator = accelerator
|
| 122 |
+
if self.accelerator.is_local_main_process:
|
| 123 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 124 |
+
self._rank = self.accelerator.local_process_index
|
| 125 |
+
self._world_size = self.accelerator.num_processes
|
| 126 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 127 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 128 |
+
self._rank = 0
|
| 129 |
+
self._world_size = 1
|
| 130 |
+
else:
|
| 131 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 132 |
+
self.model.to(self._device)
|
| 133 |
+
self._rank = 0
|
| 134 |
+
self._world_size = 1
|
| 135 |
+
|
| 136 |
+
def download_file(self, url, folder_path):
|
| 137 |
+
# Create the folder if it doesn't exist
|
| 138 |
+
if not os.path.exists(folder_path):
|
| 139 |
+
os.makedirs(folder_path)
|
| 140 |
+
|
| 141 |
+
# Extract filename from URL
|
| 142 |
+
filename = url.split("/")[-1]
|
| 143 |
+
|
| 144 |
+
# Define path to save the file
|
| 145 |
+
file_path = os.path.join(folder_path, filename)
|
| 146 |
+
|
| 147 |
+
# Send a GET request to the URL
|
| 148 |
+
response = requests.get(url)
|
| 149 |
+
|
| 150 |
+
# Check if request was successful (status code 200)
|
| 151 |
+
if response.status_code == 200:
|
| 152 |
+
# Save the file to the specified folder
|
| 153 |
+
with open(file_path, "wb") as f:
|
| 154 |
+
f.write(response.content)
|
| 155 |
+
print(f"File downloaded successfully to {file_path}")
|
| 156 |
+
else:
|
| 157 |
+
print(f"Failed to download file. Status code: {response.status_code}")
|
| 158 |
+
|
| 159 |
+
@property
|
| 160 |
+
def config(self):
|
| 161 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 162 |
+
return self._config
|
| 163 |
+
|
| 164 |
+
@property
|
| 165 |
+
def tokenizer(self):
|
| 166 |
+
return self._tokenizer
|
| 167 |
+
|
| 168 |
+
@property
|
| 169 |
+
def model(self):
|
| 170 |
+
# returns the model, unwrapping it if using Accelerate
|
| 171 |
+
if hasattr(self, "accelerator"):
|
| 172 |
+
return self.accelerator.unwrap_model(self._model)
|
| 173 |
+
else:
|
| 174 |
+
return self._model
|
| 175 |
+
|
| 176 |
+
@property
|
| 177 |
+
def eot_token_id(self):
|
| 178 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 179 |
+
return self.tokenizer.eos_token_id
|
| 180 |
+
|
| 181 |
+
@property
|
| 182 |
+
def max_length(self):
|
| 183 |
+
return self._max_length
|
| 184 |
+
|
| 185 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 186 |
+
""" """
|
| 187 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 188 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 189 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 190 |
+
if left_truncate_len:
|
| 191 |
+
encoding = encoding[-left_truncate_len:]
|
| 192 |
+
return encoding
|
| 193 |
+
|
| 194 |
+
def tok_decode(self, tokens):
|
| 195 |
+
return self.tokenizer.decode(tokens)
|
| 196 |
+
|
| 197 |
+
def load_video(self, video_path):
|
| 198 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 199 |
+
total_frame_num = len(vr)
|
| 200 |
+
fps = round(vr.get_avg_fps())
|
| 201 |
+
frame_idx = [i for i in range(0, len(vr), fps)]
|
| 202 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 203 |
+
return spare_frames
|
| 204 |
+
|
| 205 |
+
def flatten(self, input):
|
| 206 |
+
new_list = []
|
| 207 |
+
for i in input:
|
| 208 |
+
for j in i:
|
| 209 |
+
new_list.append(j)
|
| 210 |
+
return new_list
|
| 211 |
+
|
| 212 |
+
def generate_until(self, requests) -> List[str]:
|
| 213 |
+
res = []
|
| 214 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 215 |
+
|
| 216 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 217 |
+
# encode, pad, and truncate contexts for this batch
|
| 218 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 219 |
+
visuals = self.flatten(visuals)
|
| 220 |
+
videos = []
|
| 221 |
+
for visual in visuals:
|
| 222 |
+
video = read_video_pyav(visual, num_frm=self.num_frames)
|
| 223 |
+
video = self.image_processor.preprocess(video, return_tensors="pt")["pixel_values"].half().cuda()
|
| 224 |
+
video = [video]
|
| 225 |
+
videos += video
|
| 226 |
+
qs = contexts
|
| 227 |
+
if self.model.config.mm_use_im_start_end:
|
| 228 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 229 |
+
else:
|
| 230 |
+
qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 231 |
+
|
| 232 |
+
conv = conv_templates[self.conv_template].copy()
|
| 233 |
+
conv.append_message(conv.roles[0], qs)
|
| 234 |
+
conv.append_message(conv.roles[1], None)
|
| 235 |
+
prompt = conv.get_prompt()
|
| 236 |
+
|
| 237 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
|
| 238 |
+
|
| 239 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 240 |
+
keywords = [stop_str]
|
| 241 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 242 |
+
|
| 243 |
+
cur_prompt = contexts
|
| 244 |
+
with torch.inference_mode():
|
| 245 |
+
self.model.update_prompt([[cur_prompt]])
|
| 246 |
+
output_ids = self.model.generate(input_ids, images=video, do_sample=True, temperature=0.2, max_new_tokens=1024, use_cache=True, stopping_criteria=[stopping_criteria])
|
| 247 |
+
|
| 248 |
+
input_token_len = input_ids.shape[1]
|
| 249 |
+
n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
|
| 250 |
+
if n_diff_input_output > 0:
|
| 251 |
+
print(f"[Warning] {n_diff_input_output} output_ids are not the same as the input_ids")
|
| 252 |
+
outputs = self.tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
|
| 253 |
+
outputs = outputs.strip()
|
| 254 |
+
if outputs.endswith(stop_str):
|
| 255 |
+
outputs = outputs[: -len(stop_str)]
|
| 256 |
+
outputs = outputs.strip()
|
| 257 |
+
pbar.update(1)
|
| 258 |
+
res.append(outputs)
|
| 259 |
+
|
| 260 |
+
return res
|
| 261 |
+
|
| 262 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 263 |
+
return super().loglikelihood(requests)
|
| 264 |
+
|
| 265 |
+
@property
|
| 266 |
+
def batch_size(self):
|
| 267 |
+
return self.batch_size_per_gpu
|
| 268 |
+
|
| 269 |
+
@property
|
| 270 |
+
def device(self):
|
| 271 |
+
return self._device
|
| 272 |
+
|
| 273 |
+
@property
|
| 274 |
+
def rank(self):
|
| 275 |
+
return self._rank
|
| 276 |
+
|
| 277 |
+
@property
|
| 278 |
+
def world_size(self):
|
| 279 |
+
return self._world_size
|
| 280 |
+
|
| 281 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 282 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaMAVid")
|
LaViDa/eval/lmms_eval/models/llama_vision.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import PIL
|
| 6 |
+
import torch
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from accelerate.state import AcceleratorState
|
| 9 |
+
from decord import VideoReader, cpu
|
| 10 |
+
from torchvision.transforms.functional import to_pil_image
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from transformers import AutoConfig, AutoProcessor, MllamaForConditionalGeneration
|
| 13 |
+
|
| 14 |
+
from lmms_eval import utils
|
| 15 |
+
from lmms_eval.api.instance import Instance
|
| 16 |
+
from lmms_eval.api.model import lmms
|
| 17 |
+
from lmms_eval.api.registry import register_model
|
| 18 |
+
|
| 19 |
+
warnings.filterwarnings("ignore")
|
| 20 |
+
|
| 21 |
+
from loguru import logger as eval_logger
|
| 22 |
+
|
| 23 |
+
DEFAULT_IMAGE_TOKEN = "<|image|>"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@register_model("llama_vision")
|
| 27 |
+
class LlamaVision(lmms):
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
pretrained: str = "meta-llama/Llama-3.2-11B-Vision",
|
| 31 |
+
revision: str = "main",
|
| 32 |
+
device: str = "cuda",
|
| 33 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 34 |
+
batch_size: int = 1,
|
| 35 |
+
trust_remote_code: Optional[bool] = False,
|
| 36 |
+
attn_implementation: Optional[str] = None,
|
| 37 |
+
device_map: str = "",
|
| 38 |
+
max_frames_num: Optional[int] = 32,
|
| 39 |
+
**kwargs,
|
| 40 |
+
) -> None:
|
| 41 |
+
super().__init__()
|
| 42 |
+
# Do not use kwargs for now
|
| 43 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 44 |
+
|
| 45 |
+
accelerator = Accelerator()
|
| 46 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 47 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 48 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 49 |
+
else:
|
| 50 |
+
self._device = torch.device(device)
|
| 51 |
+
self.device_map = device_map
|
| 52 |
+
if isinstance(dtype, str) and dtype != "auto":
|
| 53 |
+
dtype = getattr(torch, dtype)
|
| 54 |
+
|
| 55 |
+
self.max_frames_num = max_frames_num
|
| 56 |
+
self._model = MllamaForConditionalGeneration.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
|
| 57 |
+
self.model.eval()
|
| 58 |
+
self.processor = AutoProcessor.from_pretrained(pretrained)
|
| 59 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 60 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 61 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 62 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 63 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 64 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 65 |
+
kwargs = {
|
| 66 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 67 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 68 |
+
}
|
| 69 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 70 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 71 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 72 |
+
self._model = accelerator.prepare(self.model)
|
| 73 |
+
else:
|
| 74 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 75 |
+
self.accelerator = accelerator
|
| 76 |
+
if self.accelerator.is_local_main_process:
|
| 77 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 78 |
+
self._rank = self.accelerator.local_process_index
|
| 79 |
+
self._world_size = self.accelerator.num_processes
|
| 80 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 81 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
|
| 82 |
+
self._rank = 0
|
| 83 |
+
self._world_size = 1
|
| 84 |
+
else:
|
| 85 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 86 |
+
self.model.to(self._device)
|
| 87 |
+
self._rank = 0
|
| 88 |
+
self._world_size = 1
|
| 89 |
+
self.accelerator = accelerator
|
| 90 |
+
|
| 91 |
+
@property
|
| 92 |
+
def config(self):
|
| 93 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 94 |
+
return self._config
|
| 95 |
+
|
| 96 |
+
@property
|
| 97 |
+
def tokenizer(self):
|
| 98 |
+
return self._tokenizer
|
| 99 |
+
|
| 100 |
+
@property
|
| 101 |
+
def model(self):
|
| 102 |
+
# returns the model, unwrapping it if using Accelerate
|
| 103 |
+
if hasattr(self, "accelerator"):
|
| 104 |
+
return self.accelerator.unwrap_model(self._model)
|
| 105 |
+
else:
|
| 106 |
+
return self._model
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def eot_token_id(self):
|
| 110 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 111 |
+
return self.tokenizer.eos_token_id
|
| 112 |
+
|
| 113 |
+
@property
|
| 114 |
+
def max_length(self):
|
| 115 |
+
return self._max_length
|
| 116 |
+
|
| 117 |
+
@property
|
| 118 |
+
def batch_size(self):
|
| 119 |
+
return self.batch_size_per_gpu
|
| 120 |
+
|
| 121 |
+
@property
|
| 122 |
+
def device(self):
|
| 123 |
+
return self._device
|
| 124 |
+
|
| 125 |
+
@property
|
| 126 |
+
def rank(self):
|
| 127 |
+
return self._rank
|
| 128 |
+
|
| 129 |
+
@property
|
| 130 |
+
def world_size(self):
|
| 131 |
+
return self._world_size
|
| 132 |
+
|
| 133 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 134 |
+
""" """
|
| 135 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 136 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 137 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 138 |
+
if left_truncate_len:
|
| 139 |
+
encoding = encoding[-left_truncate_len:]
|
| 140 |
+
return encoding
|
| 141 |
+
|
| 142 |
+
def tok_decode(self, tokens):
|
| 143 |
+
return self.tokenizer.decode(tokens)
|
| 144 |
+
|
| 145 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 146 |
+
assert False, "Not implemented"
|
| 147 |
+
|
| 148 |
+
def flatten(self, input):
|
| 149 |
+
new_list = []
|
| 150 |
+
for i in input:
|
| 151 |
+
for j in i:
|
| 152 |
+
new_list.append(j)
|
| 153 |
+
return new_list
|
| 154 |
+
|
| 155 |
+
def load_video(self, video_path, max_frames_num):
|
| 156 |
+
if type(video_path) == str:
|
| 157 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 158 |
+
else:
|
| 159 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 160 |
+
total_frame_num = len(vr)
|
| 161 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 162 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 163 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 164 |
+
return spare_frames # (frames, height, width, channels)
|
| 165 |
+
|
| 166 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 167 |
+
res = []
|
| 168 |
+
|
| 169 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 170 |
+
|
| 171 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 172 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 173 |
+
visuals = self.flatten(visuals)
|
| 174 |
+
|
| 175 |
+
messages = [{"role": "user", "content": []}]
|
| 176 |
+
images = []
|
| 177 |
+
|
| 178 |
+
for visual in visuals:
|
| 179 |
+
if isinstance(visual, str):
|
| 180 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 181 |
+
frames = torch.from_numpy(frames).permute(0, 3, 1, 2)
|
| 182 |
+
images.extend([to_pil_image(frame) for frame in frames])
|
| 183 |
+
elif isinstance(visual, PIL.Image.Image):
|
| 184 |
+
images.append(visual)
|
| 185 |
+
|
| 186 |
+
for _ in range(len(images)):
|
| 187 |
+
messages[-1]["content"].append({"type": "image"})
|
| 188 |
+
messages[-1]["content"].append({"type": "text", "text": contexts})
|
| 189 |
+
prompt = self.processor.apply_chat_template(messages, add_generation_prompt=True)
|
| 190 |
+
inputs = self.processor(images, prompt, add_special_tokens=False, return_tensors="pt").to(self.model.device)
|
| 191 |
+
|
| 192 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 193 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 194 |
+
if "temperature" not in gen_kwargs:
|
| 195 |
+
gen_kwargs["temperature"] = 0
|
| 196 |
+
if "top_p" not in gen_kwargs:
|
| 197 |
+
gen_kwargs["top_p"] = None
|
| 198 |
+
if "num_beams" not in gen_kwargs:
|
| 199 |
+
gen_kwargs["num_beams"] = 1
|
| 200 |
+
if "do_sample" not in gen_kwargs:
|
| 201 |
+
gen_kwargs["do_sample"] = False
|
| 202 |
+
|
| 203 |
+
with torch.no_grad():
|
| 204 |
+
output = self.model.generate(
|
| 205 |
+
**inputs,
|
| 206 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 207 |
+
temperature=gen_kwargs["temperature"],
|
| 208 |
+
do_sample=gen_kwargs["do_sample"],
|
| 209 |
+
)
|
| 210 |
+
output = output[:, inputs["input_ids"].shape[-1] :]
|
| 211 |
+
res.append(self.processor.decode(output[0], skip_special_tokens=True))
|
| 212 |
+
|
| 213 |
+
pbar.update(1)
|
| 214 |
+
pbar.close()
|
| 215 |
+
return res
|
| 216 |
+
|
| 217 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 218 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
|
LaViDa/eval/lmms_eval/models/llava.py
ADDED
|
@@ -0,0 +1,442 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
import copy
|
| 7 |
+
import warnings
|
| 8 |
+
from datetime import timedelta
|
| 9 |
+
from typing import List, Optional, Tuple, Union
|
| 10 |
+
|
| 11 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 12 |
+
from accelerate.state import AcceleratorState
|
| 13 |
+
from packaging import version
|
| 14 |
+
from tqdm import tqdm
|
| 15 |
+
|
| 16 |
+
from lmms_eval import utils
|
| 17 |
+
from lmms_eval.api.instance import Instance
|
| 18 |
+
from lmms_eval.api.model import lmms
|
| 19 |
+
from lmms_eval.api.registry import register_model
|
| 20 |
+
from lmms_eval.utils import stop_sequences_criteria
|
| 21 |
+
import time
|
| 22 |
+
warnings.filterwarnings("ignore")
|
| 23 |
+
|
| 24 |
+
from loguru import logger as eval_logger
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
|
| 28 |
+
from llava.conversation import conv_templates
|
| 29 |
+
from llava.mm_utils import (
|
| 30 |
+
get_model_name_from_path,
|
| 31 |
+
process_images,
|
| 32 |
+
tokenizer_image_token,
|
| 33 |
+
)
|
| 34 |
+
from llava.model.builder import load_pretrained_model
|
| 35 |
+
except Exception as e:
|
| 36 |
+
eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
|
| 37 |
+
|
| 38 |
+
# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 39 |
+
# if is_flash_attn_2_available:
|
| 40 |
+
# best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
|
| 41 |
+
|
| 42 |
+
if version.parse(torch.__version__) >= version.parse("2.1.2"):
|
| 43 |
+
best_fit_attn_implementation = "sdpa"
|
| 44 |
+
else:
|
| 45 |
+
best_fit_attn_implementation = "eager"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@register_model("llava")
|
| 49 |
+
class Llava(lmms):
|
| 50 |
+
"""
|
| 51 |
+
Llava Model
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
def __init__(
|
| 55 |
+
self,
|
| 56 |
+
pretrained: str = "liuhaotian/llava-v1.5-7b",
|
| 57 |
+
truncation: Optional[bool] = True,
|
| 58 |
+
device: Optional[str] = "cuda:0",
|
| 59 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 60 |
+
model_name=None,
|
| 61 |
+
attn_implementation=best_fit_attn_implementation,
|
| 62 |
+
device_map="cuda:0",
|
| 63 |
+
conv_template="vicuna_v1",
|
| 64 |
+
use_cache=True,
|
| 65 |
+
tie_weights: bool = True,
|
| 66 |
+
truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 67 |
+
customized_config=None, # ends in json
|
| 68 |
+
**kwargs,
|
| 69 |
+
) -> None:
|
| 70 |
+
super().__init__()
|
| 71 |
+
# Do not use kwargs for now
|
| 72 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 73 |
+
|
| 74 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 75 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 76 |
+
self.accelerator = accelerator
|
| 77 |
+
if accelerator.num_processes > 1:
|
| 78 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 79 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 80 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 81 |
+
self._device = torch.device(device)
|
| 82 |
+
self.device_map = device_map
|
| 83 |
+
else:
|
| 84 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 85 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 86 |
+
|
| 87 |
+
llava_model_args = {
|
| 88 |
+
"multimodal": True,
|
| 89 |
+
}
|
| 90 |
+
if customized_config is not None:
|
| 91 |
+
llava_model_args["customized_config"] = customized_config
|
| 92 |
+
if attn_implementation is not None:
|
| 93 |
+
llava_model_args["attn_implementation"] = attn_implementation
|
| 94 |
+
if "use_flash_attention_2" in kwargs:
|
| 95 |
+
llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
|
| 96 |
+
model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
|
| 97 |
+
|
| 98 |
+
try:
|
| 99 |
+
# Try to load the model with the multimodal argument
|
| 100 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map='cpu', **llava_model_args)
|
| 101 |
+
self._model.to(torch.float16).to('cuda')
|
| 102 |
+
except TypeError:
|
| 103 |
+
# for older versions of LLaVA that don't have multimodal argument
|
| 104 |
+
llava_model_args.pop("multimodal", None)
|
| 105 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 106 |
+
self._config = self._model.config
|
| 107 |
+
self.model.eval()
|
| 108 |
+
if tie_weights:
|
| 109 |
+
self.model.tie_weights()
|
| 110 |
+
|
| 111 |
+
self.truncation = truncation
|
| 112 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 113 |
+
self.conv_template = conv_template
|
| 114 |
+
self.use_cache = use_cache
|
| 115 |
+
self.truncate_context = truncate_context
|
| 116 |
+
# assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 117 |
+
if accelerator.num_processes > 1:
|
| 118 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 119 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 120 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 121 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 122 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 123 |
+
kwargs = {
|
| 124 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 125 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 126 |
+
}
|
| 127 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 128 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 129 |
+
|
| 130 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 131 |
+
self._model = accelerator.prepare(self.model)
|
| 132 |
+
else:
|
| 133 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 134 |
+
self.accelerator = accelerator
|
| 135 |
+
if self.accelerator.is_local_main_process:
|
| 136 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 137 |
+
self._rank = self.accelerator.local_process_index
|
| 138 |
+
self._world_size = self.accelerator.num_processes
|
| 139 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 140 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 141 |
+
self._rank = 0
|
| 142 |
+
self._world_size = 1
|
| 143 |
+
else:
|
| 144 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 145 |
+
self.model.to(self._device)
|
| 146 |
+
self._rank = 0
|
| 147 |
+
self._world_size = 1
|
| 148 |
+
|
| 149 |
+
@property
|
| 150 |
+
def config(self):
|
| 151 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 152 |
+
return self._config
|
| 153 |
+
|
| 154 |
+
@property
|
| 155 |
+
def tokenizer(self):
|
| 156 |
+
return self._tokenizer
|
| 157 |
+
|
| 158 |
+
@property
|
| 159 |
+
def model(self):
|
| 160 |
+
# returns the model, unwrapping it if using Accelerate
|
| 161 |
+
if hasattr(self, "accelerator"):
|
| 162 |
+
return self.accelerator.unwrap_model(self._model)
|
| 163 |
+
else:
|
| 164 |
+
return self._model
|
| 165 |
+
|
| 166 |
+
@property
|
| 167 |
+
def eot_token_id(self):
|
| 168 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 169 |
+
return self.tokenizer.eos_token_id
|
| 170 |
+
|
| 171 |
+
@property
|
| 172 |
+
def max_length(self):
|
| 173 |
+
return self._max_length
|
| 174 |
+
|
| 175 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 176 |
+
if self.tokenizer.padding_side == "left":
|
| 177 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 178 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 179 |
+
if self.tokenizer.padding_side == "left":
|
| 180 |
+
input_ids = torch.flip(input_ids, [1])
|
| 181 |
+
return input_ids
|
| 182 |
+
|
| 183 |
+
@property
|
| 184 |
+
def batch_size(self):
|
| 185 |
+
return self.batch_size_per_gpu
|
| 186 |
+
|
| 187 |
+
@property
|
| 188 |
+
def device(self):
|
| 189 |
+
return self._device
|
| 190 |
+
|
| 191 |
+
@property
|
| 192 |
+
def rank(self):
|
| 193 |
+
return self._rank
|
| 194 |
+
|
| 195 |
+
@property
|
| 196 |
+
def world_size(self):
|
| 197 |
+
return self._world_size
|
| 198 |
+
|
| 199 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 200 |
+
""" """
|
| 201 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 202 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 203 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 204 |
+
if left_truncate_len:
|
| 205 |
+
encoding = encoding[-left_truncate_len:]
|
| 206 |
+
return encoding
|
| 207 |
+
|
| 208 |
+
def tok_decode(self, tokens):
|
| 209 |
+
try:
|
| 210 |
+
return self.tokenizer.decode(tokens)
|
| 211 |
+
except:
|
| 212 |
+
return self.tokenizer.decode([tokens])
|
| 213 |
+
|
| 214 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 215 |
+
# TODO
|
| 216 |
+
res = []
|
| 217 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 218 |
+
|
| 219 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 220 |
+
# encode, pad, and truncate contexts for this batch
|
| 221 |
+
if type(doc_to_target) == str:
|
| 222 |
+
continuation = doc_to_target
|
| 223 |
+
else:
|
| 224 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 225 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 226 |
+
visuals = self.flatten(visuals)
|
| 227 |
+
image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
|
| 228 |
+
if visuals:
|
| 229 |
+
image = process_images(visuals, self._image_processor, self._config)
|
| 230 |
+
if type(image) is list:
|
| 231 |
+
image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
|
| 232 |
+
else:
|
| 233 |
+
image = image.to(dtype=torch.float16, device=self.device)
|
| 234 |
+
else:
|
| 235 |
+
image = None
|
| 236 |
+
|
| 237 |
+
prompts_input = contexts[0] if isinstance(contexts, list) else contexts
|
| 238 |
+
|
| 239 |
+
if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
|
| 240 |
+
"""
|
| 241 |
+
Three senarios:
|
| 242 |
+
1. No image, and there for, no image token should be added.
|
| 243 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 244 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 245 |
+
"""
|
| 246 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
|
| 247 |
+
image_tokens = " ".join(image_tokens)
|
| 248 |
+
prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts)
|
| 249 |
+
|
| 250 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 251 |
+
if "llama_3" in self.conv_template:
|
| 252 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 253 |
+
else:
|
| 254 |
+
conv = conv_templates[self.conv_template].copy()
|
| 255 |
+
conv.append_message(conv.roles[0], prompts_input)
|
| 256 |
+
conv.append_message(conv.roles[1], None)
|
| 257 |
+
prompt = conv.get_prompt()
|
| 258 |
+
pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 259 |
+
contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 260 |
+
# Add the answer of the second role
|
| 261 |
+
conv.messages[1][1] = continuation
|
| 262 |
+
|
| 263 |
+
prompt = conv.get_prompt()
|
| 264 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 265 |
+
labels = input_ids.clone()
|
| 266 |
+
# Context part no need to calculate for loss
|
| 267 |
+
labels[0, : contxt_id.shape[1]] = -100
|
| 268 |
+
with torch.inference_mode():
|
| 269 |
+
outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
|
| 270 |
+
loss = outputs["loss"]
|
| 271 |
+
# loss = torch.exp(loss)
|
| 272 |
+
logits = outputs["logits"]
|
| 273 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 274 |
+
cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
|
| 275 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
|
| 276 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 277 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 278 |
+
pbar.update(1)
|
| 279 |
+
pbar.close()
|
| 280 |
+
return res
|
| 281 |
+
|
| 282 |
+
def flatten(self, input):
|
| 283 |
+
if not input or any(i is None for i in input):
|
| 284 |
+
return []
|
| 285 |
+
new_list = []
|
| 286 |
+
for i in input:
|
| 287 |
+
if i:
|
| 288 |
+
for j in i:
|
| 289 |
+
new_list.append(j)
|
| 290 |
+
return new_list
|
| 291 |
+
|
| 292 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 293 |
+
res = []
|
| 294 |
+
|
| 295 |
+
def _collate(x):
|
| 296 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 297 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 298 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 299 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 300 |
+
# automatic adaptive batches much much easier to implement
|
| 301 |
+
# - any OOMs will happen right away rather than near the end
|
| 302 |
+
toks = self.tok_encode(x[0])
|
| 303 |
+
return -len(toks), x[0]
|
| 304 |
+
|
| 305 |
+
# we group requests by their generation_kwargs,
|
| 306 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 307 |
+
# in the same batch.
|
| 308 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 309 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 310 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 311 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 312 |
+
delta_t = 0
|
| 313 |
+
num_generated = 0
|
| 314 |
+
for chunk in chunks:
|
| 315 |
+
t0 = time.time()
|
| 316 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 317 |
+
task = task[0]
|
| 318 |
+
split = split[0]
|
| 319 |
+
batched_visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # [B, N]
|
| 320 |
+
flattened_visuals = self.flatten(batched_visuals) # [B*N]
|
| 321 |
+
# we assume all gen kwargs in the batch are the same
|
| 322 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 323 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 324 |
+
|
| 325 |
+
# Set default values for until and max_new_tokens
|
| 326 |
+
until = [self.tok_decode(self.eot_token_id)]
|
| 327 |
+
|
| 328 |
+
# Update values from gen_kwargs if present
|
| 329 |
+
if "until" in gen_kwargs:
|
| 330 |
+
until = gen_kwargs.pop("until")
|
| 331 |
+
if isinstance(until, str):
|
| 332 |
+
until = [until]
|
| 333 |
+
elif not isinstance(until, list):
|
| 334 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 335 |
+
# self._config.image_aspect_ratio = 'baseline'
|
| 336 |
+
if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
|
| 337 |
+
# here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
|
| 338 |
+
self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
|
| 339 |
+
eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
|
| 340 |
+
# encode, pad, and truncate contexts for this batch
|
| 341 |
+
if flattened_visuals:
|
| 342 |
+
image_tensor = process_images(flattened_visuals, self._image_processor, self._config)
|
| 343 |
+
if type(image_tensor) is list:
|
| 344 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 345 |
+
else:
|
| 346 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 347 |
+
else:
|
| 348 |
+
image_tensor = None
|
| 349 |
+
|
| 350 |
+
# prompts_input = contexts[0]
|
| 351 |
+
|
| 352 |
+
question_input = []
|
| 353 |
+
|
| 354 |
+
for visual, context in zip(batched_visuals, contexts):
|
| 355 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 356 |
+
"""
|
| 357 |
+
Three senarios:
|
| 358 |
+
1. No image, and there for, no image token should be added.
|
| 359 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 360 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 361 |
+
"""
|
| 362 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 363 |
+
image_tokens = " ".join(image_tokens)
|
| 364 |
+
question = image_tokens + "\n" + context
|
| 365 |
+
else:
|
| 366 |
+
question = context
|
| 367 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 368 |
+
if "llama_3" in self.conv_template:
|
| 369 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 370 |
+
else:
|
| 371 |
+
conv = conv_templates[self.conv_template].copy()
|
| 372 |
+
conv.append_message(conv.roles[0], question)
|
| 373 |
+
conv.append_message(conv.roles[1], None)
|
| 374 |
+
prompt_question = conv.get_prompt()
|
| 375 |
+
question_input.append(prompt_question)
|
| 376 |
+
|
| 377 |
+
# input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 378 |
+
# preconfigure gen_kwargs with defaults
|
| 379 |
+
gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
|
| 380 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 381 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 382 |
+
if "temperature" not in gen_kwargs:
|
| 383 |
+
gen_kwargs["temperature"] = 0
|
| 384 |
+
if "top_p" not in gen_kwargs:
|
| 385 |
+
gen_kwargs["top_p"] = None
|
| 386 |
+
if "num_beams" not in gen_kwargs:
|
| 387 |
+
gen_kwargs["num_beams"] = 1
|
| 388 |
+
|
| 389 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 390 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 391 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 392 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 393 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 394 |
+
# TODO: attention to this major generation step...
|
| 395 |
+
try:
|
| 396 |
+
cont = self.model.generate(
|
| 397 |
+
input_ids,
|
| 398 |
+
attention_mask=attention_masks,
|
| 399 |
+
pad_token_id=pad_token_ids,
|
| 400 |
+
images=image_tensor,
|
| 401 |
+
image_sizes=gen_kwargs["image_sizes"],
|
| 402 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 403 |
+
temperature=gen_kwargs["temperature"],
|
| 404 |
+
top_p=gen_kwargs["top_p"],
|
| 405 |
+
num_beams=gen_kwargs["num_beams"],
|
| 406 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 407 |
+
use_cache=self.use_cache,
|
| 408 |
+
)
|
| 409 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 410 |
+
except Exception as e:
|
| 411 |
+
raise e
|
| 412 |
+
eval_logger.error(f"Error {e} in generating")
|
| 413 |
+
cont = ""
|
| 414 |
+
text_outputs = [""]
|
| 415 |
+
|
| 416 |
+
# cont_toks_list = cont.tolist()
|
| 417 |
+
# for cont_toks, context in zip(cont_toks_list, contexts):
|
| 418 |
+
# discard context + left-padding toks if using causal decoder-only LMM
|
| 419 |
+
# if self.truncate_context:
|
| 420 |
+
# cont_toks = cont_toks[input_ids.shape[1] :]
|
| 421 |
+
# use secondary stop seqs to cut off should-have-been-stopped content post-hoc
|
| 422 |
+
# if self.truncate_context:
|
| 423 |
+
# for term in until:
|
| 424 |
+
# if len(term) > 0:
|
| 425 |
+
# # ignore '' separator,
|
| 426 |
+
# # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
|
| 427 |
+
# text_outputs = text_outputs.split(term)[0]
|
| 428 |
+
res.extend(text_outputs)
|
| 429 |
+
t1 = time.time()
|
| 430 |
+
delta_t += t1-t0
|
| 431 |
+
num_generated +=1
|
| 432 |
+
print(f"Avg Latenc (of {num_generated}): {delta_t/num_generated}")
|
| 433 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 434 |
+
pbar.update(1)
|
| 435 |
+
# reorder this group of results back to original unsorted form
|
| 436 |
+
res = re_ords.get_original(res)
|
| 437 |
+
|
| 438 |
+
pbar.close()
|
| 439 |
+
return res
|
| 440 |
+
|
| 441 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 442 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaVA")
|
LaViDa/eval/lmms_eval/models/llava_dream.py
ADDED
|
@@ -0,0 +1,855 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import re
|
| 6 |
+
import warnings
|
| 7 |
+
from datetime import timedelta
|
| 8 |
+
from typing import List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import os
|
| 12 |
+
import PIL
|
| 13 |
+
import torch
|
| 14 |
+
import accelerate
|
| 15 |
+
import transformers
|
| 16 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 17 |
+
from accelerate.state import AcceleratorState
|
| 18 |
+
from decord import VideoReader, cpu
|
| 19 |
+
from packaging import version
|
| 20 |
+
from tqdm import tqdm
|
| 21 |
+
from transformers import AutoConfig
|
| 22 |
+
|
| 23 |
+
from lmms_eval import utils
|
| 24 |
+
from lmms_eval.api.instance import Instance
|
| 25 |
+
from lmms_eval.api.model import lmms
|
| 26 |
+
from lmms_eval.api.registry import register_model
|
| 27 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 28 |
+
|
| 29 |
+
# Suppress warnings
|
| 30 |
+
warnings.filterwarnings("ignore")
|
| 31 |
+
|
| 32 |
+
# Configure logging
|
| 33 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 34 |
+
|
| 35 |
+
# Enable TF32 for CUDA
|
| 36 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 37 |
+
|
| 38 |
+
# Import LLaVA modules
|
| 39 |
+
try:
|
| 40 |
+
from llava.constants import (
|
| 41 |
+
DEFAULT_IM_END_TOKEN,
|
| 42 |
+
DEFAULT_IM_START_TOKEN,
|
| 43 |
+
DEFAULT_IMAGE_TOKEN,
|
| 44 |
+
IGNORE_INDEX,
|
| 45 |
+
IMAGE_TOKEN_INDEX,
|
| 46 |
+
)
|
| 47 |
+
from llava.conversation import SeparatorStyle, conv_templates
|
| 48 |
+
from llava.mm_utils import (
|
| 49 |
+
KeywordsStoppingCriteria,
|
| 50 |
+
get_model_name_from_path,
|
| 51 |
+
process_images,
|
| 52 |
+
tokenizer_image_token,
|
| 53 |
+
)
|
| 54 |
+
from llava.model.builder import load_pretrained_model
|
| 55 |
+
except ImportError as e:
|
| 56 |
+
eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# Determine best attention implementation
|
| 60 |
+
if version.parse(torch.__version__) >= version.parse("2.1.2"):
|
| 61 |
+
best_fit_attn_implementation = "sdpa"
|
| 62 |
+
else:
|
| 63 |
+
best_fit_attn_implementation = "eager"
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
@register_model("llava_llada")
|
| 67 |
+
class Llava_Dream(lmms):
|
| 68 |
+
"""
|
| 69 |
+
Llava Model
|
| 70 |
+
"""
|
| 71 |
+
|
| 72 |
+
def __init__(
|
| 73 |
+
self,
|
| 74 |
+
pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
|
| 75 |
+
truncation: Optional[bool] = True,
|
| 76 |
+
device: Optional[str] = "cuda:0",
|
| 77 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 78 |
+
model_name: Optional[str] = None,
|
| 79 |
+
attn_implementation: Optional[str] = best_fit_attn_implementation,
|
| 80 |
+
device_map: Optional[str] = "cuda:0",
|
| 81 |
+
conv_template: Optional[str] = "dream",
|
| 82 |
+
use_cache: Optional[bool] = True,
|
| 83 |
+
truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 84 |
+
customized_config: Optional[str] = None, # ends in json
|
| 85 |
+
max_frames_num: Optional[int] = 32,
|
| 86 |
+
mm_spatial_pool_stride: Optional[int] = 2,
|
| 87 |
+
mm_spatial_pool_mode: Optional[str] = "bilinear",
|
| 88 |
+
token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
|
| 89 |
+
video_decode_backend: str = "decord",
|
| 90 |
+
mc_num=16,
|
| 91 |
+
**kwargs,
|
| 92 |
+
) -> None:
|
| 93 |
+
super().__init__()
|
| 94 |
+
# Do not use kwargs for now
|
| 95 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 96 |
+
|
| 97 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 98 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 99 |
+
if accelerator.num_processes > 1:
|
| 100 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 101 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 102 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 103 |
+
self._device = torch.device(device)
|
| 104 |
+
self.device_map = device_map
|
| 105 |
+
else:
|
| 106 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 107 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 108 |
+
self.mc_num = mc_num
|
| 109 |
+
llava_model_args = {
|
| 110 |
+
"multimodal": True,
|
| 111 |
+
}
|
| 112 |
+
if customized_config is not None:
|
| 113 |
+
llava_model_args["customized_config"] = customized_config
|
| 114 |
+
if attn_implementation is not None:
|
| 115 |
+
llava_model_args["attn_implementation"] = attn_implementation
|
| 116 |
+
if "use_flash_attention_2" in kwargs:
|
| 117 |
+
llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
|
| 118 |
+
model_name = 'llava_dream'# if model_name is not None else get_model_name_from_path(pretrained)
|
| 119 |
+
|
| 120 |
+
self.pretrained = pretrained
|
| 121 |
+
self.token_strategy = token_strategy
|
| 122 |
+
self.max_frames_num = max_frames_num
|
| 123 |
+
self.mm_spatial_pool_stride = mm_spatial_pool_stride
|
| 124 |
+
self.mm_spatial_pool_mode = mm_spatial_pool_mode
|
| 125 |
+
self.video_decode_backend = video_decode_backend
|
| 126 |
+
|
| 127 |
+
overwrite_config = {}
|
| 128 |
+
overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
|
| 129 |
+
overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
|
| 130 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained,trust_remote_code=True)
|
| 131 |
+
|
| 132 |
+
llava_model_args["overwrite_config"] = overwrite_config
|
| 133 |
+
# try:
|
| 134 |
+
# Try to load the model with the multimodal argument
|
| 135 |
+
|
| 136 |
+
if os.path.exists('/data1/jacklishufan/siglip-so400m-patch14-384'):
|
| 137 |
+
vision_tower_path = "/data1/jacklishufan/siglip-so400m-patch14-384"
|
| 138 |
+
else:
|
| 139 |
+
vision_tower_path="/data0/jacklishufan/siglip-so400m-patch14-384"
|
| 140 |
+
# print(vision_tower_path)
|
| 141 |
+
vision_kwargs = dict(
|
| 142 |
+
mm_vision_tower=os.environ.get('LLADA_VISION_ENCODER',vision_tower_path),
|
| 143 |
+
mm_resampler_type=None,
|
| 144 |
+
mm_projector_type='mlp2x_gelu',
|
| 145 |
+
mm_hidden_size=1152,
|
| 146 |
+
use_mm_proj=True
|
| 147 |
+
|
| 148 |
+
)
|
| 149 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args,vision_kwargs=vision_kwargs)
|
| 150 |
+
assert self._tokenizer is not None
|
| 151 |
+
# except TypeError:
|
| 152 |
+
# # for older versions of LLaVA that don't have multimodal argument
|
| 153 |
+
# llava_model_args.pop("multimodal", None)
|
| 154 |
+
# self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 155 |
+
|
| 156 |
+
self._config = self._model.config
|
| 157 |
+
self.model.eval()
|
| 158 |
+
# self.model.model.set_activation_checkpointing(None)
|
| 159 |
+
self.model.requires_grad_(False)
|
| 160 |
+
self.truncation = truncation
|
| 161 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 162 |
+
self.conv_template = conv_template
|
| 163 |
+
self.use_cache = use_cache
|
| 164 |
+
self.truncate_context = truncate_context
|
| 165 |
+
assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 166 |
+
|
| 167 |
+
if accelerator.num_processes > 1:
|
| 168 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 169 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 170 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 171 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 172 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 173 |
+
kwargs = {
|
| 174 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 175 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 176 |
+
}
|
| 177 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 178 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 179 |
+
|
| 180 |
+
# if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 181 |
+
# self._model = accelerator.prepare(self.model)
|
| 182 |
+
# else:
|
| 183 |
+
# self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 184 |
+
self.model.to(self._device).to(torch.bfloat16)
|
| 185 |
+
#self._model.model.transformer = accelerator.prepare(self.model.model.transformer)
|
| 186 |
+
|
| 187 |
+
self.accelerator = accelerator
|
| 188 |
+
if self.accelerator.is_local_main_process:
|
| 189 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 190 |
+
self._rank = self.accelerator.local_process_index
|
| 191 |
+
self._world_size = self.accelerator.num_processes
|
| 192 |
+
|
| 193 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 194 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 195 |
+
self._rank = 0
|
| 196 |
+
self._world_size = 1
|
| 197 |
+
|
| 198 |
+
else:
|
| 199 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 200 |
+
self.model.to(self._device).to(torch.bfloat16)
|
| 201 |
+
self._rank = 0
|
| 202 |
+
self._world_size = 1
|
| 203 |
+
#self.model.model.transformer = accelerate.cpu_offload(self.model.model.transformer)
|
| 204 |
+
|
| 205 |
+
@property
|
| 206 |
+
def config(self):
|
| 207 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 208 |
+
return self._config
|
| 209 |
+
|
| 210 |
+
@property
|
| 211 |
+
def tokenizer(self):
|
| 212 |
+
return self._tokenizer
|
| 213 |
+
|
| 214 |
+
@property
|
| 215 |
+
def model(self):
|
| 216 |
+
# returns the model, unwrapping it if using Accelerate
|
| 217 |
+
if hasattr(self, "accelerator"):
|
| 218 |
+
return self.accelerator.unwrap_model(self._model)
|
| 219 |
+
else:
|
| 220 |
+
return self._model
|
| 221 |
+
|
| 222 |
+
@property
|
| 223 |
+
def eot_token_id(self):
|
| 224 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 225 |
+
return self.tokenizer.eos_token_id
|
| 226 |
+
|
| 227 |
+
@property
|
| 228 |
+
def max_length(self):
|
| 229 |
+
return self._max_length
|
| 230 |
+
|
| 231 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 232 |
+
if self.tokenizer.padding_side == "left":
|
| 233 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 234 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 235 |
+
if self.tokenizer.padding_side == "left":
|
| 236 |
+
input_ids = torch.flip(input_ids, [1])
|
| 237 |
+
return input_ids
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def batch_size(self):
|
| 241 |
+
return self.batch_size_per_gpu
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def device(self):
|
| 245 |
+
return self._device
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def rank(self):
|
| 249 |
+
return self._rank
|
| 250 |
+
|
| 251 |
+
@property
|
| 252 |
+
def world_size(self):
|
| 253 |
+
return self._world_size
|
| 254 |
+
|
| 255 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 256 |
+
""" """
|
| 257 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 258 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 259 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 260 |
+
if left_truncate_len:
|
| 261 |
+
encoding = encoding[-left_truncate_len:]
|
| 262 |
+
return encoding
|
| 263 |
+
|
| 264 |
+
def tok_decode(self, tokens):
|
| 265 |
+
try:
|
| 266 |
+
return self.tokenizer.decode(tokens)
|
| 267 |
+
except:
|
| 268 |
+
return self.tokenizer.decode([tokens])
|
| 269 |
+
|
| 270 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 271 |
+
res = []
|
| 272 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 273 |
+
|
| 274 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 275 |
+
|
| 276 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 277 |
+
visual = doc_to_visual(self.task_dict[task][split][doc_id])
|
| 278 |
+
|
| 279 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 280 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 281 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 282 |
+
|
| 283 |
+
if visual is None or visual == []:
|
| 284 |
+
visual = None
|
| 285 |
+
task_type = "text"
|
| 286 |
+
image_tensor = None
|
| 287 |
+
else:
|
| 288 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__:
|
| 289 |
+
self._config.image_aspect_ratio = "pad"
|
| 290 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 291 |
+
|
| 292 |
+
if "task_type" in self.metadata and self.metadata["task_type"] == "video" and "sample_frames" in self.metadata:
|
| 293 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 294 |
+
sample_indices = np.linspace(0, len(visual) - 1, self.metadata["sample_frames"], dtype=int)
|
| 295 |
+
visual = [visual[i] for i in sample_indices]
|
| 296 |
+
assert len(visual) == self.metadata["sample_frames"]
|
| 297 |
+
|
| 298 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 299 |
+
if type(image_tensor) is list:
|
| 300 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 301 |
+
else:
|
| 302 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 303 |
+
|
| 304 |
+
task_type = "video"
|
| 305 |
+
|
| 306 |
+
# elif type(visual[0]) == PIL.Image.Image:
|
| 307 |
+
elif isinstance(visual[0], PIL.Image.Image):
|
| 308 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 309 |
+
if type(image_tensor) is list:
|
| 310 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 311 |
+
else:
|
| 312 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 313 |
+
|
| 314 |
+
task_type = "image"
|
| 315 |
+
|
| 316 |
+
elif type(visual[0]) == str:
|
| 317 |
+
image_tensor = []
|
| 318 |
+
try:
|
| 319 |
+
if self.video_decode_backend == "decord":
|
| 320 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 321 |
+
elif self.video_decode_backend == "pyav":
|
| 322 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 323 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 324 |
+
image_tensor.append(frames)
|
| 325 |
+
except Exception as e:
|
| 326 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 327 |
+
image_tensor = None
|
| 328 |
+
|
| 329 |
+
task_type = "video"
|
| 330 |
+
|
| 331 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in contexts:
|
| 332 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 333 |
+
if task_type == "video":
|
| 334 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 335 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 336 |
+
image_tokens = " ".join(image_tokens)
|
| 337 |
+
prompts_input = image_tokens + "\n" + contexts
|
| 338 |
+
else:
|
| 339 |
+
prompts_input = contexts
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
assert self.conv_template=="dream"
|
| 343 |
+
|
| 344 |
+
if "llama_3" in self.conv_template or 'llada' in self.conv_template or 'dream' in self.conv_template:
|
| 345 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 346 |
+
else:
|
| 347 |
+
conv = conv_templates[self.conv_template].copy()
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
conv.append_message(conv.roles[0], prompts_input)
|
| 351 |
+
conv.append_message(conv.roles[1], None)
|
| 352 |
+
prompt = conv.get_prompt()
|
| 353 |
+
# breakpoint()
|
| 354 |
+
|
| 355 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 356 |
+
|
| 357 |
+
if type(doc_to_target) == str:
|
| 358 |
+
continuation = doc_to_target
|
| 359 |
+
else:
|
| 360 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 361 |
+
|
| 362 |
+
# conv.messages[-1][1] = continuation
|
| 363 |
+
# full_prompt = conv.get_prompt()
|
| 364 |
+
# full_input_ids = tokenizer_image_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 365 |
+
|
| 366 |
+
# labels = full_input_ids.clone()
|
| 367 |
+
# labels[0, : input_ids.shape[1]] = -100
|
| 368 |
+
input_prompt = conv.get_prompt()
|
| 369 |
+
input_ids = tokenizer_image_token(input_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 370 |
+
answers = continuation
|
| 371 |
+
answer_ids = self.tokenizer(continuation)['input_ids']
|
| 372 |
+
answer_ids = torch.tensor(continuation).to(input_ids.device).unsqueeze(0)
|
| 373 |
+
|
| 374 |
+
|
| 375 |
+
kwargs = {}
|
| 376 |
+
if task_type == "image":
|
| 377 |
+
kwargs["image_sizes"] = [[v.size[0], v.size[1]] for v in visual] if isinstance(visual, list) else [[visual.size[0], visual.size[1]]]
|
| 378 |
+
elif task_type == "video":
|
| 379 |
+
kwargs["modalities"] = ["video"]
|
| 380 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 381 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 382 |
+
|
| 383 |
+
torch.cuda.empty_cache()
|
| 384 |
+
# with torch.inference_mode():
|
| 385 |
+
#outputs = self.model(input_ids=full_input_ids, labels=labels, images=image_tensor, use_cache=True, **kwargs)
|
| 386 |
+
likelyhoods = self.model.log_likelyhood_inference(
|
| 387 |
+
input_ids,
|
| 388 |
+
images=image_tensor.to(torch.bfloat16),
|
| 389 |
+
image_sizes=None,
|
| 390 |
+
verbose=True,
|
| 391 |
+
answer=answer_ids,
|
| 392 |
+
mc_num=self.mc_num,
|
| 393 |
+
)
|
| 394 |
+
|
| 395 |
+
# loss = outputs["loss"]
|
| 396 |
+
# logits = outputs["logits"]
|
| 397 |
+
# greedy_tokens = logits.argmax(dim=-1)
|
| 398 |
+
# cont_toks = full_input_ids[:, input_ids.shape[1] :]
|
| 399 |
+
# greedy_tokens = greedy_tokens[:, input_ids.shape[1] : full_input_ids.shape[1]]
|
| 400 |
+
# max_equal = (greedy_tokens == cont_toks).all()
|
| 401 |
+
# lmms eval return loss
|
| 402 |
+
res.append((float(-likelyhoods.item()), False))
|
| 403 |
+
pbar.update(1)
|
| 404 |
+
|
| 405 |
+
pbar.close()
|
| 406 |
+
return res
|
| 407 |
+
|
| 408 |
+
def flatten(self, input):
|
| 409 |
+
if not input or any(i is None for i in input):
|
| 410 |
+
return []
|
| 411 |
+
new_list = []
|
| 412 |
+
for i in input:
|
| 413 |
+
if i:
|
| 414 |
+
for j in i:
|
| 415 |
+
new_list.append(j)
|
| 416 |
+
return new_list
|
| 417 |
+
|
| 418 |
+
def load_video(self, video_path, max_frames_num):
|
| 419 |
+
if type(video_path) == str:
|
| 420 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 421 |
+
else:
|
| 422 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 423 |
+
total_frame_num = len(vr)
|
| 424 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 425 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 426 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 427 |
+
return spare_frames # (frames, height, width, channels)
|
| 428 |
+
|
| 429 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 430 |
+
res = []
|
| 431 |
+
|
| 432 |
+
def _collate(x):
|
| 433 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 434 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 435 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 436 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 437 |
+
# automatic adaptive batches much much easier to implement
|
| 438 |
+
# - any OOMs will happen right away rather than near the end
|
| 439 |
+
toks = self.tok_encode(x[0])
|
| 440 |
+
return -len(toks), x[0]
|
| 441 |
+
|
| 442 |
+
# we group requests by their generation_kwargs,
|
| 443 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 444 |
+
# in the same batch.
|
| 445 |
+
metadata = requests[0].metadata
|
| 446 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 447 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 448 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 449 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 450 |
+
|
| 451 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 452 |
+
|
| 453 |
+
for chunk in chunks:
|
| 454 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 455 |
+
task = batched_task[0]
|
| 456 |
+
split = batched_split[0]
|
| 457 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 458 |
+
assert len(batched_visuals) == 1
|
| 459 |
+
|
| 460 |
+
# we assume all gen kwargs in the batch are the same
|
| 461 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 462 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 463 |
+
if "until" in gen_kwargs:
|
| 464 |
+
gen_kwargs.pop("until")
|
| 465 |
+
|
| 466 |
+
question_input = []
|
| 467 |
+
# import ipdb; ipdb.set_trace()
|
| 468 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 469 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 470 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 471 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 472 |
+
|
| 473 |
+
if visual is None or visual == []: # for text-only tasks.
|
| 474 |
+
visual = None
|
| 475 |
+
task_type = "text"
|
| 476 |
+
placeholder_count = 0
|
| 477 |
+
image_tensor = None
|
| 478 |
+
else:
|
| 479 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 480 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 481 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 482 |
+
print(self._config.image_aspect_ratio )
|
| 483 |
+
if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
|
| 484 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 485 |
+
sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
|
| 486 |
+
visual = [visual[i] for i in sample_indices]
|
| 487 |
+
assert len(visual) == metadata["sample_frames"]
|
| 488 |
+
|
| 489 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 490 |
+
if type(image_tensor) is list:
|
| 491 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 492 |
+
else:
|
| 493 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 494 |
+
|
| 495 |
+
task_type = "video"
|
| 496 |
+
placeholder_count = 1
|
| 497 |
+
|
| 498 |
+
elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
|
| 499 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 500 |
+
if type(image_tensor) is list:
|
| 501 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 502 |
+
else:
|
| 503 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 504 |
+
|
| 505 |
+
task_type = "image"
|
| 506 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 507 |
+
|
| 508 |
+
elif type(visual[0]) == str: # For video task
|
| 509 |
+
image_tensor = []
|
| 510 |
+
try:
|
| 511 |
+
if self.video_decode_backend == "decord":
|
| 512 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 513 |
+
elif self.video_decode_backend == "pyav":
|
| 514 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 515 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 516 |
+
image_tensor.append(frames)
|
| 517 |
+
except Exception as e:
|
| 518 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 519 |
+
image_tensor = None
|
| 520 |
+
|
| 521 |
+
task_type = "video"
|
| 522 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 523 |
+
|
| 524 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 525 |
+
"""
|
| 526 |
+
Three senarios:
|
| 527 |
+
1. No image, and there for, no image token should be added.
|
| 528 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 529 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 530 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 531 |
+
"""
|
| 532 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 533 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 534 |
+
# elif task_type == "video":
|
| 535 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 536 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 537 |
+
image_tokens = " ".join(image_tokens)
|
| 538 |
+
question = image_tokens + "\n" + context
|
| 539 |
+
else:
|
| 540 |
+
question = context
|
| 541 |
+
|
| 542 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 543 |
+
# print(self.conv_template)
|
| 544 |
+
if "llama_3" in self.conv_template or 'llada' in self.conv_template or 'dream' in self.conv_template:
|
| 545 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 546 |
+
else:
|
| 547 |
+
conv = conv_templates[self.conv_template].copy()
|
| 548 |
+
|
| 549 |
+
if utils.is_json(question): # conversational question input
|
| 550 |
+
question = json.loads(question)
|
| 551 |
+
for idx, item in enumerate(question):
|
| 552 |
+
role = conv.roles[idx % 2]
|
| 553 |
+
message = item["value"]
|
| 554 |
+
conv.append_message(role, message)
|
| 555 |
+
|
| 556 |
+
assert len(conv.messages) % 2 == 1
|
| 557 |
+
conv.append_message(conv.roles[1], None)
|
| 558 |
+
prompt_question = conv.get_prompt()
|
| 559 |
+
question_input.append(prompt_question)
|
| 560 |
+
else: # only simple string for question
|
| 561 |
+
conv.append_message(conv.roles[0], question)
|
| 562 |
+
conv.append_message(conv.roles[1], None)
|
| 563 |
+
# breakpoint()
|
| 564 |
+
prompt_question = conv.get_prompt()
|
| 565 |
+
question_input.append(prompt_question)
|
| 566 |
+
# breakpoint()
|
| 567 |
+
# preconfigure gen_kwargs with defaults
|
| 568 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 569 |
+
gen_kwargs["max_new_tokens"] = 256
|
| 570 |
+
|
| 571 |
+
if "temperature" not in gen_kwargs:
|
| 572 |
+
gen_kwargs["temperature"] = 0
|
| 573 |
+
if "do_sample" not in gen_kwargs:
|
| 574 |
+
gen_kwargs["do_sample"] = False
|
| 575 |
+
if "top_p" not in gen_kwargs:
|
| 576 |
+
gen_kwargs["top_p"] = None
|
| 577 |
+
if "num_beams" not in gen_kwargs:
|
| 578 |
+
gen_kwargs["num_beams"] = 1
|
| 579 |
+
schedule_kwargs = {}
|
| 580 |
+
for key in list(gen_kwargs.keys()):
|
| 581 |
+
if key.startswith('schedule__'):
|
| 582 |
+
value = gen_kwargs.pop(key)
|
| 583 |
+
schedule_kwargs[key.replace('schedule__','')] = value
|
| 584 |
+
|
| 585 |
+
if 'block_length' not in gen_kwargs:
|
| 586 |
+
gen_kwargs['block_length'] = min(128,gen_kwargs["max_new_tokens"])
|
| 587 |
+
if 'step_per_block' not in gen_kwargs:
|
| 588 |
+
gen_kwargs['step_per_block'] = gen_kwargs['block_length']
|
| 589 |
+
gen_kwargs["temperature"] = 0
|
| 590 |
+
|
| 591 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 592 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 593 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 594 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 595 |
+
|
| 596 |
+
if task_type == "image":
|
| 597 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 598 |
+
elif task_type == "video":
|
| 599 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 600 |
+
keywords = [stop_str]
|
| 601 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 602 |
+
gen_kwargs["modalities"] = ["video"]
|
| 603 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 604 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 605 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 606 |
+
|
| 607 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 608 |
+
# TODO: attention to this major generation step...
|
| 609 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 610 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 611 |
+
# breakpoint()
|
| 612 |
+
# breakpoint()
|
| 613 |
+
if len(schedule_kwargs) > 0:
|
| 614 |
+
gen_kwargs['schedule_kwargs'] = schedule_kwargs
|
| 615 |
+
try:
|
| 616 |
+
with torch.inference_mode():
|
| 617 |
+
cont = self.model.generate(input_ids,
|
| 618 |
+
attention_mask=attention_masks,
|
| 619 |
+
pad_token_id=pad_token_ids,
|
| 620 |
+
images=image_tensor,
|
| 621 |
+
use_cache=self.use_cache,
|
| 622 |
+
**gen_kwargs).sequences
|
| 623 |
+
# cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 624 |
+
|
| 625 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 626 |
+
text_outputs = [text_output.lstrip('!').replace('<|im_end|>\n','') for text_output in text_outputs]
|
| 627 |
+
# breakpoint()
|
| 628 |
+
except Exception as e:
|
| 629 |
+
raise e
|
| 630 |
+
|
| 631 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 632 |
+
res.extend(text_outputs)
|
| 633 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 634 |
+
pbar.update(1)
|
| 635 |
+
# reorder this group of results back to original unsorted form
|
| 636 |
+
res = re_ords.get_original(res)
|
| 637 |
+
|
| 638 |
+
pbar.close()
|
| 639 |
+
return res
|
| 640 |
+
|
| 641 |
+
def generate_until_multi_round(self, requests: List[Instance]) -> List[str]:
|
| 642 |
+
res = []
|
| 643 |
+
raise NotImplementedError()
|
| 644 |
+
|
| 645 |
+
def _collate(x):
|
| 646 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 647 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 648 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 649 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 650 |
+
# automatic adaptive batches much much easier to implement
|
| 651 |
+
# - any OOMs will happen right away rather than near the end
|
| 652 |
+
toks = self.tok_encode(x[0])
|
| 653 |
+
return -len(toks), x[0]
|
| 654 |
+
|
| 655 |
+
# we group requests by their generation_kwargs,
|
| 656 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 657 |
+
# in the same batch.
|
| 658 |
+
metadata = requests[0].metadata
|
| 659 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 660 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 661 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 662 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 663 |
+
|
| 664 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 665 |
+
|
| 666 |
+
for chunk in chunks:
|
| 667 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_to_text, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 668 |
+
task = batched_task[0]
|
| 669 |
+
split = batched_split[0]
|
| 670 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 671 |
+
assert len(batched_visuals) == 1
|
| 672 |
+
|
| 673 |
+
# we assume all gen kwargs in the batch are the same
|
| 674 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 675 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 676 |
+
if "until" in gen_kwargs:
|
| 677 |
+
gen_kwargs.pop("until")
|
| 678 |
+
|
| 679 |
+
# multi round inference: terminate when receiving signal from the doc_to_text
|
| 680 |
+
round_idx = 0
|
| 681 |
+
batched_round_res = []
|
| 682 |
+
batched_previous_round_info = None
|
| 683 |
+
while True:
|
| 684 |
+
question_input = []
|
| 685 |
+
|
| 686 |
+
if round_idx != 0: # get current round visual and context from doc_to_text function
|
| 687 |
+
batched_visuals, batched_contexts, batched_terminal_singal, batched_round_res, batched_previous_round_info = list(
|
| 688 |
+
zip(
|
| 689 |
+
*[
|
| 690 |
+
batched_doc_to_text[0](
|
| 691 |
+
self.task_dict[task][split][ids],
|
| 692 |
+
previous_output=[round_res[ids_idx] for round_res in batched_round_res],
|
| 693 |
+
round_idx=round_idx,
|
| 694 |
+
previous_round_info=batched_previous_round_info[ids_idx] if batched_previous_round_info is not None else None,
|
| 695 |
+
)
|
| 696 |
+
for ids_idx, ids in enumerate(batched_doc_id)
|
| 697 |
+
]
|
| 698 |
+
)
|
| 699 |
+
)
|
| 700 |
+
# import ipdb; ipdb.set_trace()
|
| 701 |
+
batched_round_res = list(zip(*batched_round_res)) # [(r1_1, r1_2), (r2_1, r2_2), ...]
|
| 702 |
+
if batched_terminal_singal[0]: # terminal signal from doc_to_text function
|
| 703 |
+
break
|
| 704 |
+
|
| 705 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 706 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 707 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 708 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 709 |
+
|
| 710 |
+
if visual is None or visual == []: # for text-only tasks.
|
| 711 |
+
visual = None
|
| 712 |
+
task_type = "text"
|
| 713 |
+
placeholder_count = 0
|
| 714 |
+
image_tensor = None
|
| 715 |
+
else:
|
| 716 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 717 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 718 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 719 |
+
|
| 720 |
+
if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
|
| 721 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 722 |
+
sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
|
| 723 |
+
visual = [visual[i] for i in sample_indices]
|
| 724 |
+
assert len(visual) == metadata["sample_frames"]
|
| 725 |
+
|
| 726 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 727 |
+
if type(image_tensor) is list:
|
| 728 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 729 |
+
else:
|
| 730 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 731 |
+
|
| 732 |
+
task_type = "video"
|
| 733 |
+
placeholder_count = 1
|
| 734 |
+
|
| 735 |
+
elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
|
| 736 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 737 |
+
if type(image_tensor) is list:
|
| 738 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 739 |
+
else:
|
| 740 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 741 |
+
|
| 742 |
+
task_type = "image"
|
| 743 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 744 |
+
|
| 745 |
+
elif type(visual[0]) == str: # For video task
|
| 746 |
+
image_tensor = []
|
| 747 |
+
try:
|
| 748 |
+
if self.video_decode_backend == "decord":
|
| 749 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 750 |
+
elif self.video_decode_backend == "pyav":
|
| 751 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 752 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 753 |
+
image_tensor.append(frames)
|
| 754 |
+
except Exception as e:
|
| 755 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 756 |
+
image_tensor = None
|
| 757 |
+
|
| 758 |
+
task_type = "video"
|
| 759 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 760 |
+
|
| 761 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 762 |
+
"""
|
| 763 |
+
Three senarios:
|
| 764 |
+
1. No image, and there for, no image token should be added.
|
| 765 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 766 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 767 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 768 |
+
"""
|
| 769 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 770 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 771 |
+
# elif task_type == "video":
|
| 772 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 773 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 774 |
+
image_tokens = " ".join(image_tokens)
|
| 775 |
+
question = image_tokens + "\n" + context
|
| 776 |
+
else:
|
| 777 |
+
question = context
|
| 778 |
+
|
| 779 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 780 |
+
if "llama_3" in self.conv_template:
|
| 781 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 782 |
+
else:
|
| 783 |
+
conv = conv_templates[self.conv_template].copy()
|
| 784 |
+
|
| 785 |
+
if utils.is_json(question): # conversational question input
|
| 786 |
+
question = json.loads(question)
|
| 787 |
+
for idx, item in enumerate(question):
|
| 788 |
+
role = conv.roles[idx % 2]
|
| 789 |
+
message = item["value"]
|
| 790 |
+
conv.append_message(role, message)
|
| 791 |
+
|
| 792 |
+
assert len(conv.messages) % 2 == 1
|
| 793 |
+
conv.append_message(conv.roles[1], None)
|
| 794 |
+
prompt_question = conv.get_prompt()
|
| 795 |
+
question_input.append(prompt_question)
|
| 796 |
+
else: # only simple string for question
|
| 797 |
+
conv.append_message(conv.roles[0], question)
|
| 798 |
+
conv.append_message(conv.roles[1], None)
|
| 799 |
+
prompt_question = conv.get_prompt()
|
| 800 |
+
question_input.append(prompt_question)
|
| 801 |
+
|
| 802 |
+
# preconfigure gen_kwargs with defaults
|
| 803 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 804 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 805 |
+
if "temperature" not in gen_kwargs:
|
| 806 |
+
gen_kwargs["temperature"] = 0
|
| 807 |
+
if "do_sample" not in gen_kwargs:
|
| 808 |
+
gen_kwargs["do_sample"] = False
|
| 809 |
+
if "top_p" not in gen_kwargs:
|
| 810 |
+
gen_kwargs["top_p"] = None
|
| 811 |
+
if "num_beams" not in gen_kwargs:
|
| 812 |
+
gen_kwargs["num_beams"] = 1
|
| 813 |
+
|
| 814 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 815 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 816 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 817 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 818 |
+
|
| 819 |
+
if task_type == "image":
|
| 820 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 821 |
+
elif task_type == "video":
|
| 822 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 823 |
+
keywords = [stop_str]
|
| 824 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 825 |
+
gen_kwargs["modalities"] = ["video"]
|
| 826 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 827 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 828 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 829 |
+
|
| 830 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 831 |
+
# TODO: attention to this major generation step...
|
| 832 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 833 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 834 |
+
try:
|
| 835 |
+
with torch.inference_mode():
|
| 836 |
+
cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 837 |
+
# cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 838 |
+
|
| 839 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 840 |
+
except Exception as e:
|
| 841 |
+
raise e
|
| 842 |
+
|
| 843 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 844 |
+
batched_round_res.append(text_outputs)
|
| 845 |
+
|
| 846 |
+
round_idx += 1
|
| 847 |
+
|
| 848 |
+
res.extend(list(zip(*batched_round_res)))
|
| 849 |
+
self.cache_hook.add_partial("generate_until_multi_round", (context, gen_kwargs), batched_round_res)
|
| 850 |
+
pbar.update(1)
|
| 851 |
+
# reorder this group of results back to original unsorted form
|
| 852 |
+
res = re_ords.get_original(res)
|
| 853 |
+
|
| 854 |
+
pbar.close()
|
| 855 |
+
return res
|
LaViDa/eval/lmms_eval/models/llava_hf.py
ADDED
|
@@ -0,0 +1,390 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import PIL
|
| 6 |
+
import torch
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from accelerate.state import AcceleratorState
|
| 9 |
+
from decord import VideoReader, cpu
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
from transformers import (
|
| 12 |
+
AutoConfig,
|
| 13 |
+
AutoProcessor,
|
| 14 |
+
LlavaForConditionalGeneration,
|
| 15 |
+
LlavaNextForConditionalGeneration,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
from lmms_eval import utils
|
| 19 |
+
from lmms_eval.api.instance import Instance
|
| 20 |
+
from lmms_eval.api.model import lmms
|
| 21 |
+
from lmms_eval.api.registry import register_model
|
| 22 |
+
|
| 23 |
+
warnings.filterwarnings("ignore")
|
| 24 |
+
|
| 25 |
+
from loguru import logger as eval_logger
|
| 26 |
+
|
| 27 |
+
DEFAULT_IMAGE_TOKEN = "<image>"
|
| 28 |
+
DEFAULT_VIDEO_TOKEN = "<video>"
|
| 29 |
+
|
| 30 |
+
# Default chat for llava-hf/llava-1.5 models: https://huggingface.co/collections/llava-hf/llava-15-65f762d5b6941db5c2ba07e0
|
| 31 |
+
VICUNA_CHAT_TEMPLATE = "{% for message in messages %}{% if loop.index0 == 0 %}A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {{ message['content'] }} {% elif message['role'] == 'user' %}USER: {{ message['content'] }} {% else %} ASSISTANT: {{ message['content'] }}{{ eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}"
|
| 32 |
+
|
| 33 |
+
model_map = {
|
| 34 |
+
"llava": LlavaForConditionalGeneration,
|
| 35 |
+
"llava_next": LlavaNextForConditionalGeneration,
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
try:
|
| 39 |
+
from transformers import LlavaOnevisionForConditionalGeneration
|
| 40 |
+
|
| 41 |
+
model_map["llava_onevision"] = LlavaOnevisionForConditionalGeneration
|
| 42 |
+
except Exception as e:
|
| 43 |
+
eval_logger.debug("Transformers version does not support llava-onevision. Skipping.")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@register_model("llava_hf")
|
| 47 |
+
class LlavaHf(lmms):
|
| 48 |
+
"""
|
| 49 |
+
Llava Model for Hugging Face Transformers: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava
|
| 50 |
+
|
| 51 |
+
Adapted from the InstructBLIP model in lmms_eval/models/instructblip.py
|
| 52 |
+
|
| 53 |
+
Example usage:
|
| 54 |
+
|
| 55 |
+
accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
|
| 56 |
+
--model llava_hf \
|
| 57 |
+
--model_args pretrained=llava-hf/llava-1.5-7b-hf \
|
| 58 |
+
--tasks seedbench \
|
| 59 |
+
--batch_size 1 \
|
| 60 |
+
--output_path ./logs/ \
|
| 61 |
+
--log_samples
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
pretrained: str = "llava-hf/llava-1.5-7b-hf",
|
| 67 |
+
revision: str = "main",
|
| 68 |
+
device: str = "cuda",
|
| 69 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 70 |
+
batch_size: int = 1,
|
| 71 |
+
trust_remote_code: Optional[bool] = False,
|
| 72 |
+
attn_implementation: Optional[str] = None,
|
| 73 |
+
device_map: str = "",
|
| 74 |
+
chat_template: Optional[str] = None,
|
| 75 |
+
use_cache: bool = True,
|
| 76 |
+
max_frames_num: Optional[int] = 32,
|
| 77 |
+
**kwargs,
|
| 78 |
+
) -> None:
|
| 79 |
+
super().__init__()
|
| 80 |
+
# Do not use kwargs for now
|
| 81 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 82 |
+
|
| 83 |
+
accelerator = Accelerator()
|
| 84 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 85 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 86 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 87 |
+
else:
|
| 88 |
+
self._device = torch.device(device)
|
| 89 |
+
self.device_map = device_map
|
| 90 |
+
if isinstance(dtype, str) and dtype != "auto":
|
| 91 |
+
dtype = getattr(torch, dtype)
|
| 92 |
+
|
| 93 |
+
config = AutoConfig.from_pretrained(pretrained)
|
| 94 |
+
self.max_frames_num = max_frames_num
|
| 95 |
+
model_type = getattr(config, "model_type", "llava")
|
| 96 |
+
model_type = model_map[model_type]
|
| 97 |
+
self._model = model_type.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
|
| 98 |
+
|
| 99 |
+
self.pretrained = pretrained
|
| 100 |
+
self._image_processor = AutoProcessor.from_pretrained(pretrained, revision=revision, trust_remote_code=trust_remote_code)
|
| 101 |
+
# Pad from left for batched generation: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava#usage-tips
|
| 102 |
+
self._image_processor.tokenizer.padding_side = "left"
|
| 103 |
+
self._tokenizer = self._image_processor.tokenizer
|
| 104 |
+
self._config = self._model.config
|
| 105 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 106 |
+
self.chat_template = chat_template
|
| 107 |
+
self.use_cache = use_cache
|
| 108 |
+
if accelerator.num_processes > 1 and device_map == "":
|
| 109 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 110 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 111 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 112 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 113 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 114 |
+
kwargs = {
|
| 115 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 116 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 117 |
+
}
|
| 118 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 119 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 120 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 121 |
+
self._model = accelerator.prepare(self.model)
|
| 122 |
+
else:
|
| 123 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 124 |
+
self.accelerator = accelerator
|
| 125 |
+
if self.accelerator.is_local_main_process:
|
| 126 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 127 |
+
self._rank = self.accelerator.local_process_index
|
| 128 |
+
self._world_size = self.accelerator.num_processes
|
| 129 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 130 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
|
| 131 |
+
self._rank = 0
|
| 132 |
+
self._world_size = 1
|
| 133 |
+
else:
|
| 134 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 135 |
+
self.model.to(self._device)
|
| 136 |
+
self._rank = 0
|
| 137 |
+
self._world_size = 1
|
| 138 |
+
self.accelerator = accelerator
|
| 139 |
+
|
| 140 |
+
@property
|
| 141 |
+
def config(self):
|
| 142 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 143 |
+
return self._config
|
| 144 |
+
|
| 145 |
+
@property
|
| 146 |
+
def tokenizer(self):
|
| 147 |
+
return self._tokenizer
|
| 148 |
+
|
| 149 |
+
@property
|
| 150 |
+
def model(self):
|
| 151 |
+
# returns the model, unwrapping it if using Accelerate
|
| 152 |
+
if hasattr(self, "accelerator"):
|
| 153 |
+
return self.accelerator.unwrap_model(self._model)
|
| 154 |
+
else:
|
| 155 |
+
return self._model
|
| 156 |
+
|
| 157 |
+
@property
|
| 158 |
+
def eot_token_id(self):
|
| 159 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 160 |
+
return self.tokenizer.eos_token_id
|
| 161 |
+
|
| 162 |
+
@property
|
| 163 |
+
def max_length(self):
|
| 164 |
+
return self._max_length
|
| 165 |
+
|
| 166 |
+
@property
|
| 167 |
+
def batch_size(self):
|
| 168 |
+
return self.batch_size_per_gpu
|
| 169 |
+
|
| 170 |
+
@property
|
| 171 |
+
def device(self):
|
| 172 |
+
return self._device
|
| 173 |
+
|
| 174 |
+
@property
|
| 175 |
+
def rank(self):
|
| 176 |
+
return self._rank
|
| 177 |
+
|
| 178 |
+
@property
|
| 179 |
+
def world_size(self):
|
| 180 |
+
return self._world_size
|
| 181 |
+
|
| 182 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 183 |
+
""" """
|
| 184 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 185 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 186 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 187 |
+
if left_truncate_len:
|
| 188 |
+
encoding = encoding[-left_truncate_len:]
|
| 189 |
+
return encoding
|
| 190 |
+
|
| 191 |
+
def tok_decode(self, tokens):
|
| 192 |
+
return self.tokenizer.decode(tokens)
|
| 193 |
+
|
| 194 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 195 |
+
res = []
|
| 196 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 197 |
+
|
| 198 |
+
for context, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 199 |
+
# encode, pad, and truncate contexts for this batch
|
| 200 |
+
if type(doc_to_target) == str:
|
| 201 |
+
continuation = doc_to_target
|
| 202 |
+
else:
|
| 203 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 204 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 205 |
+
visuals = self.flatten(visuals)
|
| 206 |
+
|
| 207 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
|
| 208 |
+
image_tokens = " ".join(image_tokens)
|
| 209 |
+
context = f"{image_tokens}\n{context}"
|
| 210 |
+
# Apply chat template
|
| 211 |
+
messages = [{"role": "user", "content": context}, {"role": "assistant", "content": continuation}]
|
| 212 |
+
if self.chat_template is not None:
|
| 213 |
+
self.tokenizer.chat_template = self.chat_template
|
| 214 |
+
prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
|
| 215 |
+
prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
|
| 216 |
+
elif self.tokenizer.chat_template is not None:
|
| 217 |
+
prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
|
| 218 |
+
prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
|
| 219 |
+
else:
|
| 220 |
+
self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
|
| 221 |
+
prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
|
| 222 |
+
prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
|
| 223 |
+
|
| 224 |
+
formatted_contexts = [prompt]
|
| 225 |
+
formatted_continuation = [prompt_and_continuation]
|
| 226 |
+
model_inputs = self._image_processor(text=formatted_continuation, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
|
| 227 |
+
labels = model_inputs["input_ids"].clone()
|
| 228 |
+
contxt_id = self._image_processor(text=formatted_contexts, return_tensors="pt")["input_ids"]
|
| 229 |
+
labels[:, : contxt_id.shape[1]] = -100
|
| 230 |
+
|
| 231 |
+
if self.accelerator.is_main_process and doc_id % 100 == 0:
|
| 232 |
+
eval_logger.debug(f"Prompt for doc ID {doc_id}:\n\n{formatted_contexts[0]}\n")
|
| 233 |
+
eval_logger.debug(f"Prompt and continuation for doc ID {doc_id}:\n\n{formatted_continuation[0]}\n")
|
| 234 |
+
|
| 235 |
+
with torch.inference_mode():
|
| 236 |
+
outputs = self.model(**model_inputs, labels=labels)
|
| 237 |
+
loss = outputs["loss"]
|
| 238 |
+
logits = outputs["logits"]
|
| 239 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 240 |
+
cont_toks = model_inputs["input_ids"][:, contxt_id.shape[1] :] # [1, seq]
|
| 241 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : model_inputs["input_ids"].shape[1]] # [1, seq]
|
| 242 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 243 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 244 |
+
pbar.update(1)
|
| 245 |
+
|
| 246 |
+
pbar.close()
|
| 247 |
+
return res
|
| 248 |
+
|
| 249 |
+
def flatten(self, input):
|
| 250 |
+
new_list = []
|
| 251 |
+
for i in input:
|
| 252 |
+
for j in i:
|
| 253 |
+
new_list.append(j)
|
| 254 |
+
return new_list
|
| 255 |
+
|
| 256 |
+
def load_video(self, video_path, max_frames_num):
|
| 257 |
+
if type(video_path) == str:
|
| 258 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 259 |
+
else:
|
| 260 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 261 |
+
total_frame_num = len(vr)
|
| 262 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 263 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 264 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 265 |
+
return spare_frames # (frames, height, width, channels)
|
| 266 |
+
|
| 267 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 268 |
+
res = []
|
| 269 |
+
|
| 270 |
+
def _collate(x):
|
| 271 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 272 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 273 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 274 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 275 |
+
# automatic adaptive batches much much easier to implement
|
| 276 |
+
# - any OOMs will happen right away rather than near the end
|
| 277 |
+
toks = self.tok_encode(x[0])
|
| 278 |
+
return -len(toks), x[0]
|
| 279 |
+
|
| 280 |
+
# we group requests by their generation_kwargs,
|
| 281 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 282 |
+
# in the same batch.
|
| 283 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 284 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 285 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 286 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 287 |
+
for chunk in chunks:
|
| 288 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 289 |
+
task = task[0]
|
| 290 |
+
split = split[0]
|
| 291 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 292 |
+
visuals = self.flatten(visuals)
|
| 293 |
+
if len(visuals) == 0:
|
| 294 |
+
task_type = "text"
|
| 295 |
+
elif isinstance(visuals[0], PIL.Image.Image):
|
| 296 |
+
task_type = "image"
|
| 297 |
+
elif isinstance(visuals[0], str):
|
| 298 |
+
task_type = "video"
|
| 299 |
+
# we assume all gen kwargs in the batch are the same
|
| 300 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 301 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 302 |
+
|
| 303 |
+
# Set default values for until and max_new_tokens
|
| 304 |
+
until = [self.tok_decode(self.eot_token_id)]
|
| 305 |
+
|
| 306 |
+
# Update values from gen_kwargs if present
|
| 307 |
+
if "until" in gen_kwargs:
|
| 308 |
+
until = gen_kwargs.pop("until")
|
| 309 |
+
if isinstance(until, str):
|
| 310 |
+
until = [until]
|
| 311 |
+
elif not isinstance(until, list):
|
| 312 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 313 |
+
assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
|
| 314 |
+
context = contexts[0]
|
| 315 |
+
|
| 316 |
+
# Some benchmarks like MME do not contain image tokens, so we prepend them to the prompt.
|
| 317 |
+
if DEFAULT_IMAGE_TOKEN not in context:
|
| 318 |
+
if task_type == "image":
|
| 319 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
|
| 320 |
+
elif task_type == "video":
|
| 321 |
+
image_tokens = [DEFAULT_VIDEO_TOKEN] * len(visuals)
|
| 322 |
+
image_tokens = " ".join(image_tokens)
|
| 323 |
+
context = f"{image_tokens}\n{context}"
|
| 324 |
+
# Apply chat template
|
| 325 |
+
messages = [{"role": "user", "content": context}]
|
| 326 |
+
if self.chat_template is not None:
|
| 327 |
+
self.tokenizer.chat_template = self.chat_template
|
| 328 |
+
text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 329 |
+
elif self.tokenizer.chat_template is not None:
|
| 330 |
+
text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 331 |
+
else:
|
| 332 |
+
self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
|
| 333 |
+
text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 334 |
+
|
| 335 |
+
if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
|
| 336 |
+
eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
|
| 337 |
+
|
| 338 |
+
if task_type == "video":
|
| 339 |
+
try:
|
| 340 |
+
visuals = [self.load_video(visuals, self.max_frames_num)]
|
| 341 |
+
except Exception as e:
|
| 342 |
+
res.append("")
|
| 343 |
+
eval_logger.info(f"Error {e} when loading video : {visuals}")
|
| 344 |
+
pbar.update(1)
|
| 345 |
+
|
| 346 |
+
if task_type == "image":
|
| 347 |
+
inputs = self._image_processor(images=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
|
| 348 |
+
elif task_type == "video":
|
| 349 |
+
inputs = self._image_processor(videos=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
|
| 350 |
+
|
| 351 |
+
gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
|
| 352 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 353 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 354 |
+
if "temperature" not in gen_kwargs:
|
| 355 |
+
gen_kwargs["temperature"] = 0
|
| 356 |
+
if "top_p" not in gen_kwargs:
|
| 357 |
+
gen_kwargs["top_p"] = None
|
| 358 |
+
if "num_beams" not in gen_kwargs:
|
| 359 |
+
gen_kwargs["num_beams"] = 1
|
| 360 |
+
try:
|
| 361 |
+
cont = self.model.generate(
|
| 362 |
+
**inputs,
|
| 363 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 364 |
+
temperature=gen_kwargs["temperature"],
|
| 365 |
+
top_p=gen_kwargs["top_p"],
|
| 366 |
+
num_beams=gen_kwargs["num_beams"],
|
| 367 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 368 |
+
use_cache=self.use_cache,
|
| 369 |
+
pad_token_id=self.eot_token_id,
|
| 370 |
+
eos_token_id=self.eot_token_id,
|
| 371 |
+
)
|
| 372 |
+
cont = cont[:, inputs["input_ids"].shape[-1] :]
|
| 373 |
+
except Exception as e:
|
| 374 |
+
eval_logger.error(f"Error {e} in generating")
|
| 375 |
+
cont = ""
|
| 376 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0]
|
| 377 |
+
if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
|
| 378 |
+
eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n")
|
| 379 |
+
|
| 380 |
+
res.append(text_outputs)
|
| 381 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 382 |
+
pbar.update(1)
|
| 383 |
+
# reorder this group of results back to original unsorted form
|
| 384 |
+
res = re_ords.get_original(res)
|
| 385 |
+
|
| 386 |
+
pbar.close()
|
| 387 |
+
return res
|
| 388 |
+
|
| 389 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 390 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
|
LaViDa/eval/lmms_eval/models/llava_llada.py
ADDED
|
@@ -0,0 +1,894 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import re
|
| 6 |
+
import warnings
|
| 7 |
+
from datetime import timedelta
|
| 8 |
+
from typing import List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import os
|
| 12 |
+
import PIL
|
| 13 |
+
import torch
|
| 14 |
+
import accelerate
|
| 15 |
+
import transformers
|
| 16 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 17 |
+
from accelerate.state import AcceleratorState
|
| 18 |
+
from decord import VideoReader, cpu
|
| 19 |
+
from packaging import version
|
| 20 |
+
from tqdm import tqdm
|
| 21 |
+
from transformers import AutoConfig
|
| 22 |
+
|
| 23 |
+
from lmms_eval import utils
|
| 24 |
+
from lmms_eval.api.instance import Instance
|
| 25 |
+
from lmms_eval.api.model import lmms
|
| 26 |
+
from lmms_eval.api.registry import register_model
|
| 27 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 28 |
+
import time
|
| 29 |
+
# Suppress warnings
|
| 30 |
+
warnings.filterwarnings("ignore")
|
| 31 |
+
|
| 32 |
+
# Configure logging
|
| 33 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 34 |
+
|
| 35 |
+
# Enable TF32 for CUDA
|
| 36 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 37 |
+
DEBUG_PRINT_OUTPUT = os.environ.get('DEBUG_PRINT_OUTPUT',False)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _shuffle_video_tensor(video_tensor: torch.Tensor, seed: int | None = None) -> torch.Tensor:
|
| 41 |
+
if video_tensor.ndim < 1 or video_tensor.shape[0] <= 1:
|
| 42 |
+
return video_tensor.clone()
|
| 43 |
+
generator = torch.Generator(device="cpu")
|
| 44 |
+
if seed is not None:
|
| 45 |
+
generator.manual_seed(int(seed))
|
| 46 |
+
perm = torch.randperm(video_tensor.shape[0], generator=generator)
|
| 47 |
+
return video_tensor[perm.to(video_tensor.device)]
|
| 48 |
+
# Import LLaVA modules
|
| 49 |
+
DEBUG_LOAD_TRAINER = os.environ.get('DEBUG_LOAD_TRAINER',False)
|
| 50 |
+
try:
|
| 51 |
+
from llava.constants import (
|
| 52 |
+
DEFAULT_IM_END_TOKEN,
|
| 53 |
+
DEFAULT_IM_START_TOKEN,
|
| 54 |
+
DEFAULT_IMAGE_TOKEN,
|
| 55 |
+
IGNORE_INDEX,
|
| 56 |
+
IMAGE_TOKEN_INDEX,
|
| 57 |
+
)
|
| 58 |
+
from llava.conversation import SeparatorStyle, conv_templates
|
| 59 |
+
from llava.mm_utils import (
|
| 60 |
+
KeywordsStoppingCriteria,
|
| 61 |
+
get_model_name_from_path,
|
| 62 |
+
process_images,
|
| 63 |
+
tokenizer_image_token,
|
| 64 |
+
)
|
| 65 |
+
from llava.model.builder import load_pretrained_model
|
| 66 |
+
except ImportError as e:
|
| 67 |
+
eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# Determine best attention implementation
|
| 71 |
+
if version.parse(torch.__version__) >= version.parse("2.1.2"):
|
| 72 |
+
best_fit_attn_implementation = "sdpa"
|
| 73 |
+
else:
|
| 74 |
+
best_fit_attn_implementation = "eager"
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@register_model("llava_llada")
|
| 78 |
+
class Llava_Llada(lmms):
|
| 79 |
+
"""
|
| 80 |
+
Llava Model
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
def __init__(
|
| 84 |
+
self,
|
| 85 |
+
pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
|
| 86 |
+
truncation: Optional[bool] = True,
|
| 87 |
+
device: Optional[str] = "cuda:0",
|
| 88 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 89 |
+
model_name: Optional[str] = None,
|
| 90 |
+
attn_implementation: Optional[str] = best_fit_attn_implementation,
|
| 91 |
+
device_map: Optional[str] = "cuda:0",
|
| 92 |
+
conv_template: Optional[str] = "llava_llada",
|
| 93 |
+
use_cache: Optional[bool] = True,
|
| 94 |
+
truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 95 |
+
customized_config: Optional[str] = None, # ends in json
|
| 96 |
+
max_frames_num: Optional[int] = 32,
|
| 97 |
+
mm_spatial_pool_stride: Optional[int] = 2,
|
| 98 |
+
mm_spatial_pool_mode: Optional[str] = "bilinear",
|
| 99 |
+
token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
|
| 100 |
+
video_decode_backend: str = "decord",
|
| 101 |
+
mc_num=16,
|
| 102 |
+
**kwargs,
|
| 103 |
+
) -> None:
|
| 104 |
+
super().__init__()
|
| 105 |
+
# Do not use kwargs for now
|
| 106 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 107 |
+
|
| 108 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 109 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 110 |
+
if accelerator.num_processes > 1:
|
| 111 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 112 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 113 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 114 |
+
self._device = torch.device(device)
|
| 115 |
+
self.device_map = device_map
|
| 116 |
+
else:
|
| 117 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 118 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 119 |
+
self.mc_num = mc_num
|
| 120 |
+
llava_model_args = {
|
| 121 |
+
"multimodal": True,
|
| 122 |
+
}
|
| 123 |
+
if customized_config is not None:
|
| 124 |
+
llava_model_args["customized_config"] = customized_config
|
| 125 |
+
if attn_implementation is not None:
|
| 126 |
+
llava_model_args["attn_implementation"] = attn_implementation
|
| 127 |
+
if "use_flash_attention_2" in kwargs:
|
| 128 |
+
llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
|
| 129 |
+
model_name = 'llava_llada'# if model_name is not None else get_model_name_from_path(pretrained)
|
| 130 |
+
self.overwrite_image_aspect = os.environ.get("LLAVA_OVERWRITE_IMAGE_ASPECT", None)
|
| 131 |
+
self.pretrained = pretrained
|
| 132 |
+
self.token_strategy = token_strategy
|
| 133 |
+
self.max_frames_num = max_frames_num
|
| 134 |
+
self.mm_spatial_pool_stride = mm_spatial_pool_stride
|
| 135 |
+
self.mm_spatial_pool_mode = mm_spatial_pool_mode
|
| 136 |
+
self.video_decode_backend = video_decode_backend
|
| 137 |
+
|
| 138 |
+
overwrite_config = {}
|
| 139 |
+
overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
|
| 140 |
+
overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
|
| 141 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 142 |
+
|
| 143 |
+
llava_model_args["overwrite_config"] = overwrite_config
|
| 144 |
+
# try:
|
| 145 |
+
# Try to load the model with the multimodal argument
|
| 146 |
+
|
| 147 |
+
if os.path.exists('/data1/jacklishufan/siglip-so400m-patch14-384'):
|
| 148 |
+
vision_tower_path = "/data1/jacklishufan/siglip-so400m-patch14-384"
|
| 149 |
+
else:
|
| 150 |
+
vision_tower_path="/data0/jacklishufan/siglip-so400m-patch14-384"
|
| 151 |
+
# print(vision_tower_path)
|
| 152 |
+
vision_kwargs = dict(
|
| 153 |
+
mm_vision_tower=os.environ.get('LLADA_VISION_ENCODER',vision_tower_path),
|
| 154 |
+
mm_resampler_type=None,
|
| 155 |
+
mm_projector_type=os.environ.get('LLADA_VISION_PROJECTOR','mlp2x_gelu'),
|
| 156 |
+
mm_hidden_size=int(os.environ.get('LLADA_VISION_ENCODER_HIDDEN_SIZE',1152)),
|
| 157 |
+
mm_pooler_ratio=int(os.environ.get('LLADA_MM_POOLER_RATIO',2)),
|
| 158 |
+
use_mm_proj=True,
|
| 159 |
+
mm_patch_merge_type='spatial_unpad',
|
| 160 |
+
)
|
| 161 |
+
resize_embeddings = True # default behavior
|
| 162 |
+
if DEBUG_LOAD_TRAINER:
|
| 163 |
+
resize_embeddings = False
|
| 164 |
+
|
| 165 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args,vision_kwargs=vision_kwargs,resize_embeddings=resize_embeddings)
|
| 166 |
+
# breakpoint()
|
| 167 |
+
assert self._tokenizer is not None
|
| 168 |
+
# except TypeError:
|
| 169 |
+
# # for older versions of LLaVA that don't have multimodal argument
|
| 170 |
+
# llava_model_args.pop("multimodal", None)
|
| 171 |
+
# self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 172 |
+
|
| 173 |
+
self._config = self._model.config
|
| 174 |
+
self.model.eval()
|
| 175 |
+
self.model.model.set_activation_checkpointing(None)
|
| 176 |
+
self.model.requires_grad_(False)
|
| 177 |
+
self.truncation = truncation
|
| 178 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 179 |
+
self.conv_template = conv_template
|
| 180 |
+
self.use_cache = use_cache
|
| 181 |
+
self.truncate_context = truncate_context
|
| 182 |
+
assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 183 |
+
|
| 184 |
+
if accelerator.num_processes > 1:
|
| 185 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 186 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 187 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 188 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 189 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 190 |
+
kwargs = {
|
| 191 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 192 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 193 |
+
}
|
| 194 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 195 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 196 |
+
|
| 197 |
+
# if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 198 |
+
# self._model = accelerator.prepare(self.model)
|
| 199 |
+
# else:
|
| 200 |
+
# self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 201 |
+
self.model.to(self._device).to(torch.bfloat16)
|
| 202 |
+
self._model.model.transformer = accelerator.prepare(self.model.model.transformer)
|
| 203 |
+
|
| 204 |
+
self.accelerator = accelerator
|
| 205 |
+
if self.accelerator.is_local_main_process:
|
| 206 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 207 |
+
self._rank = self.accelerator.local_process_index
|
| 208 |
+
self._world_size = self.accelerator.num_processes
|
| 209 |
+
|
| 210 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 211 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 212 |
+
self._rank = 0
|
| 213 |
+
self._world_size = 1
|
| 214 |
+
|
| 215 |
+
else:
|
| 216 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 217 |
+
self.model.to(self._device).to(torch.bfloat16)
|
| 218 |
+
self._rank = 0
|
| 219 |
+
self._world_size = 1
|
| 220 |
+
#self.model.model.transformer = accelerate.cpu_offload(self.model.model.transformer)
|
| 221 |
+
|
| 222 |
+
@property
|
| 223 |
+
def config(self):
|
| 224 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 225 |
+
return self._config
|
| 226 |
+
|
| 227 |
+
@property
|
| 228 |
+
def tokenizer(self):
|
| 229 |
+
return self._tokenizer
|
| 230 |
+
|
| 231 |
+
@property
|
| 232 |
+
def model(self):
|
| 233 |
+
# returns the model, unwrapping it if using Accelerate
|
| 234 |
+
if hasattr(self, "accelerator"):
|
| 235 |
+
return self.accelerator.unwrap_model(self._model)
|
| 236 |
+
else:
|
| 237 |
+
return self._model
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def eot_token_id(self):
|
| 241 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 242 |
+
return self.tokenizer.eos_token_id
|
| 243 |
+
|
| 244 |
+
@property
|
| 245 |
+
def max_length(self):
|
| 246 |
+
return self._max_length
|
| 247 |
+
|
| 248 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 249 |
+
if self.tokenizer.padding_side == "left":
|
| 250 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 251 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 252 |
+
if self.tokenizer.padding_side == "left":
|
| 253 |
+
input_ids = torch.flip(input_ids, [1])
|
| 254 |
+
return input_ids
|
| 255 |
+
|
| 256 |
+
@property
|
| 257 |
+
def batch_size(self):
|
| 258 |
+
return self.batch_size_per_gpu
|
| 259 |
+
|
| 260 |
+
@property
|
| 261 |
+
def device(self):
|
| 262 |
+
return self._device
|
| 263 |
+
|
| 264 |
+
@property
|
| 265 |
+
def rank(self):
|
| 266 |
+
return self._rank
|
| 267 |
+
|
| 268 |
+
@property
|
| 269 |
+
def world_size(self):
|
| 270 |
+
return self._world_size
|
| 271 |
+
|
| 272 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 273 |
+
""" """
|
| 274 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 275 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 276 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 277 |
+
if left_truncate_len:
|
| 278 |
+
encoding = encoding[-left_truncate_len:]
|
| 279 |
+
return encoding
|
| 280 |
+
|
| 281 |
+
def tok_decode(self, tokens):
|
| 282 |
+
try:
|
| 283 |
+
return self.tokenizer.decode(tokens)
|
| 284 |
+
except:
|
| 285 |
+
return self.tokenizer.decode([tokens])
|
| 286 |
+
|
| 287 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 288 |
+
res = []
|
| 289 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 290 |
+
|
| 291 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 292 |
+
|
| 293 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 294 |
+
visual = doc_to_visual(self.task_dict[task][split][doc_id])
|
| 295 |
+
|
| 296 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 297 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 298 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 299 |
+
|
| 300 |
+
if visual is None or visual == []:
|
| 301 |
+
visual = None
|
| 302 |
+
task_type = "text"
|
| 303 |
+
image_tensor = None
|
| 304 |
+
else:
|
| 305 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__:
|
| 306 |
+
self._config.image_aspect_ratio = "pad"
|
| 307 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 308 |
+
|
| 309 |
+
if "task_type" in self.metadata and self.metadata["task_type"] == "video" and "sample_frames" in self.metadata:
|
| 310 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 311 |
+
sample_indices = np.linspace(0, len(visual) - 1, self.metadata["sample_frames"], dtype=int)
|
| 312 |
+
visual = [visual[i] for i in sample_indices]
|
| 313 |
+
assert len(visual) == self.metadata["sample_frames"]
|
| 314 |
+
|
| 315 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 316 |
+
if type(image_tensor) is list:
|
| 317 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 318 |
+
else:
|
| 319 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 320 |
+
|
| 321 |
+
task_type = "video"
|
| 322 |
+
|
| 323 |
+
# elif type(visual[0]) == PIL.Image.Image:
|
| 324 |
+
elif isinstance(visual[0], PIL.Image.Image):
|
| 325 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 326 |
+
if type(image_tensor) is list:
|
| 327 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 328 |
+
else:
|
| 329 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 330 |
+
|
| 331 |
+
task_type = "image"
|
| 332 |
+
|
| 333 |
+
elif type(visual[0]) == str:
|
| 334 |
+
image_tensor = []
|
| 335 |
+
try:
|
| 336 |
+
if self.video_decode_backend == "decord":
|
| 337 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 338 |
+
elif self.video_decode_backend == "pyav":
|
| 339 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 340 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 341 |
+
image_tensor.append(frames)
|
| 342 |
+
except Exception as e:
|
| 343 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 344 |
+
image_tensor = None
|
| 345 |
+
|
| 346 |
+
task_type = "video"
|
| 347 |
+
|
| 348 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in contexts:
|
| 349 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 350 |
+
if task_type == "video":
|
| 351 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 352 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 353 |
+
image_tokens = " ".join(image_tokens)
|
| 354 |
+
prompts_input = image_tokens + "\n" + contexts
|
| 355 |
+
else:
|
| 356 |
+
prompts_input = contexts
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
if "llama_3" in self.conv_template or 'llada' in self.conv_template:
|
| 360 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 361 |
+
else:
|
| 362 |
+
conv = conv_templates[self.conv_template].copy()
|
| 363 |
+
|
| 364 |
+
conv.append_message(conv.roles[0], prompts_input)
|
| 365 |
+
conv.append_message(conv.roles[1], None)
|
| 366 |
+
prompt = conv.get_prompt()
|
| 367 |
+
|
| 368 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 369 |
+
|
| 370 |
+
if type(doc_to_target) == str:
|
| 371 |
+
continuation = doc_to_target
|
| 372 |
+
else:
|
| 373 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 374 |
+
|
| 375 |
+
# conv.messages[-1][1] = continuation
|
| 376 |
+
# full_prompt = conv.get_prompt()
|
| 377 |
+
# full_input_ids = tokenizer_image_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 378 |
+
|
| 379 |
+
# labels = full_input_ids.clone()
|
| 380 |
+
# labels[0, : input_ids.shape[1]] = -100
|
| 381 |
+
input_prompt = conv.get_prompt()
|
| 382 |
+
input_ids = tokenizer_image_token(input_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 383 |
+
answers = continuation
|
| 384 |
+
answer_ids = self.tokenizer(continuation)['input_ids']
|
| 385 |
+
answer_ids = torch.tensor(continuation).to(input_ids.device).unsqueeze(0)
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
kwargs = {}
|
| 389 |
+
if task_type == "image":
|
| 390 |
+
kwargs["image_sizes"] = [[v.size[0], v.size[1]] for v in visual] if isinstance(visual, list) else [[visual.size[0], visual.size[1]]]
|
| 391 |
+
elif task_type == "video":
|
| 392 |
+
kwargs["modalities"] = ["video"]
|
| 393 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 394 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 395 |
+
|
| 396 |
+
torch.cuda.empty_cache()
|
| 397 |
+
# with torch.inference_mode():
|
| 398 |
+
#outputs = self.model(input_ids=full_input_ids, labels=labels, images=image_tensor, use_cache=True, **kwargs)
|
| 399 |
+
likelyhoods = self.model.log_likelyhood_inference(
|
| 400 |
+
input_ids,
|
| 401 |
+
images=image_tensor.to(torch.bfloat16),
|
| 402 |
+
image_sizes=None,
|
| 403 |
+
verbose=True,
|
| 404 |
+
answer=answer_ids,
|
| 405 |
+
mc_num=self.mc_num,
|
| 406 |
+
)
|
| 407 |
+
|
| 408 |
+
# loss = outputs["loss"]
|
| 409 |
+
# logits = outputs["logits"]
|
| 410 |
+
# greedy_tokens = logits.argmax(dim=-1)
|
| 411 |
+
# cont_toks = full_input_ids[:, input_ids.shape[1] :]
|
| 412 |
+
# greedy_tokens = greedy_tokens[:, input_ids.shape[1] : full_input_ids.shape[1]]
|
| 413 |
+
# max_equal = (greedy_tokens == cont_toks).all()
|
| 414 |
+
# lmms eval return loss
|
| 415 |
+
res.append((float(-likelyhoods.item()), False))
|
| 416 |
+
pbar.update(1)
|
| 417 |
+
|
| 418 |
+
pbar.close()
|
| 419 |
+
return res
|
| 420 |
+
|
| 421 |
+
def flatten(self, input):
|
| 422 |
+
if not input or any(i is None for i in input):
|
| 423 |
+
return []
|
| 424 |
+
new_list = []
|
| 425 |
+
for i in input:
|
| 426 |
+
if i:
|
| 427 |
+
for j in i:
|
| 428 |
+
new_list.append(j)
|
| 429 |
+
return new_list
|
| 430 |
+
|
| 431 |
+
def load_video(self, video_path, max_frames_num):
|
| 432 |
+
if type(video_path) == str:
|
| 433 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 434 |
+
else:
|
| 435 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 436 |
+
total_frame_num = len(vr)
|
| 437 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 438 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 439 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 440 |
+
return spare_frames # (frames, height, width, channels)
|
| 441 |
+
|
| 442 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 443 |
+
res = []
|
| 444 |
+
|
| 445 |
+
def _collate(x):
|
| 446 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 447 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 448 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 449 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 450 |
+
# automatic adaptive batches much much easier to implement
|
| 451 |
+
# - any OOMs will happen right away rather than near the end
|
| 452 |
+
toks = self.tok_encode(x[0])
|
| 453 |
+
# breakpoint()
|
| 454 |
+
return -len(toks), x[0]
|
| 455 |
+
|
| 456 |
+
# we group requests by their generation_kwargs,
|
| 457 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 458 |
+
# in the same batch.
|
| 459 |
+
metadata = requests[0].metadata
|
| 460 |
+
if DEBUG_PRINT_OUTPUT:
|
| 461 |
+
# do not sort by length, instead using lambda x:x[-3]
|
| 462 |
+
re_ords = utils.Collator([reg.args for reg in requests], lambda x:x[-3], grouping=True)
|
| 463 |
+
else:
|
| 464 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 465 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 466 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 467 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 468 |
+
|
| 469 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 470 |
+
if DEBUG_LOAD_TRAINER:
|
| 471 |
+
ckpt1 = torch.load(DEBUG_LOAD_TRAINER, map_location='cpu')
|
| 472 |
+
ckpt1 = {k.replace('module.model','model'):v for k,v in ckpt1.items()}
|
| 473 |
+
_res = self.model.load_state_dict(ckpt1,strict=False)
|
| 474 |
+
print(f"DEBUG_LOAD_TRAINER:{DEBUG_LOAD_TRAINER} {_res}")
|
| 475 |
+
print("Something is broken if above line does not show all keys matched!!!")
|
| 476 |
+
del ckpt1
|
| 477 |
+
delta_t = 0
|
| 478 |
+
num_generated = 0
|
| 479 |
+
for chunk in chunks:
|
| 480 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 481 |
+
task = batched_task[0]
|
| 482 |
+
split = batched_split[0]
|
| 483 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 484 |
+
assert len(batched_visuals) == 1
|
| 485 |
+
|
| 486 |
+
# we assume all gen kwargs in the batch are the same
|
| 487 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 488 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 489 |
+
if "until" in gen_kwargs:
|
| 490 |
+
gen_kwargs.pop("until")
|
| 491 |
+
|
| 492 |
+
question_input = []
|
| 493 |
+
# import ipdb; ipdb.set_trace()
|
| 494 |
+
|
| 495 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 496 |
+
t0 = time.time()
|
| 497 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 498 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 499 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 500 |
+
if self.overwrite_image_aspect:
|
| 501 |
+
self._config.image_aspect_ratio = self.overwrite_image_aspect
|
| 502 |
+
if visual is None or visual == []: # for text-only tasks.
|
| 503 |
+
visual = None
|
| 504 |
+
task_type = "text"
|
| 505 |
+
placeholder_count = 0
|
| 506 |
+
image_tensor = None
|
| 507 |
+
else:
|
| 508 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 509 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 510 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 511 |
+
|
| 512 |
+
if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
|
| 513 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 514 |
+
sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
|
| 515 |
+
visual = [visual[i] for i in sample_indices]
|
| 516 |
+
assert len(visual) == metadata["sample_frames"]
|
| 517 |
+
|
| 518 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 519 |
+
if type(image_tensor) is list:
|
| 520 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 521 |
+
else:
|
| 522 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 523 |
+
|
| 524 |
+
task_type = "video"
|
| 525 |
+
placeholder_count = 1
|
| 526 |
+
|
| 527 |
+
elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
|
| 528 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 529 |
+
if type(image_tensor) is list:
|
| 530 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 531 |
+
else:
|
| 532 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 533 |
+
|
| 534 |
+
task_type = "image"
|
| 535 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 536 |
+
|
| 537 |
+
elif type(visual[0]) == str: # For video task
|
| 538 |
+
image_tensor = []
|
| 539 |
+
try:
|
| 540 |
+
if self.video_decode_backend == "decord":
|
| 541 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 542 |
+
elif self.video_decode_backend == "pyav":
|
| 543 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 544 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 545 |
+
image_tensor.append(frames)
|
| 546 |
+
except Exception as e:
|
| 547 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 548 |
+
image_tensor = None
|
| 549 |
+
|
| 550 |
+
task_type = "video"
|
| 551 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 552 |
+
|
| 553 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 554 |
+
"""
|
| 555 |
+
Three senarios:
|
| 556 |
+
1. No image, and there for, no image token should be added.
|
| 557 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 558 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 559 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 560 |
+
"""
|
| 561 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 562 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 563 |
+
# elif task_type == "video":
|
| 564 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 565 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 566 |
+
image_tokens = " ".join(image_tokens)
|
| 567 |
+
question = image_tokens + "\n" + context
|
| 568 |
+
else:
|
| 569 |
+
question = context
|
| 570 |
+
|
| 571 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 572 |
+
# print(self.conv_template)
|
| 573 |
+
if "llama_3" in self.conv_template or 'llada' in self.conv_template:
|
| 574 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 575 |
+
else:
|
| 576 |
+
conv = conv_templates[self.conv_template].copy()
|
| 577 |
+
|
| 578 |
+
if utils.is_json(question): # conversational question input
|
| 579 |
+
question = json.loads(question)
|
| 580 |
+
for idx, item in enumerate(question):
|
| 581 |
+
role = conv.roles[idx % 2]
|
| 582 |
+
message = item["value"]
|
| 583 |
+
conv.append_message(role, message)
|
| 584 |
+
|
| 585 |
+
assert len(conv.messages) % 2 == 1
|
| 586 |
+
conv.append_message(conv.roles[1], None)
|
| 587 |
+
prompt_question = conv.get_prompt()
|
| 588 |
+
question_input.append(prompt_question)
|
| 589 |
+
else: # only simple string for question
|
| 590 |
+
conv.append_message(conv.roles[0], question)
|
| 591 |
+
conv.append_message(conv.roles[1], None)
|
| 592 |
+
# breakpoint()
|
| 593 |
+
prompt_question = conv.get_prompt()
|
| 594 |
+
question_input.append(prompt_question)
|
| 595 |
+
|
| 596 |
+
# preconfigure gen_kwargs with defaults
|
| 597 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 598 |
+
gen_kwargs["max_new_tokens"] = 256
|
| 599 |
+
|
| 600 |
+
if "temperature" not in gen_kwargs:
|
| 601 |
+
gen_kwargs["temperature"] = 0
|
| 602 |
+
if "do_sample" not in gen_kwargs:
|
| 603 |
+
gen_kwargs["do_sample"] = False
|
| 604 |
+
if "top_p" not in gen_kwargs:
|
| 605 |
+
gen_kwargs["top_p"] = None
|
| 606 |
+
if "num_beams" not in gen_kwargs:
|
| 607 |
+
gen_kwargs["num_beams"] = 1
|
| 608 |
+
schedule_kwargs = {}
|
| 609 |
+
for key in list(gen_kwargs.keys()):
|
| 610 |
+
if key.startswith('schedule__'):
|
| 611 |
+
value = gen_kwargs.pop(key)
|
| 612 |
+
schedule_kwargs[key.replace('schedule__','')] = value
|
| 613 |
+
if len(schedule_kwargs) > 0:
|
| 614 |
+
gen_kwargs['schedule_kwargs'] = schedule_kwargs
|
| 615 |
+
|
| 616 |
+
if 'block_length' not in gen_kwargs:
|
| 617 |
+
gen_kwargs['block_length'] = min(128,gen_kwargs["max_new_tokens"])
|
| 618 |
+
if 'step_per_block' not in gen_kwargs and 'step_ratio' not in gen_kwargs:
|
| 619 |
+
gen_kwargs['step_per_block'] = gen_kwargs['block_length']
|
| 620 |
+
gen_kwargs["temperature"] = 0
|
| 621 |
+
|
| 622 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 623 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 624 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 625 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 626 |
+
|
| 627 |
+
if task_type == "image":
|
| 628 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 629 |
+
elif task_type == "video":
|
| 630 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 631 |
+
keywords = [stop_str]
|
| 632 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 633 |
+
gen_kwargs["modalities"] = ["video"]
|
| 634 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 635 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 636 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 637 |
+
if gen_kwargs.pop("use_tvcd", False):
|
| 638 |
+
shuffle_seed = gen_kwargs.pop("tvcd_shuffle_seed", 42)
|
| 639 |
+
gen_kwargs["images_cd"] = [_shuffle_video_tensor(image_tensor[0], shuffle_seed)]
|
| 640 |
+
|
| 641 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 642 |
+
# TODO: attention to this major generation step...
|
| 643 |
+
# breakpoint()
|
| 644 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 645 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 646 |
+
try:
|
| 647 |
+
with torch.inference_mode():
|
| 648 |
+
cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 649 |
+
# cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 650 |
+
|
| 651 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 652 |
+
text_outputs = [text_output.lstrip('!') for text_output in text_outputs]
|
| 653 |
+
except Exception as e:
|
| 654 |
+
raise e
|
| 655 |
+
# with open('/data1/jacklishufan/lmms-eval.pt', 'wb') as f:
|
| 656 |
+
# torch.save(self.model.state_dict(), f)
|
| 657 |
+
# print('saved')
|
| 658 |
+
# print(1/0)
|
| 659 |
+
t1 = time.time()
|
| 660 |
+
delta_t += t1-t0
|
| 661 |
+
num_generated +=1
|
| 662 |
+
print(f"Avg Latency (of {num_generated}): {delta_t/num_generated}")
|
| 663 |
+
if DEBUG_PRINT_OUTPUT:
|
| 664 |
+
print(f'\n--------Start of Sample {batched_doc_id[0]}---------')
|
| 665 |
+
print("Question: ",prompt_question)
|
| 666 |
+
print("Answer: ",text_outputs)
|
| 667 |
+
print("Answer: ",gen_kwargs)
|
| 668 |
+
print('--------End---------')
|
| 669 |
+
|
| 670 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 671 |
+
res.extend(text_outputs)
|
| 672 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 673 |
+
pbar.update(1)
|
| 674 |
+
# reorder this group of results back to original unsorted form
|
| 675 |
+
res = re_ords.get_original(res)
|
| 676 |
+
|
| 677 |
+
pbar.close()
|
| 678 |
+
return res
|
| 679 |
+
|
| 680 |
+
def generate_until_multi_round(self, requests: List[Instance]) -> List[str]:
|
| 681 |
+
res = []
|
| 682 |
+
raise NotImplementedError()
|
| 683 |
+
|
| 684 |
+
def _collate(x):
|
| 685 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 686 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 687 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 688 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 689 |
+
# automatic adaptive batches much much easier to implement
|
| 690 |
+
# - any OOMs will happen right away rather than near the end
|
| 691 |
+
toks = self.tok_encode(x[0])
|
| 692 |
+
return -len(toks), x[0]
|
| 693 |
+
|
| 694 |
+
# we group requests by their generation_kwargs,
|
| 695 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 696 |
+
# in the same batch.
|
| 697 |
+
metadata = requests[0].metadata
|
| 698 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 699 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 700 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 701 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 702 |
+
|
| 703 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 704 |
+
|
| 705 |
+
for chunk in chunks:
|
| 706 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_to_text, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 707 |
+
task = batched_task[0]
|
| 708 |
+
split = batched_split[0]
|
| 709 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 710 |
+
assert len(batched_visuals) == 1
|
| 711 |
+
|
| 712 |
+
# we assume all gen kwargs in the batch are the same
|
| 713 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 714 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 715 |
+
if "until" in gen_kwargs:
|
| 716 |
+
gen_kwargs.pop("until")
|
| 717 |
+
|
| 718 |
+
# multi round inference: terminate when receiving signal from the doc_to_text
|
| 719 |
+
round_idx = 0
|
| 720 |
+
batched_round_res = []
|
| 721 |
+
batched_previous_round_info = None
|
| 722 |
+
while True:
|
| 723 |
+
question_input = []
|
| 724 |
+
|
| 725 |
+
if round_idx != 0: # get current round visual and context from doc_to_text function
|
| 726 |
+
batched_visuals, batched_contexts, batched_terminal_singal, batched_round_res, batched_previous_round_info = list(
|
| 727 |
+
zip(
|
| 728 |
+
*[
|
| 729 |
+
batched_doc_to_text[0](
|
| 730 |
+
self.task_dict[task][split][ids],
|
| 731 |
+
previous_output=[round_res[ids_idx] for round_res in batched_round_res],
|
| 732 |
+
round_idx=round_idx,
|
| 733 |
+
previous_round_info=batched_previous_round_info[ids_idx] if batched_previous_round_info is not None else None,
|
| 734 |
+
)
|
| 735 |
+
for ids_idx, ids in enumerate(batched_doc_id)
|
| 736 |
+
]
|
| 737 |
+
)
|
| 738 |
+
)
|
| 739 |
+
# import ipdb; ipdb.set_trace()
|
| 740 |
+
batched_round_res = list(zip(*batched_round_res)) # [(r1_1, r1_2), (r2_1, r2_2), ...]
|
| 741 |
+
if batched_terminal_singal[0]: # terminal signal from doc_to_text function
|
| 742 |
+
break
|
| 743 |
+
|
| 744 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 745 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 746 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 747 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 748 |
+
|
| 749 |
+
if visual is None or visual == []: # for text-only tasks.
|
| 750 |
+
visual = None
|
| 751 |
+
task_type = "text"
|
| 752 |
+
placeholder_count = 0
|
| 753 |
+
image_tensor = None
|
| 754 |
+
else:
|
| 755 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 756 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 757 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 758 |
+
|
| 759 |
+
if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
|
| 760 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 761 |
+
sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
|
| 762 |
+
visual = [visual[i] for i in sample_indices]
|
| 763 |
+
assert len(visual) == metadata["sample_frames"]
|
| 764 |
+
|
| 765 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 766 |
+
if type(image_tensor) is list:
|
| 767 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 768 |
+
else:
|
| 769 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 770 |
+
|
| 771 |
+
task_type = "video"
|
| 772 |
+
placeholder_count = 1
|
| 773 |
+
|
| 774 |
+
elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
|
| 775 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 776 |
+
if type(image_tensor) is list:
|
| 777 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 778 |
+
else:
|
| 779 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 780 |
+
|
| 781 |
+
task_type = "image"
|
| 782 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 783 |
+
|
| 784 |
+
elif type(visual[0]) == str: # For video task
|
| 785 |
+
image_tensor = []
|
| 786 |
+
try:
|
| 787 |
+
if self.video_decode_backend == "decord":
|
| 788 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 789 |
+
elif self.video_decode_backend == "pyav":
|
| 790 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 791 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 792 |
+
image_tensor.append(frames)
|
| 793 |
+
except Exception as e:
|
| 794 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 795 |
+
image_tensor = None
|
| 796 |
+
|
| 797 |
+
task_type = "video"
|
| 798 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 799 |
+
|
| 800 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 801 |
+
"""
|
| 802 |
+
Three senarios:
|
| 803 |
+
1. No image, and there for, no image token should be added.
|
| 804 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 805 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 806 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 807 |
+
"""
|
| 808 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 809 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 810 |
+
# elif task_type == "video":
|
| 811 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 812 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 813 |
+
image_tokens = " ".join(image_tokens)
|
| 814 |
+
question = image_tokens + "\n" + context
|
| 815 |
+
else:
|
| 816 |
+
question = context
|
| 817 |
+
|
| 818 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 819 |
+
if "llama_3" in self.conv_template:
|
| 820 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 821 |
+
else:
|
| 822 |
+
conv = conv_templates[self.conv_template].copy()
|
| 823 |
+
|
| 824 |
+
if utils.is_json(question): # conversational question input
|
| 825 |
+
question = json.loads(question)
|
| 826 |
+
for idx, item in enumerate(question):
|
| 827 |
+
role = conv.roles[idx % 2]
|
| 828 |
+
message = item["value"]
|
| 829 |
+
conv.append_message(role, message)
|
| 830 |
+
|
| 831 |
+
assert len(conv.messages) % 2 == 1
|
| 832 |
+
conv.append_message(conv.roles[1], None)
|
| 833 |
+
prompt_question = conv.get_prompt()
|
| 834 |
+
question_input.append(prompt_question)
|
| 835 |
+
else: # only simple string for question
|
| 836 |
+
conv.append_message(conv.roles[0], question)
|
| 837 |
+
conv.append_message(conv.roles[1], None)
|
| 838 |
+
prompt_question = conv.get_prompt()
|
| 839 |
+
question_input.append(prompt_question)
|
| 840 |
+
|
| 841 |
+
# preconfigure gen_kwargs with defaults
|
| 842 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 843 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 844 |
+
if "temperature" not in gen_kwargs:
|
| 845 |
+
gen_kwargs["temperature"] = 0
|
| 846 |
+
if "do_sample" not in gen_kwargs:
|
| 847 |
+
gen_kwargs["do_sample"] = False
|
| 848 |
+
if "top_p" not in gen_kwargs:
|
| 849 |
+
gen_kwargs["top_p"] = None
|
| 850 |
+
if "num_beams" not in gen_kwargs:
|
| 851 |
+
gen_kwargs["num_beams"] = 1
|
| 852 |
+
|
| 853 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 854 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 855 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 856 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 857 |
+
|
| 858 |
+
if task_type == "image":
|
| 859 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 860 |
+
elif task_type == "video":
|
| 861 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 862 |
+
keywords = [stop_str]
|
| 863 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 864 |
+
gen_kwargs["modalities"] = ["video"]
|
| 865 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 866 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 867 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 868 |
+
|
| 869 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 870 |
+
# TODO: attention to this major generation step...
|
| 871 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 872 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 873 |
+
try:
|
| 874 |
+
with torch.inference_mode():
|
| 875 |
+
cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 876 |
+
# cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 877 |
+
|
| 878 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 879 |
+
except Exception as e:
|
| 880 |
+
raise e
|
| 881 |
+
|
| 882 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 883 |
+
batched_round_res.append(text_outputs)
|
| 884 |
+
|
| 885 |
+
round_idx += 1
|
| 886 |
+
|
| 887 |
+
res.extend(list(zip(*batched_round_res)))
|
| 888 |
+
self.cache_hook.add_partial("generate_until_multi_round", (context, gen_kwargs), batched_round_res)
|
| 889 |
+
pbar.update(1)
|
| 890 |
+
# reorder this group of results back to original unsorted form
|
| 891 |
+
res = re_ords.get_original(res)
|
| 892 |
+
|
| 893 |
+
pbar.close()
|
| 894 |
+
return res
|
LaViDa/eval/lmms_eval/models/llava_onevision.py
ADDED
|
@@ -0,0 +1,785 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import re
|
| 6 |
+
import warnings
|
| 7 |
+
from datetime import timedelta
|
| 8 |
+
from typing import List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import PIL
|
| 12 |
+
import torch
|
| 13 |
+
import transformers
|
| 14 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 15 |
+
from accelerate.state import AcceleratorState
|
| 16 |
+
from decord import VideoReader, cpu
|
| 17 |
+
from packaging import version
|
| 18 |
+
from tqdm import tqdm
|
| 19 |
+
from transformers import AutoConfig
|
| 20 |
+
|
| 21 |
+
from lmms_eval import utils
|
| 22 |
+
from lmms_eval.api.instance import Instance
|
| 23 |
+
from lmms_eval.api.model import lmms
|
| 24 |
+
from lmms_eval.api.registry import register_model
|
| 25 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 26 |
+
|
| 27 |
+
# Suppress warnings
|
| 28 |
+
warnings.filterwarnings("ignore")
|
| 29 |
+
|
| 30 |
+
# Configure logging
|
| 31 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 32 |
+
|
| 33 |
+
# Enable TF32 for CUDA
|
| 34 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 35 |
+
|
| 36 |
+
# Import LLaVA modules
|
| 37 |
+
try:
|
| 38 |
+
from llava.constants import (
|
| 39 |
+
DEFAULT_IM_END_TOKEN,
|
| 40 |
+
DEFAULT_IM_START_TOKEN,
|
| 41 |
+
DEFAULT_IMAGE_TOKEN,
|
| 42 |
+
IGNORE_INDEX,
|
| 43 |
+
IMAGE_TOKEN_INDEX,
|
| 44 |
+
)
|
| 45 |
+
from llava.conversation import SeparatorStyle, conv_templates
|
| 46 |
+
from llava.mm_utils import (
|
| 47 |
+
KeywordsStoppingCriteria,
|
| 48 |
+
get_model_name_from_path,
|
| 49 |
+
process_images,
|
| 50 |
+
tokenizer_image_token,
|
| 51 |
+
)
|
| 52 |
+
from llava.model.builder import load_pretrained_model
|
| 53 |
+
except ImportError as e:
|
| 54 |
+
eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# Determine best attention implementation
|
| 58 |
+
if version.parse(torch.__version__) >= version.parse("2.1.2"):
|
| 59 |
+
best_fit_attn_implementation = "sdpa"
|
| 60 |
+
else:
|
| 61 |
+
best_fit_attn_implementation = "eager"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@register_model("llava_onevision")
|
| 65 |
+
class Llava_OneVision(lmms):
|
| 66 |
+
"""
|
| 67 |
+
Llava Model
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
def __init__(
|
| 71 |
+
self,
|
| 72 |
+
pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
|
| 73 |
+
truncation: Optional[bool] = True,
|
| 74 |
+
device: Optional[str] = "cuda:0",
|
| 75 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 76 |
+
model_name: Optional[str] = None,
|
| 77 |
+
attn_implementation: Optional[str] = best_fit_attn_implementation,
|
| 78 |
+
device_map: Optional[str] = "cuda:0",
|
| 79 |
+
conv_template: Optional[str] = "qwen_1_5",
|
| 80 |
+
use_cache: Optional[bool] = True,
|
| 81 |
+
truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 82 |
+
customized_config: Optional[str] = None, # ends in json
|
| 83 |
+
max_frames_num: Optional[int] = 32,
|
| 84 |
+
mm_spatial_pool_stride: Optional[int] = 2,
|
| 85 |
+
mm_spatial_pool_mode: Optional[str] = "bilinear",
|
| 86 |
+
token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
|
| 87 |
+
video_decode_backend: str = "decord",
|
| 88 |
+
**kwargs,
|
| 89 |
+
) -> None:
|
| 90 |
+
super().__init__()
|
| 91 |
+
# Do not use kwargs for now
|
| 92 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 93 |
+
|
| 94 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 95 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 96 |
+
if accelerator.num_processes > 1:
|
| 97 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 98 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 99 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 100 |
+
self._device = torch.device(device)
|
| 101 |
+
self.device_map = device_map
|
| 102 |
+
else:
|
| 103 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 104 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 105 |
+
|
| 106 |
+
llava_model_args = {
|
| 107 |
+
"multimodal": True,
|
| 108 |
+
}
|
| 109 |
+
if customized_config is not None:
|
| 110 |
+
llava_model_args["customized_config"] = customized_config
|
| 111 |
+
if attn_implementation is not None:
|
| 112 |
+
llava_model_args["attn_implementation"] = attn_implementation
|
| 113 |
+
if "use_flash_attention_2" in kwargs:
|
| 114 |
+
llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
|
| 115 |
+
model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
|
| 116 |
+
|
| 117 |
+
self.pretrained = pretrained
|
| 118 |
+
self.token_strategy = token_strategy
|
| 119 |
+
self.max_frames_num = max_frames_num
|
| 120 |
+
self.mm_spatial_pool_stride = mm_spatial_pool_stride
|
| 121 |
+
self.mm_spatial_pool_mode = mm_spatial_pool_mode
|
| 122 |
+
self.video_decode_backend = video_decode_backend
|
| 123 |
+
|
| 124 |
+
overwrite_config = {}
|
| 125 |
+
overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
|
| 126 |
+
overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
|
| 127 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 128 |
+
|
| 129 |
+
llava_model_args["overwrite_config"] = overwrite_config
|
| 130 |
+
try:
|
| 131 |
+
# Try to load the model with the multimodal argument
|
| 132 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 133 |
+
except TypeError:
|
| 134 |
+
# for older versions of LLaVA that don't have multimodal argument
|
| 135 |
+
llava_model_args.pop("multimodal", None)
|
| 136 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 137 |
+
|
| 138 |
+
self._config = self._model.config
|
| 139 |
+
self.model.eval()
|
| 140 |
+
self.truncation = truncation
|
| 141 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 142 |
+
self.conv_template = conv_template
|
| 143 |
+
self.use_cache = use_cache
|
| 144 |
+
self.truncate_context = truncate_context
|
| 145 |
+
assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 146 |
+
|
| 147 |
+
if accelerator.num_processes > 1:
|
| 148 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 149 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 150 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 151 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 152 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 153 |
+
kwargs = {
|
| 154 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 155 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 156 |
+
}
|
| 157 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 158 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 159 |
+
|
| 160 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 161 |
+
self._model = accelerator.prepare(self.model)
|
| 162 |
+
else:
|
| 163 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 164 |
+
self.accelerator = accelerator
|
| 165 |
+
if self.accelerator.is_local_main_process:
|
| 166 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 167 |
+
self._rank = self.accelerator.local_process_index
|
| 168 |
+
self._world_size = self.accelerator.num_processes
|
| 169 |
+
|
| 170 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 171 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 172 |
+
self._rank = 0
|
| 173 |
+
self._world_size = 1
|
| 174 |
+
|
| 175 |
+
else:
|
| 176 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 177 |
+
self.model.to(self._device)
|
| 178 |
+
self._rank = 0
|
| 179 |
+
self._world_size = 1
|
| 180 |
+
|
| 181 |
+
@property
|
| 182 |
+
def config(self):
|
| 183 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 184 |
+
return self._config
|
| 185 |
+
|
| 186 |
+
@property
|
| 187 |
+
def tokenizer(self):
|
| 188 |
+
return self._tokenizer
|
| 189 |
+
|
| 190 |
+
@property
|
| 191 |
+
def model(self):
|
| 192 |
+
# returns the model, unwrapping it if using Accelerate
|
| 193 |
+
if hasattr(self, "accelerator"):
|
| 194 |
+
return self.accelerator.unwrap_model(self._model)
|
| 195 |
+
else:
|
| 196 |
+
return self._model
|
| 197 |
+
|
| 198 |
+
@property
|
| 199 |
+
def eot_token_id(self):
|
| 200 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 201 |
+
return self.tokenizer.eos_token_id
|
| 202 |
+
|
| 203 |
+
@property
|
| 204 |
+
def max_length(self):
|
| 205 |
+
return self._max_length
|
| 206 |
+
|
| 207 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 208 |
+
if self.tokenizer.padding_side == "left":
|
| 209 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 210 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 211 |
+
if self.tokenizer.padding_side == "left":
|
| 212 |
+
input_ids = torch.flip(input_ids, [1])
|
| 213 |
+
return input_ids
|
| 214 |
+
|
| 215 |
+
@property
|
| 216 |
+
def batch_size(self):
|
| 217 |
+
return self.batch_size_per_gpu
|
| 218 |
+
|
| 219 |
+
@property
|
| 220 |
+
def device(self):
|
| 221 |
+
return self._device
|
| 222 |
+
|
| 223 |
+
@property
|
| 224 |
+
def rank(self):
|
| 225 |
+
return self._rank
|
| 226 |
+
|
| 227 |
+
@property
|
| 228 |
+
def world_size(self):
|
| 229 |
+
return self._world_size
|
| 230 |
+
|
| 231 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 232 |
+
""" """
|
| 233 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 234 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 235 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 236 |
+
if left_truncate_len:
|
| 237 |
+
encoding = encoding[-left_truncate_len:]
|
| 238 |
+
return encoding
|
| 239 |
+
|
| 240 |
+
def tok_decode(self, tokens):
|
| 241 |
+
try:
|
| 242 |
+
return self.tokenizer.decode(tokens)
|
| 243 |
+
except:
|
| 244 |
+
return self.tokenizer.decode([tokens])
|
| 245 |
+
|
| 246 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 247 |
+
res = []
|
| 248 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 249 |
+
|
| 250 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 251 |
+
|
| 252 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 253 |
+
visual = doc_to_visual(self.task_dict[task][split][doc_id])
|
| 254 |
+
|
| 255 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 256 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 257 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 258 |
+
|
| 259 |
+
if visual is None or visual == []:
|
| 260 |
+
visual = None
|
| 261 |
+
task_type = "text"
|
| 262 |
+
image_tensor = None
|
| 263 |
+
else:
|
| 264 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__:
|
| 265 |
+
self._config.image_aspect_ratio = "pad"
|
| 266 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 267 |
+
|
| 268 |
+
if "task_type" in self.metadata and self.metadata["task_type"] == "video" and "sample_frames" in self.metadata:
|
| 269 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 270 |
+
sample_indices = np.linspace(0, len(visual) - 1, self.metadata["sample_frames"], dtype=int)
|
| 271 |
+
visual = [visual[i] for i in sample_indices]
|
| 272 |
+
assert len(visual) == self.metadata["sample_frames"]
|
| 273 |
+
|
| 274 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 275 |
+
if type(image_tensor) is list:
|
| 276 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 277 |
+
else:
|
| 278 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 279 |
+
|
| 280 |
+
task_type = "video"
|
| 281 |
+
|
| 282 |
+
# elif type(visual[0]) == PIL.Image.Image:
|
| 283 |
+
elif isinstance(visual[0], PIL.Image.Image):
|
| 284 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 285 |
+
if type(image_tensor) is list:
|
| 286 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 287 |
+
else:
|
| 288 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 289 |
+
|
| 290 |
+
task_type = "image"
|
| 291 |
+
|
| 292 |
+
elif type(visual[0]) == str:
|
| 293 |
+
image_tensor = []
|
| 294 |
+
try:
|
| 295 |
+
if self.video_decode_backend == "decord":
|
| 296 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 297 |
+
elif self.video_decode_backend == "pyav":
|
| 298 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 299 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 300 |
+
image_tensor.append(frames)
|
| 301 |
+
except Exception as e:
|
| 302 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 303 |
+
image_tensor = None
|
| 304 |
+
|
| 305 |
+
task_type = "video"
|
| 306 |
+
|
| 307 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in contexts:
|
| 308 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 309 |
+
if task_type == "video":
|
| 310 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 311 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 312 |
+
image_tokens = " ".join(image_tokens)
|
| 313 |
+
prompts_input = image_tokens + "\n" + contexts
|
| 314 |
+
else:
|
| 315 |
+
prompts_input = contexts
|
| 316 |
+
|
| 317 |
+
if "llama_3" in self.conv_template:
|
| 318 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 319 |
+
else:
|
| 320 |
+
conv = conv_templates[self.conv_template].copy()
|
| 321 |
+
|
| 322 |
+
conv.append_message(conv.roles[0], prompts_input)
|
| 323 |
+
conv.append_message(conv.roles[1], None)
|
| 324 |
+
prompt = conv.get_prompt()
|
| 325 |
+
|
| 326 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 327 |
+
|
| 328 |
+
if type(doc_to_target) == str:
|
| 329 |
+
continuation = doc_to_target
|
| 330 |
+
else:
|
| 331 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 332 |
+
|
| 333 |
+
conv.messages[-1][1] = continuation
|
| 334 |
+
full_prompt = conv.get_prompt()
|
| 335 |
+
full_input_ids = tokenizer_image_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 336 |
+
|
| 337 |
+
labels = full_input_ids.clone()
|
| 338 |
+
labels[0, : input_ids.shape[1]] = -100
|
| 339 |
+
|
| 340 |
+
kwargs = {}
|
| 341 |
+
if task_type == "image":
|
| 342 |
+
kwargs["image_sizes"] = [[v.size[0], v.size[1]] for v in visual] if isinstance(visual, list) else [[visual.size[0], visual.size[1]]]
|
| 343 |
+
elif task_type == "video":
|
| 344 |
+
kwargs["modalities"] = ["video"]
|
| 345 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 346 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 347 |
+
|
| 348 |
+
with torch.inference_mode():
|
| 349 |
+
outputs = self.model(input_ids=full_input_ids, labels=labels, images=image_tensor, use_cache=True, **kwargs)
|
| 350 |
+
|
| 351 |
+
loss = outputs["loss"]
|
| 352 |
+
logits = outputs["logits"]
|
| 353 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 354 |
+
cont_toks = full_input_ids[:, input_ids.shape[1] :]
|
| 355 |
+
greedy_tokens = greedy_tokens[:, input_ids.shape[1] : full_input_ids.shape[1]]
|
| 356 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 357 |
+
|
| 358 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 359 |
+
pbar.update(1)
|
| 360 |
+
|
| 361 |
+
pbar.close()
|
| 362 |
+
return res
|
| 363 |
+
|
| 364 |
+
def flatten(self, input):
|
| 365 |
+
if not input or any(i is None for i in input):
|
| 366 |
+
return []
|
| 367 |
+
new_list = []
|
| 368 |
+
for i in input:
|
| 369 |
+
if i:
|
| 370 |
+
for j in i:
|
| 371 |
+
new_list.append(j)
|
| 372 |
+
return new_list
|
| 373 |
+
|
| 374 |
+
def load_video(self, video_path, max_frames_num):
|
| 375 |
+
if type(video_path) == str:
|
| 376 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 377 |
+
else:
|
| 378 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 379 |
+
total_frame_num = len(vr)
|
| 380 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 381 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 382 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 383 |
+
return spare_frames # (frames, height, width, channels)
|
| 384 |
+
|
| 385 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 386 |
+
res = []
|
| 387 |
+
|
| 388 |
+
def _collate(x):
|
| 389 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 390 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 391 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 392 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 393 |
+
# automatic adaptive batches much much easier to implement
|
| 394 |
+
# - any OOMs will happen right away rather than near the end
|
| 395 |
+
toks = self.tok_encode(x[0])
|
| 396 |
+
return -len(toks), x[0]
|
| 397 |
+
|
| 398 |
+
# we group requests by their generation_kwargs,
|
| 399 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 400 |
+
# in the same batch.
|
| 401 |
+
metadata = requests[0].metadata
|
| 402 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 403 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 404 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 405 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 406 |
+
|
| 407 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 408 |
+
|
| 409 |
+
for chunk in chunks:
|
| 410 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 411 |
+
task = batched_task[0]
|
| 412 |
+
split = batched_split[0]
|
| 413 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 414 |
+
assert len(batched_visuals) == 1
|
| 415 |
+
|
| 416 |
+
# we assume all gen kwargs in the batch are the same
|
| 417 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 418 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 419 |
+
if "until" in gen_kwargs:
|
| 420 |
+
gen_kwargs.pop("until")
|
| 421 |
+
|
| 422 |
+
question_input = []
|
| 423 |
+
# import ipdb; ipdb.set_trace()
|
| 424 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 425 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 426 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 427 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 428 |
+
|
| 429 |
+
if visual is None or visual == []: # for text-only tasks.
|
| 430 |
+
visual = None
|
| 431 |
+
task_type = "text"
|
| 432 |
+
placeholder_count = 0
|
| 433 |
+
image_tensor = None
|
| 434 |
+
else:
|
| 435 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 436 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 437 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 438 |
+
|
| 439 |
+
if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
|
| 440 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 441 |
+
sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
|
| 442 |
+
visual = [visual[i] for i in sample_indices]
|
| 443 |
+
assert len(visual) == metadata["sample_frames"]
|
| 444 |
+
|
| 445 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 446 |
+
if type(image_tensor) is list:
|
| 447 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 448 |
+
else:
|
| 449 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 450 |
+
|
| 451 |
+
task_type = "video"
|
| 452 |
+
placeholder_count = 1
|
| 453 |
+
|
| 454 |
+
elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
|
| 455 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 456 |
+
if type(image_tensor) is list:
|
| 457 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 458 |
+
else:
|
| 459 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 460 |
+
|
| 461 |
+
task_type = "image"
|
| 462 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 463 |
+
|
| 464 |
+
elif type(visual[0]) == str: # For video task
|
| 465 |
+
image_tensor = []
|
| 466 |
+
try:
|
| 467 |
+
if self.video_decode_backend == "decord":
|
| 468 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 469 |
+
elif self.video_decode_backend == "pyav":
|
| 470 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 471 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 472 |
+
image_tensor.append(frames)
|
| 473 |
+
except Exception as e:
|
| 474 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 475 |
+
image_tensor = None
|
| 476 |
+
|
| 477 |
+
task_type = "video"
|
| 478 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 479 |
+
|
| 480 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 481 |
+
"""
|
| 482 |
+
Three senarios:
|
| 483 |
+
1. No image, and there for, no image token should be added.
|
| 484 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 485 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 486 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 487 |
+
"""
|
| 488 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 489 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 490 |
+
# elif task_type == "video":
|
| 491 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 492 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 493 |
+
image_tokens = " ".join(image_tokens)
|
| 494 |
+
question = image_tokens + "\n" + context
|
| 495 |
+
else:
|
| 496 |
+
question = context
|
| 497 |
+
|
| 498 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 499 |
+
if "llama_3" in self.conv_template:
|
| 500 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 501 |
+
else:
|
| 502 |
+
conv = conv_templates[self.conv_template].copy()
|
| 503 |
+
|
| 504 |
+
if utils.is_json(question): # conversational question input
|
| 505 |
+
question = json.loads(question)
|
| 506 |
+
for idx, item in enumerate(question):
|
| 507 |
+
role = conv.roles[idx % 2]
|
| 508 |
+
message = item["value"]
|
| 509 |
+
conv.append_message(role, message)
|
| 510 |
+
|
| 511 |
+
assert len(conv.messages) % 2 == 1
|
| 512 |
+
conv.append_message(conv.roles[1], None)
|
| 513 |
+
prompt_question = conv.get_prompt()
|
| 514 |
+
question_input.append(prompt_question)
|
| 515 |
+
else: # only simple string for question
|
| 516 |
+
conv.append_message(conv.roles[0], question)
|
| 517 |
+
conv.append_message(conv.roles[1], None)
|
| 518 |
+
prompt_question = conv.get_prompt()
|
| 519 |
+
question_input.append(prompt_question)
|
| 520 |
+
|
| 521 |
+
# preconfigure gen_kwargs with defaults
|
| 522 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 523 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 524 |
+
if "temperature" not in gen_kwargs:
|
| 525 |
+
gen_kwargs["temperature"] = 0
|
| 526 |
+
if "do_sample" not in gen_kwargs:
|
| 527 |
+
gen_kwargs["do_sample"] = False
|
| 528 |
+
if "top_p" not in gen_kwargs:
|
| 529 |
+
gen_kwargs["top_p"] = None
|
| 530 |
+
if "num_beams" not in gen_kwargs:
|
| 531 |
+
gen_kwargs["num_beams"] = 1
|
| 532 |
+
|
| 533 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 534 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 535 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 536 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 537 |
+
|
| 538 |
+
if task_type == "image":
|
| 539 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 540 |
+
elif task_type == "video":
|
| 541 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 542 |
+
keywords = [stop_str]
|
| 543 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 544 |
+
gen_kwargs["modalities"] = ["video"]
|
| 545 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 546 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 547 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 548 |
+
|
| 549 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 550 |
+
# TODO: attention to this major generation step...
|
| 551 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 552 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 553 |
+
try:
|
| 554 |
+
with torch.inference_mode():
|
| 555 |
+
cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 556 |
+
# cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 557 |
+
|
| 558 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 559 |
+
except Exception as e:
|
| 560 |
+
raise e
|
| 561 |
+
|
| 562 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 563 |
+
res.extend(text_outputs)
|
| 564 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 565 |
+
pbar.update(1)
|
| 566 |
+
# reorder this group of results back to original unsorted form
|
| 567 |
+
res = re_ords.get_original(res)
|
| 568 |
+
|
| 569 |
+
pbar.close()
|
| 570 |
+
return res
|
| 571 |
+
|
| 572 |
+
def generate_until_multi_round(self, requests: List[Instance]) -> List[str]:
|
| 573 |
+
res = []
|
| 574 |
+
|
| 575 |
+
def _collate(x):
|
| 576 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 577 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 578 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 579 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 580 |
+
# automatic adaptive batches much much easier to implement
|
| 581 |
+
# - any OOMs will happen right away rather than near the end
|
| 582 |
+
toks = self.tok_encode(x[0])
|
| 583 |
+
return -len(toks), x[0]
|
| 584 |
+
|
| 585 |
+
# we group requests by their generation_kwargs,
|
| 586 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 587 |
+
# in the same batch.
|
| 588 |
+
metadata = requests[0].metadata
|
| 589 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 590 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 591 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 592 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 593 |
+
|
| 594 |
+
origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
|
| 595 |
+
|
| 596 |
+
for chunk in chunks:
|
| 597 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_to_text, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 598 |
+
task = batched_task[0]
|
| 599 |
+
split = batched_split[0]
|
| 600 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 601 |
+
assert len(batched_visuals) == 1
|
| 602 |
+
|
| 603 |
+
# we assume all gen kwargs in the batch are the same
|
| 604 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 605 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 606 |
+
if "until" in gen_kwargs:
|
| 607 |
+
gen_kwargs.pop("until")
|
| 608 |
+
|
| 609 |
+
# multi round inference: terminate when receiving signal from the doc_to_text
|
| 610 |
+
round_idx = 0
|
| 611 |
+
batched_round_res = []
|
| 612 |
+
batched_previous_round_info = None
|
| 613 |
+
while True:
|
| 614 |
+
question_input = []
|
| 615 |
+
|
| 616 |
+
if round_idx != 0: # get current round visual and context from doc_to_text function
|
| 617 |
+
batched_visuals, batched_contexts, batched_terminal_singal, batched_round_res, batched_previous_round_info = list(
|
| 618 |
+
zip(
|
| 619 |
+
*[
|
| 620 |
+
batched_doc_to_text[0](
|
| 621 |
+
self.task_dict[task][split][ids],
|
| 622 |
+
previous_output=[round_res[ids_idx] for round_res in batched_round_res],
|
| 623 |
+
round_idx=round_idx,
|
| 624 |
+
previous_round_info=batched_previous_round_info[ids_idx] if batched_previous_round_info is not None else None,
|
| 625 |
+
)
|
| 626 |
+
for ids_idx, ids in enumerate(batched_doc_id)
|
| 627 |
+
]
|
| 628 |
+
)
|
| 629 |
+
)
|
| 630 |
+
# import ipdb; ipdb.set_trace()
|
| 631 |
+
batched_round_res = list(zip(*batched_round_res)) # [(r1_1, r1_2), (r2_1, r2_2), ...]
|
| 632 |
+
if batched_terminal_singal[0]: # terminal signal from doc_to_text function
|
| 633 |
+
break
|
| 634 |
+
|
| 635 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 636 |
+
if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
|
| 637 |
+
self._config.image_aspect_ratio = origin_image_aspect_ratio
|
| 638 |
+
eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
|
| 639 |
+
|
| 640 |
+
if visual is None or visual == []: # for text-only tasks.
|
| 641 |
+
visual = None
|
| 642 |
+
task_type = "text"
|
| 643 |
+
placeholder_count = 0
|
| 644 |
+
image_tensor = None
|
| 645 |
+
else:
|
| 646 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 647 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 648 |
+
eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
|
| 649 |
+
|
| 650 |
+
if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
|
| 651 |
+
assert type(visual) == list, "sample_frames must be specified for video task"
|
| 652 |
+
sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
|
| 653 |
+
visual = [visual[i] for i in sample_indices]
|
| 654 |
+
assert len(visual) == metadata["sample_frames"]
|
| 655 |
+
|
| 656 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 657 |
+
if type(image_tensor) is list:
|
| 658 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 659 |
+
else:
|
| 660 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 661 |
+
|
| 662 |
+
task_type = "video"
|
| 663 |
+
placeholder_count = 1
|
| 664 |
+
|
| 665 |
+
elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
|
| 666 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 667 |
+
if type(image_tensor) is list:
|
| 668 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 669 |
+
else:
|
| 670 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 671 |
+
|
| 672 |
+
task_type = "image"
|
| 673 |
+
placeholder_count = len(visual) if isinstance(visual, list) else 1
|
| 674 |
+
|
| 675 |
+
elif type(visual[0]) == str: # For video task
|
| 676 |
+
image_tensor = []
|
| 677 |
+
try:
|
| 678 |
+
if self.video_decode_backend == "decord":
|
| 679 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 680 |
+
elif self.video_decode_backend == "pyav":
|
| 681 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 682 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 683 |
+
image_tensor.append(frames)
|
| 684 |
+
except Exception as e:
|
| 685 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 686 |
+
image_tensor = None
|
| 687 |
+
|
| 688 |
+
task_type = "video"
|
| 689 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 690 |
+
|
| 691 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 692 |
+
"""
|
| 693 |
+
Three senarios:
|
| 694 |
+
1. No image, and there for, no image token should be added.
|
| 695 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 696 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 697 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 698 |
+
"""
|
| 699 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 700 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 701 |
+
# elif task_type == "video":
|
| 702 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 703 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 704 |
+
image_tokens = " ".join(image_tokens)
|
| 705 |
+
question = image_tokens + "\n" + context
|
| 706 |
+
else:
|
| 707 |
+
question = context
|
| 708 |
+
|
| 709 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 710 |
+
if "llama_3" in self.conv_template:
|
| 711 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 712 |
+
else:
|
| 713 |
+
conv = conv_templates[self.conv_template].copy()
|
| 714 |
+
|
| 715 |
+
if utils.is_json(question): # conversational question input
|
| 716 |
+
question = json.loads(question)
|
| 717 |
+
for idx, item in enumerate(question):
|
| 718 |
+
role = conv.roles[idx % 2]
|
| 719 |
+
message = item["value"]
|
| 720 |
+
conv.append_message(role, message)
|
| 721 |
+
|
| 722 |
+
assert len(conv.messages) % 2 == 1
|
| 723 |
+
conv.append_message(conv.roles[1], None)
|
| 724 |
+
prompt_question = conv.get_prompt()
|
| 725 |
+
question_input.append(prompt_question)
|
| 726 |
+
else: # only simple string for question
|
| 727 |
+
conv.append_message(conv.roles[0], question)
|
| 728 |
+
conv.append_message(conv.roles[1], None)
|
| 729 |
+
prompt_question = conv.get_prompt()
|
| 730 |
+
question_input.append(prompt_question)
|
| 731 |
+
|
| 732 |
+
# preconfigure gen_kwargs with defaults
|
| 733 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 734 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 735 |
+
if "temperature" not in gen_kwargs:
|
| 736 |
+
gen_kwargs["temperature"] = 0
|
| 737 |
+
if "do_sample" not in gen_kwargs:
|
| 738 |
+
gen_kwargs["do_sample"] = False
|
| 739 |
+
if "top_p" not in gen_kwargs:
|
| 740 |
+
gen_kwargs["top_p"] = None
|
| 741 |
+
if "num_beams" not in gen_kwargs:
|
| 742 |
+
gen_kwargs["num_beams"] = 1
|
| 743 |
+
|
| 744 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 745 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 746 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 747 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 748 |
+
|
| 749 |
+
if task_type == "image":
|
| 750 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 751 |
+
elif task_type == "video":
|
| 752 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 753 |
+
keywords = [stop_str]
|
| 754 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 755 |
+
gen_kwargs["modalities"] = ["video"]
|
| 756 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 757 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 758 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 759 |
+
|
| 760 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 761 |
+
# TODO: attention to this major generation step...
|
| 762 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 763 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 764 |
+
try:
|
| 765 |
+
with torch.inference_mode():
|
| 766 |
+
cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 767 |
+
# cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 768 |
+
|
| 769 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 770 |
+
except Exception as e:
|
| 771 |
+
raise e
|
| 772 |
+
|
| 773 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 774 |
+
batched_round_res.append(text_outputs)
|
| 775 |
+
|
| 776 |
+
round_idx += 1
|
| 777 |
+
|
| 778 |
+
res.extend(list(zip(*batched_round_res)))
|
| 779 |
+
self.cache_hook.add_partial("generate_until_multi_round", (context, gen_kwargs), batched_round_res)
|
| 780 |
+
pbar.update(1)
|
| 781 |
+
# reorder this group of results back to original unsorted form
|
| 782 |
+
res = re_ords.get_original(res)
|
| 783 |
+
|
| 784 |
+
pbar.close()
|
| 785 |
+
return res
|
LaViDa/eval/lmms_eval/models/llava_onevision_moviechat.py
ADDED
|
@@ -0,0 +1,526 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
import warnings
|
| 8 |
+
from datetime import timedelta
|
| 9 |
+
from typing import List, Optional, Tuple, Union
|
| 10 |
+
|
| 11 |
+
import av
|
| 12 |
+
import numpy as np
|
| 13 |
+
import PIL
|
| 14 |
+
import torch
|
| 15 |
+
import transformers
|
| 16 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 17 |
+
from accelerate.state import AcceleratorState
|
| 18 |
+
from decord import VideoReader, cpu
|
| 19 |
+
from moviepy.video.io.VideoFileClip import VideoFileClip
|
| 20 |
+
from packaging import version
|
| 21 |
+
from PIL import Image
|
| 22 |
+
from tqdm import tqdm
|
| 23 |
+
from transformers import AutoConfig
|
| 24 |
+
|
| 25 |
+
from lmms_eval import utils
|
| 26 |
+
from lmms_eval.api.instance import Instance
|
| 27 |
+
from lmms_eval.api.model import lmms
|
| 28 |
+
from lmms_eval.api.registry import register_model
|
| 29 |
+
|
| 30 |
+
# Suppress warnings
|
| 31 |
+
warnings.filterwarnings("ignore")
|
| 32 |
+
|
| 33 |
+
# Configure logging
|
| 34 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 35 |
+
|
| 36 |
+
# Enable TF32 for CUDA
|
| 37 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 38 |
+
|
| 39 |
+
# Import LLaVA modules
|
| 40 |
+
try:
|
| 41 |
+
from llava.constants import (
|
| 42 |
+
DEFAULT_IM_END_TOKEN,
|
| 43 |
+
DEFAULT_IM_START_TOKEN,
|
| 44 |
+
DEFAULT_IMAGE_TOKEN,
|
| 45 |
+
IGNORE_INDEX,
|
| 46 |
+
IMAGE_TOKEN_INDEX,
|
| 47 |
+
)
|
| 48 |
+
from llava.conversation import SeparatorStyle, conv_templates
|
| 49 |
+
from llava.mm_utils import (
|
| 50 |
+
KeywordsStoppingCriteria,
|
| 51 |
+
get_model_name_from_path,
|
| 52 |
+
process_images,
|
| 53 |
+
tokenizer_image_token,
|
| 54 |
+
)
|
| 55 |
+
from llava.model.builder import load_pretrained_model
|
| 56 |
+
except ImportError as e:
|
| 57 |
+
eval_logger.debug(f"LLaVA_NeXT is not installed. Please install llava from `https://github.com/rese1f/MovieChat.git` to use this model.\nError: {e}")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
# Determine best attention implementation
|
| 61 |
+
if version.parse(torch.__version__) >= version.parse("2.1.2"):
|
| 62 |
+
best_fit_attn_implementation = "sdpa"
|
| 63 |
+
else:
|
| 64 |
+
best_fit_attn_implementation = "eager"
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# llava_onevision_moviechat uses the same memory consolidation mechanism with the original MovieChat, but changes the base model from VideoLLamA to LLava-OneVision
|
| 68 |
+
@register_model("llava_onevision_moviechat")
|
| 69 |
+
class Llava_OneVision_MovieChat(lmms):
|
| 70 |
+
"""
|
| 71 |
+
Llava Model
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
def __init__(
|
| 75 |
+
self,
|
| 76 |
+
pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
|
| 77 |
+
truncation: Optional[bool] = True,
|
| 78 |
+
device: Optional[str] = "cuda:0",
|
| 79 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 80 |
+
model_name: str = "llava_qwen",
|
| 81 |
+
attn_implementation: Optional[str] = best_fit_attn_implementation,
|
| 82 |
+
device_map: Optional[str] = "cuda:0",
|
| 83 |
+
conv_template: Optional[str] = "qwen_1_5",
|
| 84 |
+
use_cache: Optional[bool] = True,
|
| 85 |
+
truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 86 |
+
customized_config: Optional[str] = None, # ends in json
|
| 87 |
+
short_memory_length: Optional[int] = 18,
|
| 88 |
+
long_memory_length: Optional[int] = 64,
|
| 89 |
+
sliding_window_length: Optional[int] = 8,
|
| 90 |
+
merge_frame_length: Optional[int] = 2,
|
| 91 |
+
tmp_folder: Optional[str] = "tmp/",
|
| 92 |
+
mm_spatial_pool_stride: Optional[int] = 2,
|
| 93 |
+
mm_spatial_pool_mode: Optional[str] = "bilinear",
|
| 94 |
+
token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
|
| 95 |
+
video_decode_backend: str = "decord",
|
| 96 |
+
**kwargs,
|
| 97 |
+
) -> None:
|
| 98 |
+
super().__init__()
|
| 99 |
+
# Do not use kwargs for now
|
| 100 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 101 |
+
|
| 102 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 103 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 104 |
+
if accelerator.num_processes > 1:
|
| 105 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 106 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 107 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 108 |
+
self._device = torch.device(device)
|
| 109 |
+
self.device_map = device_map
|
| 110 |
+
else:
|
| 111 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 112 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 113 |
+
|
| 114 |
+
llava_model_args = {
|
| 115 |
+
"multimodal": True,
|
| 116 |
+
}
|
| 117 |
+
if customized_config is not None:
|
| 118 |
+
llava_model_args["customized_config"] = customized_config
|
| 119 |
+
if attn_implementation is not None:
|
| 120 |
+
llava_model_args["attn_implementation"] = attn_implementation
|
| 121 |
+
if "use_flash_attention_2" in kwargs:
|
| 122 |
+
llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
|
| 123 |
+
model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
|
| 124 |
+
|
| 125 |
+
self.pretrained = pretrained
|
| 126 |
+
self.token_strategy = token_strategy
|
| 127 |
+
self.mm_spatial_pool_stride = mm_spatial_pool_stride
|
| 128 |
+
self.mm_spatial_pool_mode = mm_spatial_pool_mode
|
| 129 |
+
self.video_decode_backend = video_decode_backend
|
| 130 |
+
|
| 131 |
+
self.short_memory_length = short_memory_length
|
| 132 |
+
self.long_memory_length = long_memory_length
|
| 133 |
+
self.merge_frame_length = merge_frame_length
|
| 134 |
+
self.sliding_window_length = sliding_window_length
|
| 135 |
+
self.num_clips = (self.long_memory_length // self.merge_frame_length) * ((self.short_memory_length - self.merge_frame_length) // self.sliding_window_length)
|
| 136 |
+
self.tmp_folder = tmp_folder
|
| 137 |
+
|
| 138 |
+
overwrite_config = {}
|
| 139 |
+
overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
|
| 140 |
+
overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
|
| 141 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 142 |
+
|
| 143 |
+
if cfg_pretrained.architectures[0] == "LlavaLlamaForCausalLM": # Ugly code, only used in vicuna that needs ROPE
|
| 144 |
+
if "224" in cfg_pretrained.mm_vision_tower:
|
| 145 |
+
least_token_number = self.max_frames_num * (16 // self.mm_spatial_pool_stride) ** 2 + 1000
|
| 146 |
+
else:
|
| 147 |
+
least_token_number = self.max_frames_num * (24 // self.mm_spatial_pool_stride) ** 2 + 1000
|
| 148 |
+
|
| 149 |
+
scaling_factor = math.ceil(least_token_number / 4096)
|
| 150 |
+
if scaling_factor >= 2:
|
| 151 |
+
overwrite_config["rope_scaling"] = {"factor": float(scaling_factor), "type": "linear"}
|
| 152 |
+
overwrite_config["max_sequence_length"] = 4096 * scaling_factor
|
| 153 |
+
overwrite_config["tokenizer_model_max_length"] = 4096 * scaling_factor
|
| 154 |
+
|
| 155 |
+
llava_model_args["overwrite_config"] = overwrite_config
|
| 156 |
+
from LLaVA_NeXT.llava.model.builder import load_pretrained_model
|
| 157 |
+
|
| 158 |
+
try:
|
| 159 |
+
# Try to load the model with the multimodal argument
|
| 160 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 161 |
+
except TypeError:
|
| 162 |
+
# for older versions of LLaVA that don't have multimodal argument
|
| 163 |
+
llava_model_args.pop("multimodal", None)
|
| 164 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 165 |
+
|
| 166 |
+
self._config = self._model.config
|
| 167 |
+
self.model.eval()
|
| 168 |
+
self.truncation = truncation
|
| 169 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 170 |
+
self.conv_template = conv_template
|
| 171 |
+
self.use_cache = use_cache
|
| 172 |
+
self.truncate_context = truncate_context
|
| 173 |
+
assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 174 |
+
|
| 175 |
+
if accelerator.num_processes > 1:
|
| 176 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 177 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 178 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 179 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 180 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 181 |
+
kwargs = {
|
| 182 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 183 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 184 |
+
}
|
| 185 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 186 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 187 |
+
|
| 188 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 189 |
+
self._model = accelerator.prepare(self.model)
|
| 190 |
+
else:
|
| 191 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 192 |
+
self.accelerator = accelerator
|
| 193 |
+
if self.accelerator.is_local_main_process:
|
| 194 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 195 |
+
self._rank = self.accelerator.local_process_index
|
| 196 |
+
self._world_size = self.accelerator.num_processes
|
| 197 |
+
|
| 198 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 199 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 200 |
+
self._rank = 0
|
| 201 |
+
self._world_size = 1
|
| 202 |
+
|
| 203 |
+
else:
|
| 204 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 205 |
+
self.model.to(self._device)
|
| 206 |
+
self._rank = 0
|
| 207 |
+
self._world_size = 1
|
| 208 |
+
|
| 209 |
+
@property
|
| 210 |
+
def config(self):
|
| 211 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 212 |
+
return self._config
|
| 213 |
+
|
| 214 |
+
@property
|
| 215 |
+
def tokenizer(self):
|
| 216 |
+
return self._tokenizer
|
| 217 |
+
|
| 218 |
+
@property
|
| 219 |
+
def model(self):
|
| 220 |
+
# returns the model, unwrapping it if using Accelerate
|
| 221 |
+
if hasattr(self, "accelerator"):
|
| 222 |
+
return self.accelerator.unwrap_model(self._model)
|
| 223 |
+
else:
|
| 224 |
+
return self._model
|
| 225 |
+
|
| 226 |
+
@property
|
| 227 |
+
def eot_token_id(self):
|
| 228 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 229 |
+
return self.tokenizer.eos_token_id
|
| 230 |
+
|
| 231 |
+
@property
|
| 232 |
+
def max_length(self):
|
| 233 |
+
return self._max_length
|
| 234 |
+
|
| 235 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 236 |
+
if self.tokenizer.padding_side == "left":
|
| 237 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 238 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 239 |
+
if self.tokenizer.padding_side == "left":
|
| 240 |
+
input_ids = torch.flip(input_ids, [1])
|
| 241 |
+
return input_ids
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def batch_size(self):
|
| 245 |
+
return self.batch_size_per_gpu
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def device(self):
|
| 249 |
+
return self._device
|
| 250 |
+
|
| 251 |
+
@property
|
| 252 |
+
def rank(self):
|
| 253 |
+
return self._rank
|
| 254 |
+
|
| 255 |
+
@property
|
| 256 |
+
def world_size(self):
|
| 257 |
+
return self._world_size
|
| 258 |
+
|
| 259 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 260 |
+
""" """
|
| 261 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 262 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 263 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 264 |
+
if left_truncate_len:
|
| 265 |
+
encoding = encoding[-left_truncate_len:]
|
| 266 |
+
return encoding
|
| 267 |
+
|
| 268 |
+
def tok_decode(self, tokens):
|
| 269 |
+
try:
|
| 270 |
+
return self.tokenizer.decode(tokens)
|
| 271 |
+
except:
|
| 272 |
+
return self.tokenizer.decode([tokens])
|
| 273 |
+
|
| 274 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 275 |
+
# TODO
|
| 276 |
+
raise NotImplementedError("MovieChat only supports generation.")
|
| 277 |
+
|
| 278 |
+
def flatten(self, input):
|
| 279 |
+
new_list = []
|
| 280 |
+
for i in input:
|
| 281 |
+
for j in i:
|
| 282 |
+
new_list.append(j)
|
| 283 |
+
return new_list
|
| 284 |
+
|
| 285 |
+
def load_video(self, video_path, max_frames_num):
|
| 286 |
+
if type(video_path) == str:
|
| 287 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 288 |
+
else:
|
| 289 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 290 |
+
total_frame_num = len(vr)
|
| 291 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 292 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 293 |
+
spare_frames = vr.get_batch(frame_idx).numpy()
|
| 294 |
+
return spare_frames # (frames, height, width, channels)
|
| 295 |
+
|
| 296 |
+
def extract_keyframes(self, video_path, keyframes):
|
| 297 |
+
container = av.open(video_path)
|
| 298 |
+
video_stream = container.streams.video[0]
|
| 299 |
+
fps = video_stream.average_rate
|
| 300 |
+
time_base = video_stream.time_base
|
| 301 |
+
frames = []
|
| 302 |
+
|
| 303 |
+
for keyframe in keyframes:
|
| 304 |
+
keyframe_time = float(keyframe)
|
| 305 |
+
frame_number = int(keyframe_time * fps)
|
| 306 |
+
container.seek(int(keyframe_time / time_base))
|
| 307 |
+
found = False
|
| 308 |
+
for packet in container.demux(video=0):
|
| 309 |
+
for frame in packet.decode():
|
| 310 |
+
if frame.index >= frame_number:
|
| 311 |
+
frames.append(frame)
|
| 312 |
+
found = True
|
| 313 |
+
break
|
| 314 |
+
if found:
|
| 315 |
+
break
|
| 316 |
+
|
| 317 |
+
if not found:
|
| 318 |
+
container.seek(-1, any_frame=False)
|
| 319 |
+
for packet in container.demux(video=0):
|
| 320 |
+
for frame in packet.decode():
|
| 321 |
+
pass
|
| 322 |
+
frames.append(frame)
|
| 323 |
+
|
| 324 |
+
video = [x.to_ndarray(format="rgb24") for x in frames]
|
| 325 |
+
video_frames = [Image.fromarray(x) for x in video]
|
| 326 |
+
return video_frames
|
| 327 |
+
|
| 328 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 329 |
+
res = []
|
| 330 |
+
|
| 331 |
+
def _collate(x):
|
| 332 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 333 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 334 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 335 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 336 |
+
# automatic adaptive batches much much easier to implement
|
| 337 |
+
# - any OOMs will happen right away rather than near the end
|
| 338 |
+
toks = self.tok_encode(x[0])
|
| 339 |
+
return -len(toks), x[0]
|
| 340 |
+
|
| 341 |
+
# we group requests by their generation_kwargs,
|
| 342 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 343 |
+
# in the same batch.
|
| 344 |
+
metadata = requests[0].metadata
|
| 345 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 346 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 347 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 348 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 349 |
+
for chunk in chunks:
|
| 350 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 351 |
+
task = batched_task[0]
|
| 352 |
+
split = batched_split[0]
|
| 353 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 354 |
+
assert len(batched_visuals) == 1
|
| 355 |
+
|
| 356 |
+
# we assume all gen kwargs in the batch are the same
|
| 357 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 358 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 359 |
+
if "until" in gen_kwargs:
|
| 360 |
+
gen_kwargs.pop("until")
|
| 361 |
+
|
| 362 |
+
question_input = []
|
| 363 |
+
|
| 364 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 365 |
+
if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
|
| 366 |
+
self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
|
| 367 |
+
eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
|
| 368 |
+
if type(visual[0]) == PIL.Image.Image and "task_type" not in metadata and "sample_frames" not in metadata: # For image task
|
| 369 |
+
raise NotImplementedError("MovieChat only supports video inputs.")
|
| 370 |
+
|
| 371 |
+
elif "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata:
|
| 372 |
+
raise NotImplementedError("MovieChat only supports video inputs.")
|
| 373 |
+
|
| 374 |
+
elif type(visual[0]) == str: # For video task
|
| 375 |
+
try:
|
| 376 |
+
self.short_memory_buffer = []
|
| 377 |
+
self.long_memory_buffer = []
|
| 378 |
+
# try:
|
| 379 |
+
os.makedirs(self.tmp_folder, exist_ok=True)
|
| 380 |
+
|
| 381 |
+
video = VideoFileClip(visual[0])
|
| 382 |
+
clip_duration = video.duration / self.num_clips
|
| 383 |
+
|
| 384 |
+
cur_frame = 0
|
| 385 |
+
for i in range(self.num_clips):
|
| 386 |
+
start_time = i * clip_duration
|
| 387 |
+
end_time = start_time + clip_duration
|
| 388 |
+
# uniformly sample self.sliding_window_length frames from the video from start_time to end_time
|
| 389 |
+
frames = list(video.subclip(start_time, end_time).iter_frames(fps=self.sliding_window_length / clip_duration))[: self.sliding_window_length]
|
| 390 |
+
frames = [Image.fromarray(frame).convert("RGB") for frame in frames]
|
| 391 |
+
preprocess_frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 392 |
+
encoded_window = self.model.encode_images(preprocess_frames) # [frames, 729,3584]
|
| 393 |
+
|
| 394 |
+
for frame in encoded_window:
|
| 395 |
+
if cur_frame < (self.short_memory_length - self.merge_frame_length):
|
| 396 |
+
if len(self.short_memory_buffer) == self.short_memory_length:
|
| 397 |
+
self.short_memory_buffer.pop(0)
|
| 398 |
+
self.short_memory_buffer.append(frame)
|
| 399 |
+
cur_frame += 1
|
| 400 |
+
|
| 401 |
+
if cur_frame == (self.short_memory_length - self.merge_frame_length):
|
| 402 |
+
cur_frame = 0
|
| 403 |
+
|
| 404 |
+
# merge short_memory_frames
|
| 405 |
+
similar_list = []
|
| 406 |
+
for frame_i in range(len(self.short_memory_buffer) - 1):
|
| 407 |
+
scores = self.short_memory_buffer[frame_i] @ self.short_memory_buffer[frame_i + 1].transpose(-1, -2)
|
| 408 |
+
frame_silimar = torch.mean(scores)
|
| 409 |
+
similar_list.append(frame_silimar)
|
| 410 |
+
|
| 411 |
+
while len(self.short_memory_buffer) > self.merge_frame_length:
|
| 412 |
+
max_value = max(similar_list)
|
| 413 |
+
max_index = similar_list.index(max_value)
|
| 414 |
+
new_frame_feature = (self.short_memory_buffer[max_index].cpu() + self.short_memory_buffer[max_index + 1].cpu()) / 2
|
| 415 |
+
self.short_memory_buffer[max_index] = new_frame_feature.cuda()
|
| 416 |
+
del self.short_memory_buffer[max_index + 1]
|
| 417 |
+
similar_list = []
|
| 418 |
+
for frame_i in range(len(self.short_memory_buffer) - 1):
|
| 419 |
+
scores = self.short_memory_buffer[frame_i] @ self.short_memory_buffer[frame_i + 1].transpose(-1, -2)
|
| 420 |
+
frame_silimar = torch.mean(scores)
|
| 421 |
+
similar_list.append(frame_silimar)
|
| 422 |
+
|
| 423 |
+
for frame in self.short_memory_buffer:
|
| 424 |
+
self.long_memory_buffer.append(frame)
|
| 425 |
+
|
| 426 |
+
image_features = torch.stack(self.long_memory_buffer)
|
| 427 |
+
except Exception as e:
|
| 428 |
+
print(e)
|
| 429 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 430 |
+
image_features = None
|
| 431 |
+
|
| 432 |
+
task_type = "video"
|
| 433 |
+
placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
|
| 434 |
+
|
| 435 |
+
if image_features is not None and DEFAULT_IMAGE_TOKEN not in context:
|
| 436 |
+
"""
|
| 437 |
+
Three senarios:
|
| 438 |
+
1. No image, and there for, no image token should be added.
|
| 439 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 440 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 441 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 442 |
+
"""
|
| 443 |
+
# if task_type == "image": # indeed in multi-image case, not the video in frames.
|
| 444 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 445 |
+
# elif task_type == "video":
|
| 446 |
+
# image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 447 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
|
| 448 |
+
image_tokens = " ".join(image_tokens)
|
| 449 |
+
question = image_tokens + "\n" + context
|
| 450 |
+
else:
|
| 451 |
+
question = context
|
| 452 |
+
|
| 453 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 454 |
+
if "llama_3" in self.conv_template:
|
| 455 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 456 |
+
else:
|
| 457 |
+
conv = conv_templates[self.conv_template].copy()
|
| 458 |
+
|
| 459 |
+
if utils.is_json(question): # conversational question input
|
| 460 |
+
question = json.loads(question)
|
| 461 |
+
for idx, item in enumerate(question):
|
| 462 |
+
role = conv.roles[idx % 2]
|
| 463 |
+
message = item["value"]
|
| 464 |
+
conv.append_message(role, message)
|
| 465 |
+
|
| 466 |
+
assert len(conv.messages) % 2 == 1
|
| 467 |
+
conv.append_message(conv.roles[1], None)
|
| 468 |
+
prompt_question = conv.get_prompt()
|
| 469 |
+
question_input.append(prompt_question)
|
| 470 |
+
else: # only simple string for question
|
| 471 |
+
conv.append_message(conv.roles[0], question)
|
| 472 |
+
conv.append_message(conv.roles[1], None)
|
| 473 |
+
prompt_question = conv.get_prompt()
|
| 474 |
+
question_input.append(prompt_question)
|
| 475 |
+
|
| 476 |
+
# preconfigure gen_kwargs with defaults
|
| 477 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 478 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 479 |
+
if "temperature" not in gen_kwargs:
|
| 480 |
+
gen_kwargs["temperature"] = 0
|
| 481 |
+
if "do_sample" not in gen_kwargs:
|
| 482 |
+
gen_kwargs["do_sample"] = False
|
| 483 |
+
if "top_p" not in gen_kwargs:
|
| 484 |
+
gen_kwargs["top_p"] = None
|
| 485 |
+
if "num_beams" not in gen_kwargs:
|
| 486 |
+
gen_kwargs["num_beams"] = 1
|
| 487 |
+
|
| 488 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 489 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 490 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 491 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 492 |
+
|
| 493 |
+
if task_type == "image":
|
| 494 |
+
gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
|
| 495 |
+
elif task_type == "video":
|
| 496 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 497 |
+
keywords = [stop_str]
|
| 498 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 499 |
+
gen_kwargs["modalities"] = ["video"]
|
| 500 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 501 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 502 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 503 |
+
|
| 504 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 505 |
+
# TODO: attention to this major generation step...
|
| 506 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 507 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 508 |
+
try:
|
| 509 |
+
with torch.inference_mode():
|
| 510 |
+
gen_kwargs.pop("modalities")
|
| 511 |
+
cont = self.model.generate_moviechat(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, image_features=image_features, use_cache=self.use_cache, **gen_kwargs)
|
| 512 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 513 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 514 |
+
except Exception as e:
|
| 515 |
+
print(e)
|
| 516 |
+
text_outputs = "Can not infer the answer."
|
| 517 |
+
|
| 518 |
+
res.extend(text_outputs)
|
| 519 |
+
print(text_outputs)
|
| 520 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 521 |
+
pbar.update(1)
|
| 522 |
+
# reorder this group of results back to original unsorted form
|
| 523 |
+
res = re_ords.get_original(res)
|
| 524 |
+
|
| 525 |
+
pbar.close()
|
| 526 |
+
return res
|
LaViDa/eval/lmms_eval/models/llava_sglang.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import random
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
import warnings
|
| 9 |
+
from datetime import timedelta
|
| 10 |
+
from typing import List, Optional, Tuple, Union
|
| 11 |
+
|
| 12 |
+
from tqdm import tqdm
|
| 13 |
+
|
| 14 |
+
from lmms_eval import utils
|
| 15 |
+
from lmms_eval.api.instance import Instance
|
| 16 |
+
from lmms_eval.api.model import lmms
|
| 17 |
+
from lmms_eval.api.registry import register_model
|
| 18 |
+
|
| 19 |
+
warnings.filterwarnings("ignore")
|
| 20 |
+
import tempfile
|
| 21 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 22 |
+
|
| 23 |
+
from loguru import logger as eval_logger
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
import sglang as sgl
|
| 27 |
+
from sglang.lang.chat_template import get_chat_template
|
| 28 |
+
except ImportError:
|
| 29 |
+
eval_logger.debug("SGLang is not installed. If you want to use llava_sglang, please install it using pip install 'sglang[all]' ")
|
| 30 |
+
|
| 31 |
+
if torch.__version__ > "2.1.2":
|
| 32 |
+
best_fit_attn_implementation = "sdpa"
|
| 33 |
+
else:
|
| 34 |
+
best_fit_attn_implementation = "eager"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@register_model("llava_sglang")
|
| 38 |
+
class LlavaSglang(lmms):
|
| 39 |
+
"""
|
| 40 |
+
Llava Sglang Model
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
def __init__(
|
| 44 |
+
self,
|
| 45 |
+
pretrained: str = "liuhaotian/llava-v1.5-7b",
|
| 46 |
+
tokenizer: str = "llava-hf/llava-1.5-7b-hf",
|
| 47 |
+
tp_size: int = 1,
|
| 48 |
+
parallel: Optional[Union[int, str]] = 64,
|
| 49 |
+
conv_template="vicuna_v1.1",
|
| 50 |
+
**kwargs,
|
| 51 |
+
) -> None:
|
| 52 |
+
super().__init__()
|
| 53 |
+
self.pretrained = pretrained
|
| 54 |
+
self.tokenizer = tokenizer
|
| 55 |
+
self.tp_size = tp_size
|
| 56 |
+
self.conv_template = conv_template
|
| 57 |
+
# torch.multiprocessing.set_start_method("spawn")
|
| 58 |
+
|
| 59 |
+
# accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 60 |
+
# accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 61 |
+
# assert accelerator.num_processes == 1, "Llava-sglang does not support multi-processes yet (it does support tensor parallelism)."
|
| 62 |
+
self._rank = 0
|
| 63 |
+
self._world_size = 1
|
| 64 |
+
self.parallel = parallel
|
| 65 |
+
|
| 66 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 67 |
+
raise NotImplementedError("Llava-sglang does not support loglikelihood evaluation yet")
|
| 68 |
+
|
| 69 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 70 |
+
torch.multiprocessing.set_start_method("spawn", force=True)
|
| 71 |
+
runtime = sgl.Runtime(model_path=self.pretrained, tokenizer_path=self.tokenizer, tp_size=self.tp_size, port=random.randint(10000, 50000))
|
| 72 |
+
runtime.endpoint.chat_template = get_chat_template(self.conv_template)
|
| 73 |
+
sgl.set_default_backend(runtime)
|
| 74 |
+
|
| 75 |
+
@sgl.function
|
| 76 |
+
def image_qa(s, image_file, question):
|
| 77 |
+
s += sgl.user(sgl.image(image_file) + question)
|
| 78 |
+
s += sgl.assistant(sgl.gen("answer"))
|
| 79 |
+
|
| 80 |
+
res = []
|
| 81 |
+
|
| 82 |
+
def _collate(x):
|
| 83 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 84 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 85 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 86 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 87 |
+
# automatic adaptive batches much much easier to implement
|
| 88 |
+
# - any OOMs will happen right away rather than near the end
|
| 89 |
+
toks = x[0].split(" ")
|
| 90 |
+
return -len(toks), x[0]
|
| 91 |
+
|
| 92 |
+
# we group requests by their generation_kwargs,
|
| 93 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 94 |
+
# in the same batch.
|
| 95 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 96 |
+
chunks = re_ords.get_batched(n=self.parallel, batch_fn=None)
|
| 97 |
+
num_iters = len(requests) // self.parallel if len(requests) % self.parallel == 0 else len(requests) // self.parallel + 1
|
| 98 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 99 |
+
for chunk in chunks:
|
| 100 |
+
contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
|
| 101 |
+
batched_visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)] # [B, N]
|
| 102 |
+
# we assume all gen kwargs in the batch are the same
|
| 103 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 104 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 105 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 106 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 107 |
+
if "temperature" not in gen_kwargs:
|
| 108 |
+
gen_kwargs["temperature"] = 0
|
| 109 |
+
if "top_p" not in gen_kwargs:
|
| 110 |
+
gen_kwargs["top_p"] = 1.0
|
| 111 |
+
if "num_beams" not in gen_kwargs:
|
| 112 |
+
gen_kwargs["num_beams"] = 1
|
| 113 |
+
assert gen_kwargs["num_beams"] == 1
|
| 114 |
+
|
| 115 |
+
def save_image_to_temp_file(image):
|
| 116 |
+
temp_file = tempfile.NamedTemporaryFile(suffix=".jpeg", delete=True)
|
| 117 |
+
image.save(temp_file.name)
|
| 118 |
+
return temp_file
|
| 119 |
+
|
| 120 |
+
def prepare_arguments_parallel(contexts, batched_visuals, max_workers=64):
|
| 121 |
+
arguments = [None] * len(contexts) # Initialize with placeholders
|
| 122 |
+
tmp_files = [None] * len(contexts) # Initialize with placeholders
|
| 123 |
+
|
| 124 |
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 125 |
+
# Associate each future with its index and content
|
| 126 |
+
future_to_info = {executor.submit(save_image_to_temp_file, pil_list[0]): (index, context, pil_list) for index, (context, pil_list) in enumerate(zip(contexts, batched_visuals))}
|
| 127 |
+
|
| 128 |
+
for future in as_completed(future_to_info):
|
| 129 |
+
index, context, pil_list = future_to_info[future]
|
| 130 |
+
if len(pil_list) > 1:
|
| 131 |
+
eval_logger.warning("Llava-sglang only supports one visual input per question. Using the first visual input.")
|
| 132 |
+
try:
|
| 133 |
+
temp_file = future.result()
|
| 134 |
+
arguments[index] = {
|
| 135 |
+
"image_file": temp_file.name,
|
| 136 |
+
"question": context,
|
| 137 |
+
}
|
| 138 |
+
tmp_files[index] = temp_file
|
| 139 |
+
except Exception as exc:
|
| 140 |
+
print(f"Generated an exception: {exc}")
|
| 141 |
+
|
| 142 |
+
# Filter out any None values in case of exceptions
|
| 143 |
+
arguments = [arg for arg in arguments if arg is not None]
|
| 144 |
+
tmp_files = [tmp_file for tmp_file in tmp_files if tmp_file is not None]
|
| 145 |
+
|
| 146 |
+
return arguments, tmp_files
|
| 147 |
+
|
| 148 |
+
arguments, tmp_files = prepare_arguments_parallel(contexts, batched_visuals, self.parallel)
|
| 149 |
+
states = image_qa.run_batch(arguments, temperature=gen_kwargs["temperature"], max_new_tokens=gen_kwargs["max_new_tokens"], top_p=gen_kwargs["top_p"], num_threads=self.parallel, progress_bar=False)
|
| 150 |
+
|
| 151 |
+
text_outputs = [state["answer"].strip() for state in states]
|
| 152 |
+
# clean up the temporary files
|
| 153 |
+
for tmp_file in tmp_files:
|
| 154 |
+
tmp_file.close()
|
| 155 |
+
res.extend(text_outputs)
|
| 156 |
+
pbar.update(1)
|
| 157 |
+
# reorder this group of results back to original unsorted form
|
| 158 |
+
res = re_ords.get_original(res)
|
| 159 |
+
|
| 160 |
+
pbar.close()
|
| 161 |
+
runtime.shutdown()
|
| 162 |
+
return res
|
| 163 |
+
|
| 164 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 165 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaVA-SGLang")
|
LaViDa/eval/lmms_eval/models/llava_vid.py
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import glob
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
from datetime import timedelta
|
| 5 |
+
from typing import List, Optional, Tuple, Union
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 10 |
+
from accelerate.state import AcceleratorState
|
| 11 |
+
from decord import VideoReader, cpu
|
| 12 |
+
from llava.constants import (
|
| 13 |
+
DEFAULT_IM_END_TOKEN,
|
| 14 |
+
DEFAULT_IM_START_TOKEN,
|
| 15 |
+
DEFAULT_IMAGE_TOKEN,
|
| 16 |
+
IGNORE_INDEX,
|
| 17 |
+
IMAGE_TOKEN_INDEX,
|
| 18 |
+
)
|
| 19 |
+
from llava.conversation import SeparatorStyle, conv_templates
|
| 20 |
+
from llava.mm_utils import (
|
| 21 |
+
KeywordsStoppingCriteria,
|
| 22 |
+
get_model_name_from_path,
|
| 23 |
+
tokenizer_image_token,
|
| 24 |
+
)
|
| 25 |
+
from llava.model.builder import load_pretrained_model
|
| 26 |
+
from llava.model.language_model.llava_llama import LlavaConfig
|
| 27 |
+
from llava.model.language_model.llava_qwen import LlavaQwenConfig
|
| 28 |
+
|
| 29 |
+
# eval_logger = logging.getLogger("lmms-eval")
|
| 30 |
+
# import sys;sys.path.append("llava-video")
|
| 31 |
+
from loguru import logger as eval_logger
|
| 32 |
+
from PIL import Image
|
| 33 |
+
from tqdm import tqdm
|
| 34 |
+
from transformers import AutoConfig, AutoModelForCausalLM
|
| 35 |
+
|
| 36 |
+
from lmms_eval.api.instance import Instance
|
| 37 |
+
from lmms_eval.api.model import lmms
|
| 38 |
+
from lmms_eval.api.registry import register_model
|
| 39 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 40 |
+
|
| 41 |
+
# try:
|
| 42 |
+
# from llavavid.model.builder import load_pretrained_model
|
| 43 |
+
# from llavavid.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
|
| 44 |
+
# from llavavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
|
| 45 |
+
# from llavavid.conversation import conv_templates, SeparatorStyle
|
| 46 |
+
# from llavavid.mm_utils import tokenizer_image_token_qwen_merge, preprocess_qwen, preprocess_llama3
|
| 47 |
+
# except ImportError:
|
| 48 |
+
# import llava
|
| 49 |
+
# import pdb;pdb.set_trace()
|
| 50 |
+
# if "llava-video-old" in llava.__file__:
|
| 51 |
+
# from llava.model.language_model.llava_llama import LlavaConfig
|
| 52 |
+
# from llava.model.language_model.llava_qwen import LlavaQwenConfig
|
| 53 |
+
# from llava.model.builder import load_pretrained_model
|
| 54 |
+
# from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
|
| 55 |
+
# from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
|
| 56 |
+
# from llava.conversation import conv_templates, SeparatorStyle
|
| 57 |
+
|
| 58 |
+
# AutoConfig.register("llava_llama", LlavaConfig)
|
| 59 |
+
# AutoConfig.register("llava_qwen", LlavaQwenConfig)
|
| 60 |
+
# else:
|
| 61 |
+
# eval_logger.debug("LLaVA-Video is not installed. Please install LLaVA-Video to use this model.")
|
| 62 |
+
|
| 63 |
+
# from llavavid.model.language_model.llava_qwen import LlavaQwenConfig
|
| 64 |
+
# from llavavid.model.language_model.llava_llama import LlavaConfig
|
| 65 |
+
|
| 66 |
+
# AutoConfig.register("llava_qwen", LlavaQwenConfig)
|
| 67 |
+
# AutoConfig.register("llava_llama", LlavaConfig)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
AutoConfig.register("llava_llama", LlavaConfig)
|
| 71 |
+
AutoConfig.register("llava_qwen", LlavaQwenConfig)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@register_model("llava_vid")
|
| 75 |
+
class LlavaVid(lmms):
|
| 76 |
+
"""
|
| 77 |
+
LlavaVid Model
|
| 78 |
+
"""
|
| 79 |
+
|
| 80 |
+
def __init__(
|
| 81 |
+
self,
|
| 82 |
+
pretrained: str = "liuhaotian/llava-v1.5-7b",
|
| 83 |
+
truncation: Optional[bool] = True,
|
| 84 |
+
torch_dtype: Optional[Union[str, torch.dtype]] = "float16",
|
| 85 |
+
device: Optional[str] = "cuda:0",
|
| 86 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 87 |
+
attn_implementation=(
|
| 88 |
+
"sdpa" if torch.__version__ >= "2.1.2" else "eager"
|
| 89 |
+
), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 90 |
+
device_map="cuda:0",
|
| 91 |
+
conv_template="vicuna_v1",
|
| 92 |
+
use_cache=True,
|
| 93 |
+
truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 94 |
+
max_frames_num: int = 20,
|
| 95 |
+
video_fps: int = 1,
|
| 96 |
+
mm_resampler_type: str = "spatial_pool",
|
| 97 |
+
mm_spatial_pool_stride: int = 2,
|
| 98 |
+
mm_spatial_pool_out_channels: int = 1024,
|
| 99 |
+
mm_spatial_pool_mode: str = "average",
|
| 100 |
+
mm_resampler_location: str = "before",
|
| 101 |
+
mm_newline_position: str = "grid",
|
| 102 |
+
overwrite: bool = True,
|
| 103 |
+
video_decode_backend: str = "decord",
|
| 104 |
+
delay_load: bool = False,
|
| 105 |
+
tie_weights: bool = True,
|
| 106 |
+
force_sample: bool = False,
|
| 107 |
+
add_time_instruction: bool = False,
|
| 108 |
+
add_faster_video: bool = False,
|
| 109 |
+
faster_token_stride: int = 10,
|
| 110 |
+
**kwargs,
|
| 111 |
+
) -> None:
|
| 112 |
+
super().__init__()
|
| 113 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 114 |
+
|
| 115 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 116 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 117 |
+
if accelerator.num_processes > 1:
|
| 118 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 119 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 120 |
+
elif accelerator.num_processes == 1 and (device_map == "auto" or device_map == "balanced_low_0"):
|
| 121 |
+
self._device = torch.device(device)
|
| 122 |
+
self.device_map = device_map
|
| 123 |
+
else:
|
| 124 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 125 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 126 |
+
|
| 127 |
+
self.pretrained = pretrained
|
| 128 |
+
self.model_name = get_model_name_from_path(pretrained)
|
| 129 |
+
self.video_decode_backend = video_decode_backend
|
| 130 |
+
# self._config = AutoConfig.from_pretrained(self.pretrained)
|
| 131 |
+
self.overwrite = overwrite
|
| 132 |
+
self.mm_resampler_type = mm_resampler_type
|
| 133 |
+
self.mm_spatial_pool_stride = int(mm_spatial_pool_stride)
|
| 134 |
+
self.mm_spatial_pool_out_channels = int(mm_spatial_pool_out_channels)
|
| 135 |
+
self.mm_spatial_pool_mode = mm_spatial_pool_mode
|
| 136 |
+
self.max_frames_num = int(max_frames_num)
|
| 137 |
+
self.fps = int(video_fps)
|
| 138 |
+
self.mm_resampler_location = mm_resampler_location
|
| 139 |
+
self.delay_load = delay_load
|
| 140 |
+
self.force_sample = force_sample
|
| 141 |
+
self.add_time_instruction = add_time_instruction
|
| 142 |
+
print("force sample:", self.force_sample)
|
| 143 |
+
# self.add_faster_video = add_faster_video
|
| 144 |
+
# self.faster_token_stride = faster_token_stride
|
| 145 |
+
self.torch_dtype = torch_dtype
|
| 146 |
+
if self.overwrite == True:
|
| 147 |
+
overwrite_config = {}
|
| 148 |
+
# overwrite_config["mm_resampler_type"] = self.mm_resampler_type
|
| 149 |
+
overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
|
| 150 |
+
overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
|
| 151 |
+
overwrite_config["mm_pooling_position"] = self.mm_resampler_location
|
| 152 |
+
overwrite_config["mm_newline_position"] = mm_newline_position
|
| 153 |
+
overwrite_config["delay_load"] = self.delay_load
|
| 154 |
+
# overwrite_config["attn_implementation"] = attn_implementation
|
| 155 |
+
|
| 156 |
+
if "vicuna" in self.pretrained.lower() or "yi" in self.pretrained.lower():
|
| 157 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 158 |
+
|
| 159 |
+
if "224" in cfg_pretrained.mm_vision_tower:
|
| 160 |
+
# suppose the length of text tokens is around 1000, from bo's report
|
| 161 |
+
least_token_number = self.max_frames_num * (16 // self.mm_spatial_pool_stride) ** 2 + 1000
|
| 162 |
+
else:
|
| 163 |
+
least_token_number = self.max_frames_num * (24 // self.mm_spatial_pool_stride) ** 2 + 1000
|
| 164 |
+
|
| 165 |
+
scaling_factor = math.ceil(least_token_number / 4096)
|
| 166 |
+
if scaling_factor >= 2:
|
| 167 |
+
eval_logger.info(f"Scaling factor: {scaling_factor}")
|
| 168 |
+
# print(float(scaling_factor))
|
| 169 |
+
overwrite_config["rope_scaling"] = {"factor": float(scaling_factor), "type": "linear"}
|
| 170 |
+
overwrite_config["max_sequence_length"] = 4096 * scaling_factor
|
| 171 |
+
overwrite_config["tokenizer_model_max_length"] = 4096 * scaling_factor
|
| 172 |
+
|
| 173 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(
|
| 174 |
+
pretrained, None, self.model_name, device_map=self.device_map, torch_dtype=self.torch_dtype, overwrite_config=overwrite_config, attn_implementation=attn_implementation
|
| 175 |
+
)
|
| 176 |
+
else:
|
| 177 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, self.model_name, device_map=self.device_map, torch_dtype=self.torch_dtype, attn_implementation=attn_implementation)
|
| 178 |
+
|
| 179 |
+
self._config = self._model.config
|
| 180 |
+
|
| 181 |
+
# import pdb;pdb.set_trace()
|
| 182 |
+
|
| 183 |
+
if self._tokenizer.pad_token_id is None:
|
| 184 |
+
if "qwen" in self._tokenizer.name_or_path.lower():
|
| 185 |
+
print("Setting pad token to bos token for qwen model.")
|
| 186 |
+
self._tokenizer.pad_token_id = 151643
|
| 187 |
+
|
| 188 |
+
self.model.eval()
|
| 189 |
+
if tie_weights:
|
| 190 |
+
self.model.tie_weights()
|
| 191 |
+
self.truncation = truncation
|
| 192 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 193 |
+
self.conv_template = conv_template
|
| 194 |
+
self.use_cache = use_cache
|
| 195 |
+
self.truncate_context = truncate_context
|
| 196 |
+
# assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 197 |
+
if accelerator.num_processes > 1:
|
| 198 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 199 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 200 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 201 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 202 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 203 |
+
kwargs = {
|
| 204 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 205 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 206 |
+
}
|
| 207 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 208 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 209 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 210 |
+
self._model = accelerator.prepare(self.model)
|
| 211 |
+
else:
|
| 212 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 213 |
+
self.accelerator = accelerator
|
| 214 |
+
if self.accelerator.is_local_main_process:
|
| 215 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 216 |
+
self._rank = self.accelerator.local_process_index
|
| 217 |
+
self._world_size = self.accelerator.num_processes
|
| 218 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 219 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 220 |
+
self._rank = 0
|
| 221 |
+
self._world_size = 1
|
| 222 |
+
else:
|
| 223 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 224 |
+
self.model.to(self._device)
|
| 225 |
+
self._rank = 0
|
| 226 |
+
self._world_size = 1
|
| 227 |
+
|
| 228 |
+
@property
|
| 229 |
+
def config(self):
|
| 230 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 231 |
+
return self._config
|
| 232 |
+
|
| 233 |
+
@property
|
| 234 |
+
def tokenizer(self):
|
| 235 |
+
return self._tokenizer
|
| 236 |
+
|
| 237 |
+
@property
|
| 238 |
+
def model(self):
|
| 239 |
+
# returns the model, unwrapping it if using Accelerate
|
| 240 |
+
if hasattr(self, "accelerator"):
|
| 241 |
+
return self.accelerator.unwrap_model(self._model)
|
| 242 |
+
else:
|
| 243 |
+
return self._model
|
| 244 |
+
|
| 245 |
+
@property
|
| 246 |
+
def eot_token_id(self):
|
| 247 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 248 |
+
return self.tokenizer.eos_token_id
|
| 249 |
+
|
| 250 |
+
@property
|
| 251 |
+
def max_length(self):
|
| 252 |
+
return self._max_length
|
| 253 |
+
|
| 254 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 255 |
+
if self.tokenizer.padding_side == "left":
|
| 256 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 257 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 258 |
+
if self.tokenizer.padding_side == "left":
|
| 259 |
+
input_ids = torch.flip(input_ids, [1])
|
| 260 |
+
return input_ids
|
| 261 |
+
|
| 262 |
+
@property
|
| 263 |
+
def batch_size(self):
|
| 264 |
+
return self.batch_size_per_gpu
|
| 265 |
+
|
| 266 |
+
@property
|
| 267 |
+
def device(self):
|
| 268 |
+
return self._device
|
| 269 |
+
|
| 270 |
+
@property
|
| 271 |
+
def rank(self):
|
| 272 |
+
return self._rank
|
| 273 |
+
|
| 274 |
+
@property
|
| 275 |
+
def world_size(self):
|
| 276 |
+
return self._world_size
|
| 277 |
+
|
| 278 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 279 |
+
""" """
|
| 280 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 281 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 282 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 283 |
+
if left_truncate_len:
|
| 284 |
+
encoding = encoding[-left_truncate_len:]
|
| 285 |
+
return encoding
|
| 286 |
+
|
| 287 |
+
def load_image(self, image_path):
|
| 288 |
+
frame_files = [os.path.join(image_path, f) for f in os.listdir(image_path) if os.path.isfile(os.path.join(image_path, f))]
|
| 289 |
+
frame_files.sort() # Ensure the frames are sorted if they are named sequentially
|
| 290 |
+
|
| 291 |
+
# TODO: Hard CODE: Determine the indices for uniformly sampling 10 frames
|
| 292 |
+
num_frames_to_sample = 10
|
| 293 |
+
|
| 294 |
+
total_frames = len(frame_files)
|
| 295 |
+
|
| 296 |
+
sampled_indices = np.linspace(0, total_frames - 1, num_frames_to_sample, dtype=int)
|
| 297 |
+
|
| 298 |
+
# Read and store the sampled frames
|
| 299 |
+
video = []
|
| 300 |
+
for idx in sampled_indices:
|
| 301 |
+
frame_path = frame_files[idx]
|
| 302 |
+
try:
|
| 303 |
+
with Image.open(frame_path) as img:
|
| 304 |
+
# Convert the PIL image to a numpy array if needed
|
| 305 |
+
# frame = np.array(img.convert('RGB'))
|
| 306 |
+
frame = img.convert("RGB")
|
| 307 |
+
video.append(frame)
|
| 308 |
+
except IOError:
|
| 309 |
+
print(f"Failed to read frame at path: {frame_path}")
|
| 310 |
+
return video
|
| 311 |
+
|
| 312 |
+
def load_video(self, video_path, max_frames_num, fps, force_sample=False):
|
| 313 |
+
if max_frames_num == 0:
|
| 314 |
+
return np.zeros((1, 336, 336, 3))
|
| 315 |
+
vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
|
| 316 |
+
total_frame_num = len(vr)
|
| 317 |
+
video_time = total_frame_num / vr.get_avg_fps()
|
| 318 |
+
fps = round(vr.get_avg_fps() / fps)
|
| 319 |
+
frame_idx = [i for i in range(0, len(vr), fps)]
|
| 320 |
+
frame_time = [i / fps for i in frame_idx]
|
| 321 |
+
if len(frame_idx) > max_frames_num or force_sample:
|
| 322 |
+
sample_fps = max_frames_num
|
| 323 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, sample_fps, dtype=int)
|
| 324 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 325 |
+
frame_time = [i / vr.get_avg_fps() for i in frame_idx]
|
| 326 |
+
frame_time = ",".join([f"{i:.2f}s" for i in frame_time])
|
| 327 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 328 |
+
# import pdb;pdb.set_trace()
|
| 329 |
+
|
| 330 |
+
return spare_frames, frame_time, video_time
|
| 331 |
+
|
| 332 |
+
def tok_decode(self, tokens):
|
| 333 |
+
return self.tokenizer.decode(tokens)
|
| 334 |
+
|
| 335 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 336 |
+
res = []
|
| 337 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 338 |
+
|
| 339 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 340 |
+
# encode, pad, and truncate contexts for this batch
|
| 341 |
+
if type(doc_to_target) == str:
|
| 342 |
+
continuation = doc_to_target
|
| 343 |
+
else:
|
| 344 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 345 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 346 |
+
visuals = self.flatten(visuals)
|
| 347 |
+
videos = []
|
| 348 |
+
for visual in visuals:
|
| 349 |
+
video, frame_time, video_time = self.load_video(visual, self.max_frames_num, self.fps, force_sample=self.force_sample)
|
| 350 |
+
video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].cuda()
|
| 351 |
+
if self.torch_dtype == "bfloat16":
|
| 352 |
+
video = video.bfloat16()
|
| 353 |
+
else:
|
| 354 |
+
video = video.half()
|
| 355 |
+
videos.append(video)
|
| 356 |
+
|
| 357 |
+
qs = contexts
|
| 358 |
+
if self.model.config.mm_use_im_start_end:
|
| 359 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 360 |
+
else:
|
| 361 |
+
qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 362 |
+
|
| 363 |
+
conv = conv_templates[self.conv_template].copy()
|
| 364 |
+
conv.append_message(conv.roles[0], qs)
|
| 365 |
+
conv.append_message(conv.roles[1], None)
|
| 366 |
+
prompt = conv.get_prompt()
|
| 367 |
+
|
| 368 |
+
contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 369 |
+
|
| 370 |
+
conv = conv_templates[self.conv_template].copy()
|
| 371 |
+
conv.append_message(conv.roles[0], qs)
|
| 372 |
+
conv.append_message(conv.roles[1], continuation)
|
| 373 |
+
prompt = conv.get_prompt()
|
| 374 |
+
|
| 375 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
|
| 376 |
+
attention_masks = input_ids.ne(self.tokenizer.pad_token_id).long().cuda()
|
| 377 |
+
|
| 378 |
+
labels = input_ids.clone()
|
| 379 |
+
# Context part no need to calculate for loss
|
| 380 |
+
labels[0, : contxt_id.shape[1]] = -100
|
| 381 |
+
|
| 382 |
+
with torch.inference_mode():
|
| 383 |
+
outputs = self.model(input_ids=input_ids, labels=labels, images=videos, modalities="video")
|
| 384 |
+
|
| 385 |
+
loss = outputs["loss"]
|
| 386 |
+
# loss = torch.exp(loss)
|
| 387 |
+
logits = outputs["logits"]
|
| 388 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 389 |
+
cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
|
| 390 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
|
| 391 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 392 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 393 |
+
pbar.update(1)
|
| 394 |
+
pbar.close()
|
| 395 |
+
return res
|
| 396 |
+
|
| 397 |
+
def flatten(self, input):
|
| 398 |
+
new_list = []
|
| 399 |
+
for i in input:
|
| 400 |
+
for j in i:
|
| 401 |
+
new_list.append(j)
|
| 402 |
+
return new_list
|
| 403 |
+
|
| 404 |
+
def generate_until(self, requests) -> List[str]:
|
| 405 |
+
res = []
|
| 406 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 407 |
+
|
| 408 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 409 |
+
# if self.task_dict[task][split][doc_id]["duration"] != "short":
|
| 410 |
+
# # if doc_id != 112:
|
| 411 |
+
# # import pdb;pdb.set_trace()
|
| 412 |
+
# res.append("A")
|
| 413 |
+
# pbar.update(1)
|
| 414 |
+
# continue
|
| 415 |
+
# encode, pad, and truncate contexts for this batch
|
| 416 |
+
# import pdb;pdb.set_trace()
|
| 417 |
+
visuals = doc_to_visual(self.task_dict[task][split][doc_id])
|
| 418 |
+
# visuals = [visuals]
|
| 419 |
+
# visuals = self.flatten(visuals)
|
| 420 |
+
if os.path.isdir(visuals[0]):
|
| 421 |
+
visuals = glob.glob(visuals[0] + "/*")
|
| 422 |
+
videos = []
|
| 423 |
+
try:
|
| 424 |
+
# for visual in visuals:
|
| 425 |
+
if len(visuals) == 1:
|
| 426 |
+
if self.video_decode_backend == "decord":
|
| 427 |
+
video, frame_time, video_time = self.load_video(visuals[0], self.max_frames_num, self.fps, force_sample=self.force_sample)
|
| 428 |
+
elif self.video_decode_backend == "pyav":
|
| 429 |
+
video, frame_time, video_time = read_video_pyav(visuals[0], self.max_frames_num, self.fps, force_sample=self.force_sample)
|
| 430 |
+
elif self.video_decode_backend == "image":
|
| 431 |
+
video = self.load_image(visuals[0])
|
| 432 |
+
else:
|
| 433 |
+
if task == "seedbench":
|
| 434 |
+
video = visuals
|
| 435 |
+
frame_time = "1.00s"
|
| 436 |
+
video_time = 1
|
| 437 |
+
elif "mvbench" in task:
|
| 438 |
+
# video = visuals
|
| 439 |
+
# Reference: https://github.com/jayleicn/TVQA/blob/dfb0e5fe4582efca574dfddfeafd1008db3b33ef/data/README.md?plain=1#L50C34-L50C60
|
| 440 |
+
fps = 3
|
| 441 |
+
video_time = len(visuals) / fps
|
| 442 |
+
sampled_indices = np.linspace(0, len(visuals) - 1, self.max_frames_num, dtype=int)
|
| 443 |
+
frame_idx = sampled_indices.tolist()
|
| 444 |
+
frame_time = [i / fps for i in frame_idx]
|
| 445 |
+
frame_time = ",".join([f"{i:.2f}s" for i in frame_time])
|
| 446 |
+
# video = [visuals[i] for i in frame_idx]
|
| 447 |
+
video = np.stack([np.array(Image.open(visuals[i])) for i in frame_idx], axis=0)
|
| 448 |
+
|
| 449 |
+
video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].cuda()
|
| 450 |
+
if self.torch_dtype == "bfloat16":
|
| 451 |
+
video = video.bfloat16()
|
| 452 |
+
else:
|
| 453 |
+
video = video.half()
|
| 454 |
+
videos.append(video)
|
| 455 |
+
except Exception as e:
|
| 456 |
+
# import pdb;pdb.set_trace()
|
| 457 |
+
eval_logger.info(f"{e}")
|
| 458 |
+
eval_logger.info(f"Video {visuals} can not load, check the source")
|
| 459 |
+
video_path = "\n".join(visuals)
|
| 460 |
+
res.append(f"Video {video_path} can not load, check the source")
|
| 461 |
+
pbar.update(1)
|
| 462 |
+
continue
|
| 463 |
+
|
| 464 |
+
qs = contexts
|
| 465 |
+
# import pdb;pdb.set_trace()
|
| 466 |
+
if self.add_time_instruction:
|
| 467 |
+
time_instruciton = f"The video lasts for {video_time:.2f} seconds, and {len(video)} frames are uniformly sampled from it. These frames are located at {frame_time}.Please answer the following questions related to this video."
|
| 468 |
+
qs = f"{time_instruciton}\n{qs}"
|
| 469 |
+
if self.model.config.mm_use_im_start_end:
|
| 470 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 471 |
+
else:
|
| 472 |
+
qs = DEFAULT_IMAGE_TOKEN * len(videos) + "\n" + qs
|
| 473 |
+
|
| 474 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 475 |
+
if "llama_3" in self.conv_template:
|
| 476 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 477 |
+
else:
|
| 478 |
+
conv = conv_templates[self.conv_template].copy()
|
| 479 |
+
|
| 480 |
+
conv.append_message(conv.roles[0], qs)
|
| 481 |
+
conv.append_message(conv.roles[1], None)
|
| 482 |
+
prompt = conv.get_prompt()
|
| 483 |
+
|
| 484 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
|
| 485 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 486 |
+
if "llama_3" in self.conv_template:
|
| 487 |
+
pad_token_ids = 0 # lmms-lab/llama3-llava-8b is trained on this pad token id. You may need to customize this for other models.
|
| 488 |
+
attention_masks = input_ids.ne(pad_token_ids).long().cuda()
|
| 489 |
+
|
| 490 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 491 |
+
keywords = [stop_str]
|
| 492 |
+
|
| 493 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 494 |
+
|
| 495 |
+
cur_prompt = qs
|
| 496 |
+
|
| 497 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 498 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 499 |
+
if "temperature" not in gen_kwargs:
|
| 500 |
+
gen_kwargs["temperature"] = 0
|
| 501 |
+
if "top_p" not in gen_kwargs:
|
| 502 |
+
gen_kwargs["top_p"] = None
|
| 503 |
+
if "num_beams" not in gen_kwargs:
|
| 504 |
+
gen_kwargs["num_beams"] = 1
|
| 505 |
+
|
| 506 |
+
# import pdb;pdb.set_trace()
|
| 507 |
+
with torch.inference_mode():
|
| 508 |
+
output_ids = self.model.generate(
|
| 509 |
+
inputs=input_ids,
|
| 510 |
+
images=videos,
|
| 511 |
+
attention_mask=attention_masks,
|
| 512 |
+
modalities="video",
|
| 513 |
+
use_cache=self.use_cache,
|
| 514 |
+
stopping_criteria=[stopping_criteria],
|
| 515 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 516 |
+
temperature=gen_kwargs["temperature"],
|
| 517 |
+
top_p=gen_kwargs["top_p"],
|
| 518 |
+
num_beams=gen_kwargs["num_beams"],
|
| 519 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 520 |
+
)
|
| 521 |
+
# output_ids_2 = self.model.generate(inputs=input_ids, images=videos, attention_mask=attention_masks, modalities="video", do_sample=False, max_new_tokens=50,stopping_criteria=[stopping_criteria])
|
| 522 |
+
# output_ids = self.model.generate(inputs=input_ids, images=videos, attention_mask=attention_masks, modalities="video", do_sample=True, temperature=0.2, max_new_tokens=50,use_cache=True)
|
| 523 |
+
|
| 524 |
+
outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
|
| 525 |
+
eval_logger.debug(f"Question: {cur_prompt}")
|
| 526 |
+
eval_logger.debug(f"Answer: {outputs}")
|
| 527 |
+
# import pdb;pdb.set_trace()
|
| 528 |
+
res.append(outputs)
|
| 529 |
+
pbar.update(1)
|
| 530 |
+
return res
|
| 531 |
+
|
| 532 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 533 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LLaVAVid")
|
LaViDa/eval/lmms_eval/models/longva.py
ADDED
|
@@ -0,0 +1,476 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 5 |
+
from accelerate.state import AcceleratorState
|
| 6 |
+
from transformers import AutoConfig
|
| 7 |
+
|
| 8 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 9 |
+
|
| 10 |
+
import copy
|
| 11 |
+
import logging
|
| 12 |
+
import warnings
|
| 13 |
+
from datetime import timedelta
|
| 14 |
+
from typing import List, Optional, Tuple, Union
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import PIL
|
| 18 |
+
from decord import VideoReader, cpu
|
| 19 |
+
from packaging import version
|
| 20 |
+
from tqdm import tqdm
|
| 21 |
+
|
| 22 |
+
warnings.filterwarnings("ignore")
|
| 23 |
+
|
| 24 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 25 |
+
|
| 26 |
+
from lmms_eval import utils
|
| 27 |
+
from lmms_eval.api.instance import Instance
|
| 28 |
+
from lmms_eval.api.model import lmms
|
| 29 |
+
from lmms_eval.api.registry import register_model
|
| 30 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 31 |
+
|
| 32 |
+
try:
|
| 33 |
+
from longva.constants import (
|
| 34 |
+
DEFAULT_IM_END_TOKEN,
|
| 35 |
+
DEFAULT_IM_START_TOKEN,
|
| 36 |
+
DEFAULT_IMAGE_TOKEN,
|
| 37 |
+
IGNORE_INDEX,
|
| 38 |
+
IMAGE_TOKEN_INDEX,
|
| 39 |
+
)
|
| 40 |
+
from longva.conversation import SeparatorStyle, conv_templates
|
| 41 |
+
from longva.mm_utils import (
|
| 42 |
+
KeywordsStoppingCriteria,
|
| 43 |
+
get_model_name_from_path,
|
| 44 |
+
process_images,
|
| 45 |
+
tokenizer_image_token,
|
| 46 |
+
)
|
| 47 |
+
from longva.model.builder import load_pretrained_model
|
| 48 |
+
|
| 49 |
+
except Exception as e:
|
| 50 |
+
eval_logger.debug("longva is not installed. Please install longva to use this model.\nError: %s" % e)
|
| 51 |
+
|
| 52 |
+
# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 53 |
+
# if is_flash_attn_2_available:
|
| 54 |
+
# best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
|
| 55 |
+
|
| 56 |
+
if version.parse(torch.__version__) >= version.parse("2.1.2"):
|
| 57 |
+
best_fit_attn_implementation = "sdpa"
|
| 58 |
+
else:
|
| 59 |
+
best_fit_attn_implementation = "eager"
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@register_model("longva")
|
| 63 |
+
class LongVA(lmms):
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
pretrained: str = "lmms-lab/LongVA-7B",
|
| 67 |
+
truncation: Optional[bool] = True,
|
| 68 |
+
device: Optional[str] = "cuda:0",
|
| 69 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 70 |
+
model_name: Optional[str] = None,
|
| 71 |
+
attn_implementation: Optional[str] = best_fit_attn_implementation,
|
| 72 |
+
device_map: Optional[str] = "cuda:0",
|
| 73 |
+
conv_template: Optional[str] = "vicuna_v1",
|
| 74 |
+
use_cache: Optional[bool] = True,
|
| 75 |
+
truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 76 |
+
customized_config: Optional[str] = None, # ends in json
|
| 77 |
+
max_frames_num: Optional[int] = 32,
|
| 78 |
+
mm_spatial_pool_stride: Optional[int] = 2,
|
| 79 |
+
mm_spatial_pool_mode: Optional[str] = "average",
|
| 80 |
+
token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
|
| 81 |
+
video_decode_backend: str = "pyav",
|
| 82 |
+
**kwargs,
|
| 83 |
+
) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
# Do not use kwargs for now
|
| 86 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 87 |
+
|
| 88 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 89 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 90 |
+
self.accelerator = accelerator
|
| 91 |
+
if accelerator.num_processes > 1:
|
| 92 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 93 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 94 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 95 |
+
self._device = torch.device(device)
|
| 96 |
+
self.device_map = device_map
|
| 97 |
+
else:
|
| 98 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 99 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 100 |
+
|
| 101 |
+
llava_model_args = {
|
| 102 |
+
"multimodal": True,
|
| 103 |
+
}
|
| 104 |
+
if customized_config is not None:
|
| 105 |
+
llava_model_args["customized_config"] = customized_config
|
| 106 |
+
if attn_implementation is not None:
|
| 107 |
+
llava_model_args["attn_implementation"] = attn_implementation
|
| 108 |
+
if "use_flash_attention_2" in kwargs:
|
| 109 |
+
llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
|
| 110 |
+
model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
|
| 111 |
+
|
| 112 |
+
self.pretrained = pretrained
|
| 113 |
+
self.token_strategy = token_strategy
|
| 114 |
+
self.max_frames_num = max_frames_num
|
| 115 |
+
self.mm_spatial_pool_stride = mm_spatial_pool_stride
|
| 116 |
+
self.mm_spatial_pool_mode = mm_spatial_pool_mode
|
| 117 |
+
self.video_decode_backend = video_decode_backend
|
| 118 |
+
|
| 119 |
+
overwrite_config = {}
|
| 120 |
+
overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
|
| 121 |
+
overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
|
| 122 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 123 |
+
|
| 124 |
+
llava_model_args["overwrite_config"] = overwrite_config
|
| 125 |
+
try:
|
| 126 |
+
# Try to load the model with the multimodal argument
|
| 127 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 128 |
+
except TypeError:
|
| 129 |
+
# for older versions of LLaVA that don't have multimodal argument
|
| 130 |
+
llava_model_args.pop("multimodal", None)
|
| 131 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
|
| 132 |
+
|
| 133 |
+
self._config = self._model.config
|
| 134 |
+
self.model.eval()
|
| 135 |
+
self.model.tie_weights()
|
| 136 |
+
self.truncation = truncation
|
| 137 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 138 |
+
self.conv_template = conv_template
|
| 139 |
+
self.use_cache = use_cache
|
| 140 |
+
self.truncate_context = truncate_context
|
| 141 |
+
assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 142 |
+
|
| 143 |
+
if accelerator.num_processes > 1:
|
| 144 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 145 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 146 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 147 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 148 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 149 |
+
kwargs = {
|
| 150 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 151 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 152 |
+
}
|
| 153 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 154 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 155 |
+
|
| 156 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 157 |
+
self._model = accelerator.prepare(self.model)
|
| 158 |
+
else:
|
| 159 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 160 |
+
self.accelerator = accelerator
|
| 161 |
+
if self.accelerator.is_local_main_process:
|
| 162 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 163 |
+
self._rank = self.accelerator.local_process_index
|
| 164 |
+
self._world_size = self.accelerator.num_processes
|
| 165 |
+
|
| 166 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 167 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 168 |
+
self._rank = 0
|
| 169 |
+
self._world_size = 1
|
| 170 |
+
|
| 171 |
+
else:
|
| 172 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 173 |
+
self.model.to(self._device)
|
| 174 |
+
self._rank = 0
|
| 175 |
+
self._world_size = 1
|
| 176 |
+
|
| 177 |
+
@property
|
| 178 |
+
def config(self):
|
| 179 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 180 |
+
return self._config
|
| 181 |
+
|
| 182 |
+
@property
|
| 183 |
+
def tokenizer(self):
|
| 184 |
+
return self._tokenizer
|
| 185 |
+
|
| 186 |
+
@property
|
| 187 |
+
def model(self):
|
| 188 |
+
# returns the model, unwrapping it if using Accelerate
|
| 189 |
+
if hasattr(self, "accelerator"):
|
| 190 |
+
return self.accelerator.unwrap_model(self._model)
|
| 191 |
+
else:
|
| 192 |
+
return self._model
|
| 193 |
+
|
| 194 |
+
@property
|
| 195 |
+
def eot_token_id(self):
|
| 196 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 197 |
+
return self.tokenizer.eos_token_id
|
| 198 |
+
|
| 199 |
+
@property
|
| 200 |
+
def max_length(self):
|
| 201 |
+
return self._max_length
|
| 202 |
+
|
| 203 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 204 |
+
if self.tokenizer.padding_side == "left":
|
| 205 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 206 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 207 |
+
if self.tokenizer.padding_side == "left":
|
| 208 |
+
input_ids = torch.flip(input_ids, [1])
|
| 209 |
+
return input_ids
|
| 210 |
+
|
| 211 |
+
@property
|
| 212 |
+
def batch_size(self):
|
| 213 |
+
return self.batch_size_per_gpu
|
| 214 |
+
|
| 215 |
+
@property
|
| 216 |
+
def device(self):
|
| 217 |
+
return self._device
|
| 218 |
+
|
| 219 |
+
@property
|
| 220 |
+
def rank(self):
|
| 221 |
+
return self._rank
|
| 222 |
+
|
| 223 |
+
@property
|
| 224 |
+
def world_size(self):
|
| 225 |
+
return self._world_size
|
| 226 |
+
|
| 227 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 228 |
+
""" """
|
| 229 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 230 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 231 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 232 |
+
if left_truncate_len:
|
| 233 |
+
encoding = encoding[-left_truncate_len:]
|
| 234 |
+
return encoding
|
| 235 |
+
|
| 236 |
+
def tok_decode(self, tokens):
|
| 237 |
+
try:
|
| 238 |
+
return self.tokenizer.decode(tokens)
|
| 239 |
+
except:
|
| 240 |
+
return self.tokenizer.decode([tokens])
|
| 241 |
+
|
| 242 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 243 |
+
# TODO
|
| 244 |
+
res = []
|
| 245 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 246 |
+
|
| 247 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 248 |
+
# encode, pad, and truncate contexts for this batch
|
| 249 |
+
if type(doc_to_target) == str:
|
| 250 |
+
continuation = doc_to_target
|
| 251 |
+
else:
|
| 252 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 253 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 254 |
+
visuals = self.flatten(visuals)
|
| 255 |
+
image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
|
| 256 |
+
if visuals:
|
| 257 |
+
image = process_images(visuals, self._image_processor, self._config)
|
| 258 |
+
if type(image) is list:
|
| 259 |
+
image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
|
| 260 |
+
else:
|
| 261 |
+
image = image.to(dtype=torch.float16, device=self.device)
|
| 262 |
+
else:
|
| 263 |
+
image = None
|
| 264 |
+
|
| 265 |
+
prompts_input = contexts[0] if isinstance(contexts, list) else contexts
|
| 266 |
+
|
| 267 |
+
if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
|
| 268 |
+
"""
|
| 269 |
+
Three senarios:
|
| 270 |
+
1. No image, and there for, no image token should be added.
|
| 271 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 272 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 273 |
+
"""
|
| 274 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
|
| 275 |
+
image_tokens = " ".join(image_tokens)
|
| 276 |
+
prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts)
|
| 277 |
+
|
| 278 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 279 |
+
if "llama_3" in self.conv_template:
|
| 280 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 281 |
+
else:
|
| 282 |
+
conv = conv_templates[self.conv_template].copy()
|
| 283 |
+
|
| 284 |
+
conv.append_message(conv.roles[0], prompts_input)
|
| 285 |
+
conv.append_message(conv.roles[1], None)
|
| 286 |
+
prompt = conv.get_prompt()
|
| 287 |
+
pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 288 |
+
contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 289 |
+
# Add the answer of the second role
|
| 290 |
+
conv.messages[1][1] = continuation
|
| 291 |
+
|
| 292 |
+
prompt = conv.get_prompt()
|
| 293 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 294 |
+
labels = input_ids.clone()
|
| 295 |
+
# Context part no need to calculate for loss
|
| 296 |
+
labels[0, : contxt_id.shape[1]] = -100
|
| 297 |
+
with torch.inference_mode():
|
| 298 |
+
outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
|
| 299 |
+
loss = outputs["loss"]
|
| 300 |
+
# loss = torch.exp(loss)
|
| 301 |
+
logits = outputs["logits"]
|
| 302 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 303 |
+
cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
|
| 304 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
|
| 305 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 306 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 307 |
+
pbar.update(1)
|
| 308 |
+
|
| 309 |
+
pbar.close()
|
| 310 |
+
return res
|
| 311 |
+
|
| 312 |
+
def flatten(self, input):
|
| 313 |
+
new_list = []
|
| 314 |
+
for i in input:
|
| 315 |
+
for j in i:
|
| 316 |
+
new_list.append(j)
|
| 317 |
+
return new_list
|
| 318 |
+
|
| 319 |
+
def load_video(self, video_path, max_frames_num):
|
| 320 |
+
if type(video_path) == str:
|
| 321 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 322 |
+
else:
|
| 323 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 324 |
+
total_frame_num = len(vr)
|
| 325 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 326 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 327 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 328 |
+
return spare_frames # (frames, height, width, channels)
|
| 329 |
+
|
| 330 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 331 |
+
res = []
|
| 332 |
+
|
| 333 |
+
def _collate(x):
|
| 334 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 335 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 336 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 337 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 338 |
+
# automatic adaptive batches much much easier to implement
|
| 339 |
+
# - any OOMs will happen right away rather than near the end
|
| 340 |
+
toks = self.tok_encode(x[0])
|
| 341 |
+
return -len(toks), x[0]
|
| 342 |
+
|
| 343 |
+
# we group requests by their generation_kwargs,
|
| 344 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 345 |
+
# in the same batch.
|
| 346 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 347 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 348 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 349 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 350 |
+
for chunk in chunks:
|
| 351 |
+
batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
|
| 352 |
+
task = batched_task[0]
|
| 353 |
+
split = batched_split[0]
|
| 354 |
+
batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
|
| 355 |
+
flattened_visuals = self.flatten(batched_visuals) # [B*N]
|
| 356 |
+
assert len(batched_visuals) == 1
|
| 357 |
+
|
| 358 |
+
# we assume all gen kwargs in the batch are the same
|
| 359 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 360 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 361 |
+
if "until" in gen_kwargs:
|
| 362 |
+
gen_kwargs.pop("until")
|
| 363 |
+
|
| 364 |
+
question_input = []
|
| 365 |
+
|
| 366 |
+
for visual, context in zip(batched_visuals, batched_contexts):
|
| 367 |
+
if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
|
| 368 |
+
# here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
|
| 369 |
+
self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
|
| 370 |
+
eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
|
| 371 |
+
|
| 372 |
+
# encode, pad, and truncate contexts for this batch
|
| 373 |
+
if type(visual[0]) == PIL.Image.Image: # For image task
|
| 374 |
+
image_tensor = process_images(visual, self._image_processor, self._config)
|
| 375 |
+
if type(image_tensor) is list:
|
| 376 |
+
image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
|
| 377 |
+
else:
|
| 378 |
+
image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
|
| 379 |
+
|
| 380 |
+
task_type = "image"
|
| 381 |
+
|
| 382 |
+
elif type(visual[0]) == str: # For video task
|
| 383 |
+
image_tensor = []
|
| 384 |
+
try:
|
| 385 |
+
if self.video_decode_backend == "decord":
|
| 386 |
+
frames = self.load_video(visual, self.max_frames_num)
|
| 387 |
+
elif self.video_decode_backend == "pyav":
|
| 388 |
+
frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
|
| 389 |
+
frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
|
| 390 |
+
image_tensor.append(frames)
|
| 391 |
+
except Exception as e:
|
| 392 |
+
eval_logger.error(f"Error {e} in loading video")
|
| 393 |
+
image_tensor = None
|
| 394 |
+
|
| 395 |
+
task_type = "video"
|
| 396 |
+
|
| 397 |
+
if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
|
| 398 |
+
"""
|
| 399 |
+
Three senarios:
|
| 400 |
+
1. No image, and there for, no image token should be added.
|
| 401 |
+
2. image token is already specified in the context, so we don't need to add it.
|
| 402 |
+
3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
|
| 403 |
+
4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
|
| 404 |
+
"""
|
| 405 |
+
if task_type == "image":
|
| 406 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
|
| 407 |
+
elif task_type == "video":
|
| 408 |
+
image_tokens = [DEFAULT_IMAGE_TOKEN] * len(frames) if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
|
| 409 |
+
|
| 410 |
+
image_tokens = " ".join(image_tokens)
|
| 411 |
+
question = image_tokens + "\n" + context
|
| 412 |
+
else:
|
| 413 |
+
question = context
|
| 414 |
+
|
| 415 |
+
# This is much safer for llama3, as we now have some object type in it
|
| 416 |
+
if "llama_3" in self.conv_template:
|
| 417 |
+
conv = copy.deepcopy(conv_templates[self.conv_template])
|
| 418 |
+
else:
|
| 419 |
+
conv = conv_templates[self.conv_template].copy()
|
| 420 |
+
conv.append_message(conv.roles[0], question)
|
| 421 |
+
conv.append_message(conv.roles[1], None)
|
| 422 |
+
prompt_question = conv.get_prompt()
|
| 423 |
+
question_input.append(prompt_question)
|
| 424 |
+
|
| 425 |
+
# preconfigure gen_kwargs with defaults
|
| 426 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 427 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 428 |
+
if "temperature" not in gen_kwargs:
|
| 429 |
+
gen_kwargs["temperature"] = 0
|
| 430 |
+
if "do_sample" not in gen_kwargs:
|
| 431 |
+
gen_kwargs["do_sample"] = False
|
| 432 |
+
if "top_p" not in gen_kwargs:
|
| 433 |
+
gen_kwargs["top_p"] = None
|
| 434 |
+
if "num_beams" not in gen_kwargs:
|
| 435 |
+
gen_kwargs["num_beams"] = 1
|
| 436 |
+
|
| 437 |
+
input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
|
| 438 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 439 |
+
input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
|
| 440 |
+
attention_masks = input_ids.ne(pad_token_ids).to(self.device)
|
| 441 |
+
|
| 442 |
+
if task_type == "image":
|
| 443 |
+
gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
|
| 444 |
+
elif task_type == "video":
|
| 445 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 446 |
+
keywords = [stop_str]
|
| 447 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 448 |
+
gen_kwargs["modalities"] = ["video"]
|
| 449 |
+
gen_kwargs["stopping_criteria"] = [stopping_criteria]
|
| 450 |
+
self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
|
| 451 |
+
self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
|
| 452 |
+
|
| 453 |
+
# These steps are not in LLaVA's original code, but are necessary for generation to work
|
| 454 |
+
# TODO: attention to this major generation step...
|
| 455 |
+
if "image_aspect_ratio" in gen_kwargs.keys():
|
| 456 |
+
gen_kwargs.pop("image_aspect_ratio")
|
| 457 |
+
try:
|
| 458 |
+
with torch.inference_mode():
|
| 459 |
+
cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
|
| 460 |
+
|
| 461 |
+
text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
|
| 462 |
+
except Exception as e:
|
| 463 |
+
raise e
|
| 464 |
+
|
| 465 |
+
text_outputs = [response.strip() for response in text_outputs]
|
| 466 |
+
res.extend(text_outputs)
|
| 467 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 468 |
+
pbar.update(1)
|
| 469 |
+
# reorder this group of results back to original unsorted form
|
| 470 |
+
res = re_ords.get_original(res)
|
| 471 |
+
|
| 472 |
+
pbar.close()
|
| 473 |
+
return res
|
| 474 |
+
|
| 475 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 476 |
+
raise NotImplementedError("TODO: Implement multi-round generation for LongVA")
|
LaViDa/eval/lmms_eval/models/mantis.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
import copy
|
| 7 |
+
import warnings
|
| 8 |
+
from datetime import timedelta
|
| 9 |
+
from typing import List, Optional, Tuple, Union
|
| 10 |
+
|
| 11 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 12 |
+
from accelerate.state import AcceleratorState
|
| 13 |
+
from loguru import logger as eval_logger
|
| 14 |
+
from packaging import version
|
| 15 |
+
from tqdm import tqdm
|
| 16 |
+
|
| 17 |
+
from lmms_eval import utils
|
| 18 |
+
from lmms_eval.api.instance import Instance
|
| 19 |
+
from lmms_eval.api.model import lmms
|
| 20 |
+
from lmms_eval.api.registry import register_model
|
| 21 |
+
from lmms_eval.utils import stop_sequences_criteria
|
| 22 |
+
|
| 23 |
+
warnings.filterwarnings("ignore")
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
from mantis.models.conversation import conv_mllava_v1 as default_conv
|
| 27 |
+
from mantis.models.conversation import conv_templates
|
| 28 |
+
from mantis.models.mfuyu import MFuyuForCausalLM, MFuyuProcessor
|
| 29 |
+
from mantis.models.mllava import LlavaForConditionalGeneration, MLlavaProcessor
|
| 30 |
+
|
| 31 |
+
except Exception as e:
|
| 32 |
+
eval_logger.debug("Mantis is not installed. Please install Mantis to use this model.\nError: %s" % e)
|
| 33 |
+
|
| 34 |
+
try:
|
| 35 |
+
from transformers import AutoModelForVision2Seq, AutoProcessor
|
| 36 |
+
except Exception as e:
|
| 37 |
+
eval_logger.debug("Upgrade transformers to use Mantis's idefics model.\nError: %s" % e)
|
| 38 |
+
|
| 39 |
+
# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 40 |
+
# if is_flash_attn_2_available:
|
| 41 |
+
# best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
|
| 42 |
+
|
| 43 |
+
try:
|
| 44 |
+
import flash_attn
|
| 45 |
+
|
| 46 |
+
best_fit_attn_implementation = "flash_attention_2"
|
| 47 |
+
except ImportError:
|
| 48 |
+
best_fit_attn_implementation = "eager"
|
| 49 |
+
|
| 50 |
+
DEFAULT_IMAGE_TOKEN = "<image>"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@register_model("mantis")
|
| 54 |
+
class Mantis(lmms):
|
| 55 |
+
"""
|
| 56 |
+
Mantis Model
|
| 57 |
+
This implementation is adpated from the Llava model from llava.py and the Idefics model from idefics.py
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
def __init__(
|
| 61 |
+
self,
|
| 62 |
+
pretrained: str = "TIGER-Lab/Mantis-8B-siglip-llama3",
|
| 63 |
+
truncation: Optional[bool] = True,
|
| 64 |
+
device: Optional[str] = "cuda:0",
|
| 65 |
+
dtype: Optional[Union[str, torch.dtype]] = "float16",
|
| 66 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 67 |
+
attn_implementation=best_fit_attn_implementation,
|
| 68 |
+
device_map="cuda:0",
|
| 69 |
+
use_cache=True,
|
| 70 |
+
truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
|
| 71 |
+
**kwargs,
|
| 72 |
+
) -> None:
|
| 73 |
+
super().__init__()
|
| 74 |
+
# Do not use kwargs for now
|
| 75 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 76 |
+
|
| 77 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 78 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 79 |
+
if accelerator.num_processes > 1:
|
| 80 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 81 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 82 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 83 |
+
self._device = torch.device(device)
|
| 84 |
+
self.device_map = device_map
|
| 85 |
+
else:
|
| 86 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 87 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 88 |
+
|
| 89 |
+
self._is_idefics = "idefics" in pretrained.lower()
|
| 90 |
+
if isinstance(dtype, str) and dtype != "auto":
|
| 91 |
+
dtype = getattr(torch, dtype)
|
| 92 |
+
|
| 93 |
+
# Here we load the "non-idefics" Mantis model.
|
| 94 |
+
if not self._is_idefics:
|
| 95 |
+
if "fuyu" in pretrained.lower():
|
| 96 |
+
self._processor = MFuyuProcessor.from_pretrained(pretrained)
|
| 97 |
+
self._model = MFuyuForCausalLM.from_pretrained(pretrained, device_map=self.device_map, attn_implementation=attn_implementation, torch_dtype=dtype)
|
| 98 |
+
else:
|
| 99 |
+
self._processor = MLlavaProcessor.from_pretrained(pretrained)
|
| 100 |
+
self._model = LlavaForConditionalGeneration.from_pretrained(pretrained, device_map=self.device_map, attn_implementation=attn_implementation, torch_dtype=dtype)
|
| 101 |
+
|
| 102 |
+
else:
|
| 103 |
+
self._processor = AutoProcessor.from_pretrained(pretrained)
|
| 104 |
+
self._model = AutoModelForVision2Seq.from_pretrained(pretrained, device_map=self.device_map, torch_dtype=dtype)
|
| 105 |
+
eval_logger.info(f"Using {type(self._model)} to instantiate the Mantis model.")
|
| 106 |
+
|
| 107 |
+
self._tokenizer = self._processor.tokenizer
|
| 108 |
+
|
| 109 |
+
self._config = self._model.config
|
| 110 |
+
self.model.eval()
|
| 111 |
+
self.model.tie_weights()
|
| 112 |
+
self.truncation = truncation
|
| 113 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 114 |
+
self.use_cache = use_cache
|
| 115 |
+
self.truncate_context = truncate_context
|
| 116 |
+
|
| 117 |
+
if accelerator.num_processes > 1:
|
| 118 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 119 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 120 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 121 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 122 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 123 |
+
kwargs = {
|
| 124 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 125 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 126 |
+
}
|
| 127 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 128 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 129 |
+
|
| 130 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 131 |
+
self._model = accelerator.prepare(self.model)
|
| 132 |
+
else:
|
| 133 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 134 |
+
self.accelerator = accelerator
|
| 135 |
+
if self.accelerator.is_local_main_process:
|
| 136 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 137 |
+
self._rank = self.accelerator.local_process_index
|
| 138 |
+
self._world_size = self.accelerator.num_processes
|
| 139 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 140 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 141 |
+
self._rank = 0
|
| 142 |
+
self._world_size = 1
|
| 143 |
+
else:
|
| 144 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 145 |
+
self.model.to(self._device)
|
| 146 |
+
self._rank = 0
|
| 147 |
+
self._world_size = 1
|
| 148 |
+
|
| 149 |
+
@property
|
| 150 |
+
def config(self):
|
| 151 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 152 |
+
return self._config
|
| 153 |
+
|
| 154 |
+
@property
|
| 155 |
+
def tokenizer(self):
|
| 156 |
+
return self._tokenizer
|
| 157 |
+
|
| 158 |
+
@property
|
| 159 |
+
def model(self):
|
| 160 |
+
# returns the model, unwrapping it if using Accelerate
|
| 161 |
+
if hasattr(self, "accelerator"):
|
| 162 |
+
return self.accelerator.unwrap_model(self._model)
|
| 163 |
+
else:
|
| 164 |
+
return self._model
|
| 165 |
+
|
| 166 |
+
@property
|
| 167 |
+
def eot_token_id(self):
|
| 168 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 169 |
+
return self.tokenizer.eos_token_id
|
| 170 |
+
|
| 171 |
+
@property
|
| 172 |
+
def max_length(self):
|
| 173 |
+
return self._max_length
|
| 174 |
+
|
| 175 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 176 |
+
if self.tokenizer.padding_side == "left":
|
| 177 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 178 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 179 |
+
if self.tokenizer.padding_side == "left":
|
| 180 |
+
input_ids = torch.flip(input_ids, [1])
|
| 181 |
+
return input_ids
|
| 182 |
+
|
| 183 |
+
@property
|
| 184 |
+
def batch_size(self):
|
| 185 |
+
return self.batch_size_per_gpu
|
| 186 |
+
|
| 187 |
+
@property
|
| 188 |
+
def device(self):
|
| 189 |
+
return self._device
|
| 190 |
+
|
| 191 |
+
@property
|
| 192 |
+
def rank(self):
|
| 193 |
+
return self._rank
|
| 194 |
+
|
| 195 |
+
@property
|
| 196 |
+
def world_size(self):
|
| 197 |
+
return self._world_size
|
| 198 |
+
|
| 199 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 200 |
+
""" """
|
| 201 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 202 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 203 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 204 |
+
if left_truncate_len:
|
| 205 |
+
encoding = encoding[-left_truncate_len:]
|
| 206 |
+
return encoding
|
| 207 |
+
|
| 208 |
+
def tok_decode(self, tokens):
|
| 209 |
+
try:
|
| 210 |
+
return self.tokenizer.decode(tokens)
|
| 211 |
+
except:
|
| 212 |
+
return self.tokenizer.decode([tokens])
|
| 213 |
+
|
| 214 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 215 |
+
raise NotImplementedError
|
| 216 |
+
|
| 217 |
+
def flatten(self, input):
|
| 218 |
+
new_list = []
|
| 219 |
+
for i in input:
|
| 220 |
+
for j in i:
|
| 221 |
+
new_list.append(j)
|
| 222 |
+
return new_list
|
| 223 |
+
|
| 224 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 225 |
+
res = []
|
| 226 |
+
|
| 227 |
+
def _collate(x):
|
| 228 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 229 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 230 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 231 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 232 |
+
# automatic adaptive batches much much easier to implement
|
| 233 |
+
# - any OOMs will happen right away rather than near the end
|
| 234 |
+
toks = self.tok_encode(x[0])
|
| 235 |
+
return -len(toks), x[0]
|
| 236 |
+
|
| 237 |
+
# we group requests by their generation_kwargs,
|
| 238 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 239 |
+
# in the same batch.
|
| 240 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 241 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 242 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 243 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 244 |
+
for chunk in chunks:
|
| 245 |
+
contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
|
| 246 |
+
visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
|
| 247 |
+
|
| 248 |
+
# we assume all gen kwargs in the batch are the same
|
| 249 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 250 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 251 |
+
|
| 252 |
+
until = gen_kwargs.pop("until", None)
|
| 253 |
+
image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio", None)
|
| 254 |
+
|
| 255 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 256 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 257 |
+
if "temperature" not in gen_kwargs:
|
| 258 |
+
gen_kwargs["temperature"] = 0
|
| 259 |
+
|
| 260 |
+
# prompts_input = contexts[0]
|
| 261 |
+
|
| 262 |
+
prompts = []
|
| 263 |
+
for visual, context in zip(visuals, contexts):
|
| 264 |
+
if self._is_idefics:
|
| 265 |
+
# Follow the idefics implementation:
|
| 266 |
+
content = []
|
| 267 |
+
if DEFAULT_IMAGE_TOKEN not in context:
|
| 268 |
+
for _ in visual:
|
| 269 |
+
content.append({"type": "image"})
|
| 270 |
+
content.append({"type": "text", "text": context})
|
| 271 |
+
message = [{"role": "user", "content": content}]
|
| 272 |
+
prompt = self._processor.apply_chat_template(message, add_generation_prompt=True)
|
| 273 |
+
prompts.append(prompt)
|
| 274 |
+
else:
|
| 275 |
+
# We follow the Mantis code base: https://github.com/TIGER-AI-Lab/Mantis/blob/main/mantis/models/mllava/utils.py#L33 to make sure they are consistent
|
| 276 |
+
# Users don't need to define chat template as it is done here
|
| 277 |
+
if "llama-3" in self._model.language_model.name_or_path.lower():
|
| 278 |
+
conv = conv_templates["llama_3"]
|
| 279 |
+
terminators = [self._processor.tokenizer.eos_token_id, self._processor.tokenizer.convert_tokens_to_ids("<|eot_id|>")]
|
| 280 |
+
else:
|
| 281 |
+
conv = default_conv
|
| 282 |
+
terminators = None
|
| 283 |
+
|
| 284 |
+
gen_kwargs["eos_token_id"] = terminators
|
| 285 |
+
|
| 286 |
+
conv = conv.copy()
|
| 287 |
+
conv.append_message(conv.roles[0], context)
|
| 288 |
+
conv.append_message(conv.roles[1], "")
|
| 289 |
+
prompt = conv.get_prompt()
|
| 290 |
+
prompts.append(prompt)
|
| 291 |
+
inputs = self._processor(images=visuals, text=prompts, return_tensors="pt", truncation=True)
|
| 292 |
+
if "image_patches" in inputs.keys():
|
| 293 |
+
inputs["image_patches"] = inputs["image_patches"][0] # FIXME: Fuyu model would return a list instead of a pytorch tensor. This weird behavior needs fixing.
|
| 294 |
+
inputs = {k: v.to(self.device) for k, v in inputs.items()}
|
| 295 |
+
|
| 296 |
+
output_ids = self.model.generate(**inputs, **gen_kwargs)
|
| 297 |
+
for output_id, input_id in zip(output_ids, inputs["input_ids"]):
|
| 298 |
+
generated_id = output_id[len(input_id) :]
|
| 299 |
+
generated_text = self.tokenizer.decode(generated_id, skip_special_tokens=True)
|
| 300 |
+
|
| 301 |
+
res.append(generated_text)
|
| 302 |
+
|
| 303 |
+
# self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 304 |
+
pbar.update(1)
|
| 305 |
+
# reorder this group of results back to original unsorted form
|
| 306 |
+
res = re_ords.get_original(res)
|
| 307 |
+
|
| 308 |
+
pbar.close()
|
| 309 |
+
return res
|
| 310 |
+
|
| 311 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 312 |
+
raise NotImplementedError("TODO: Implement multi-round generation for Mantis")
|
LaViDa/eval/lmms_eval/models/minimonkey.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from accelerate import Accelerator, DistributedType
|
| 6 |
+
from accelerate.state import AcceleratorState
|
| 7 |
+
from tqdm import tqdm
|
| 8 |
+
from transformers import AutoModel, AutoTokenizer
|
| 9 |
+
|
| 10 |
+
from lmms_eval import utils
|
| 11 |
+
from lmms_eval.api.instance import Instance
|
| 12 |
+
from lmms_eval.api.model import lmms
|
| 13 |
+
from lmms_eval.api.registry import register_model
|
| 14 |
+
|
| 15 |
+
warnings.filterwarnings("ignore")
|
| 16 |
+
|
| 17 |
+
from loguru import logger as eval_logger
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@register_model("minimonkey")
|
| 21 |
+
class MiniMonkey(lmms):
|
| 22 |
+
"""
|
| 23 |
+
MiniMonkey Model
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
def __init__(
|
| 27 |
+
self,
|
| 28 |
+
pretrained: str = "mx262/MiniMonkey",
|
| 29 |
+
device: Optional[str] = "cuda",
|
| 30 |
+
dtype: Optional[Union[str, torch.dtype]] = torch.bfloat16,
|
| 31 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 32 |
+
trust_remote_code: Optional[bool] = True,
|
| 33 |
+
**kwargs,
|
| 34 |
+
) -> None:
|
| 35 |
+
super().__init__()
|
| 36 |
+
# Do not use kwargs for now
|
| 37 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 38 |
+
|
| 39 |
+
accelerator = Accelerator()
|
| 40 |
+
if accelerator.num_processes > 1:
|
| 41 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 42 |
+
else:
|
| 43 |
+
self._device = device
|
| 44 |
+
self.dtype = dtype
|
| 45 |
+
self._model = AutoModel.from_pretrained(pretrained, trust_remote_code=trust_remote_code, torch_dtype=dtype, device_map=self._device)
|
| 46 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code)
|
| 47 |
+
self._config = self._model.config
|
| 48 |
+
self.model.eval()
|
| 49 |
+
self.model.tie_weights()
|
| 50 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 51 |
+
if accelerator.num_processes > 1:
|
| 52 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 53 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 54 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 55 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 56 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 57 |
+
kwargs = {
|
| 58 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 59 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 60 |
+
}
|
| 61 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 62 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 63 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 64 |
+
self._model = accelerator.prepare(self.model)
|
| 65 |
+
else:
|
| 66 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 67 |
+
self.accelerator = accelerator
|
| 68 |
+
if self.accelerator.is_local_main_process:
|
| 69 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 70 |
+
self._rank = self.accelerator.local_process_index
|
| 71 |
+
self._world_size = self.accelerator.num_processes
|
| 72 |
+
else:
|
| 73 |
+
# self.model.to(self._device)
|
| 74 |
+
self._rank = 0
|
| 75 |
+
self._world_size = 1
|
| 76 |
+
|
| 77 |
+
@property
|
| 78 |
+
def config(self):
|
| 79 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 80 |
+
return self._config
|
| 81 |
+
|
| 82 |
+
@property
|
| 83 |
+
def tokenizer(self):
|
| 84 |
+
return self._tokenizer
|
| 85 |
+
|
| 86 |
+
@property
|
| 87 |
+
def model(self):
|
| 88 |
+
# returns the model, unwrapping it if using Accelerate
|
| 89 |
+
if hasattr(self, "accelerator"):
|
| 90 |
+
return self.accelerator.unwrap_model(self._model)
|
| 91 |
+
else:
|
| 92 |
+
return self._model
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def eot_token_id(self):
|
| 96 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 97 |
+
return self.tokenizer.eos_token_id
|
| 98 |
+
|
| 99 |
+
@property
|
| 100 |
+
def max_length(self):
|
| 101 |
+
return self._max_length
|
| 102 |
+
|
| 103 |
+
@property
|
| 104 |
+
def batch_size(self):
|
| 105 |
+
return self.batch_size_per_gpu
|
| 106 |
+
|
| 107 |
+
@property
|
| 108 |
+
def device(self):
|
| 109 |
+
return self._device
|
| 110 |
+
|
| 111 |
+
@property
|
| 112 |
+
def rank(self):
|
| 113 |
+
return self._rank
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def world_size(self):
|
| 117 |
+
return self._world_size
|
| 118 |
+
|
| 119 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 120 |
+
""" """
|
| 121 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 122 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 123 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 124 |
+
if left_truncate_len:
|
| 125 |
+
encoding = encoding[-left_truncate_len:]
|
| 126 |
+
return encoding
|
| 127 |
+
|
| 128 |
+
def tok_decode(self, tokens):
|
| 129 |
+
return self.tokenizer.decode(tokens)
|
| 130 |
+
|
| 131 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 132 |
+
# TODO
|
| 133 |
+
assert False, "We have not implemented this function for MiniMonkey yet"
|
| 134 |
+
|
| 135 |
+
def flatten(self, input):
|
| 136 |
+
new_list = []
|
| 137 |
+
for i in input:
|
| 138 |
+
for j in i:
|
| 139 |
+
new_list.append(j)
|
| 140 |
+
return new_list
|
| 141 |
+
|
| 142 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 143 |
+
res = []
|
| 144 |
+
|
| 145 |
+
def _collate(x):
|
| 146 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 147 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 148 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 149 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 150 |
+
# automatic adaptive batches much much easier to implement
|
| 151 |
+
# - any OOMs will happen right away rather than near the end
|
| 152 |
+
toks = self.tok_encode(x[0])
|
| 153 |
+
return -len(toks), x[0]
|
| 154 |
+
|
| 155 |
+
# we group requests by their generation_kwargs,
|
| 156 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 157 |
+
# in the same batch.
|
| 158 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 159 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 160 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 161 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 162 |
+
for chunk in chunks:
|
| 163 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 164 |
+
task = task[0]
|
| 165 |
+
split = split[0]
|
| 166 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 167 |
+
visuals = self.flatten(visuals)
|
| 168 |
+
# we assume all gen kwargs in the batch are the same
|
| 169 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 170 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 171 |
+
|
| 172 |
+
# Set default values for until and max_new_tokens
|
| 173 |
+
until = [self.tok_decode(self.eot_token_id)]
|
| 174 |
+
|
| 175 |
+
# Update values from gen_kwargs if present
|
| 176 |
+
if "until" in gen_kwargs:
|
| 177 |
+
until = gen_kwargs.pop("until")
|
| 178 |
+
if isinstance(until, str):
|
| 179 |
+
until = [until]
|
| 180 |
+
elif not isinstance(until, list):
|
| 181 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 182 |
+
assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
|
| 183 |
+
assert len(visuals) == 1, "MiniMonkey interface does not support bn_image > 1 for now"
|
| 184 |
+
context = contexts[0]
|
| 185 |
+
if "<image>" in context:
|
| 186 |
+
context = context.replace("<image>", "")
|
| 187 |
+
|
| 188 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 189 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 190 |
+
if "temperature" not in gen_kwargs:
|
| 191 |
+
gen_kwargs["temperature"] = 0
|
| 192 |
+
if "top_p" not in gen_kwargs:
|
| 193 |
+
gen_kwargs["top_p"] = None
|
| 194 |
+
if "num_beams" not in gen_kwargs:
|
| 195 |
+
gen_kwargs["num_beams"] = 1
|
| 196 |
+
|
| 197 |
+
image, prompt = visuals[0], context
|
| 198 |
+
try:
|
| 199 |
+
pixel_values, target_aspect_ratio = load_image(image, min_num=4, max_num=12)
|
| 200 |
+
pixel_values2 = load_image2(image, min_num=3, max_num=7, target_aspect_ratio=target_aspect_ratio)
|
| 201 |
+
pixel_values = torch.cat([pixel_values2[:-1], pixel_values[:-1], pixel_values2[-1:]], 0).to(self._device).to(self.dtype)
|
| 202 |
+
|
| 203 |
+
response, history = self.model.chat(self.tokenizer, pixel_values, target_aspect_ratio, prompt, gen_kwargs, history=None, return_history=True)
|
| 204 |
+
|
| 205 |
+
context = [{"role": "user", "content": prompt}, {"role": "assistant", "content": response}]
|
| 206 |
+
except Exception as e:
|
| 207 |
+
eval_logger.error(f"Error {e} in generating")
|
| 208 |
+
cont = ""
|
| 209 |
+
res.append(response)
|
| 210 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response)
|
| 211 |
+
pbar.update(1)
|
| 212 |
+
# reorder this group of results back to original unsorted form
|
| 213 |
+
res = re_ords.get_original(res)
|
| 214 |
+
|
| 215 |
+
pbar.close()
|
| 216 |
+
return res
|
| 217 |
+
|
| 218 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 219 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
import numpy as np
|
| 223 |
+
import torchvision.transforms as T
|
| 224 |
+
from PIL import Image
|
| 225 |
+
from torchvision.transforms.functional import InterpolationMode
|
| 226 |
+
|
| 227 |
+
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
| 228 |
+
IMAGENET_STD = (0.229, 0.224, 0.225)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def build_transform(input_size):
|
| 232 |
+
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
|
| 233 |
+
transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
|
| 234 |
+
return transform
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
| 238 |
+
best_ratio_diff = float("inf")
|
| 239 |
+
best_ratio = (1, 1)
|
| 240 |
+
area = width * height
|
| 241 |
+
for ratio in target_ratios:
|
| 242 |
+
target_aspect_ratio = ratio[0] / ratio[1]
|
| 243 |
+
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
| 244 |
+
if ratio_diff < best_ratio_diff:
|
| 245 |
+
best_ratio_diff = ratio_diff
|
| 246 |
+
best_ratio = ratio
|
| 247 |
+
elif ratio_diff == best_ratio_diff:
|
| 248 |
+
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
| 249 |
+
best_ratio = ratio
|
| 250 |
+
return best_ratio
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
|
| 254 |
+
orig_width, orig_height = image.size
|
| 255 |
+
aspect_ratio = orig_width / orig_height
|
| 256 |
+
|
| 257 |
+
# calculate the existing image aspect ratio
|
| 258 |
+
target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
|
| 259 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 260 |
+
|
| 261 |
+
# find the closest aspect ratio to the target
|
| 262 |
+
target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
| 263 |
+
|
| 264 |
+
# calculate the target width and height
|
| 265 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 266 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 267 |
+
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
| 268 |
+
|
| 269 |
+
# resize the image
|
| 270 |
+
resized_img = image.resize((target_width, target_height))
|
| 271 |
+
processed_images = []
|
| 272 |
+
for i in range(blocks):
|
| 273 |
+
box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
|
| 274 |
+
# split the image
|
| 275 |
+
split_img = resized_img.crop(box)
|
| 276 |
+
processed_images.append(split_img)
|
| 277 |
+
assert len(processed_images) == blocks
|
| 278 |
+
if use_thumbnail and len(processed_images) != 1:
|
| 279 |
+
thumbnail_img = image.resize((image_size, image_size))
|
| 280 |
+
processed_images.append(thumbnail_img)
|
| 281 |
+
return processed_images, target_aspect_ratio
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def dynamic_preprocess2(image, min_num=1, max_num=12, prior_aspect_ratio=None, image_size=448, use_thumbnail=False):
|
| 285 |
+
orig_width, orig_height = image.size
|
| 286 |
+
aspect_ratio = orig_width / orig_height
|
| 287 |
+
|
| 288 |
+
# calculate the existing image aspect ratio
|
| 289 |
+
target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
|
| 290 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 291 |
+
new_target_ratios = []
|
| 292 |
+
for i in target_ratios:
|
| 293 |
+
if prior_aspect_ratio[0] % i[0] or prior_aspect_ratio[1] % i[1]:
|
| 294 |
+
new_target_ratios.append(i)
|
| 295 |
+
else:
|
| 296 |
+
continue
|
| 297 |
+
# find the closest aspect ratio to the target
|
| 298 |
+
target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, new_target_ratios, orig_width, orig_height, image_size)
|
| 299 |
+
# calculate the target width and height
|
| 300 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 301 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 302 |
+
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
| 303 |
+
|
| 304 |
+
# resize the image
|
| 305 |
+
resized_img = image.resize((target_width, target_height))
|
| 306 |
+
processed_images = []
|
| 307 |
+
for i in range(blocks):
|
| 308 |
+
box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
|
| 309 |
+
# split the image
|
| 310 |
+
split_img = resized_img.crop(box)
|
| 311 |
+
processed_images.append(split_img)
|
| 312 |
+
assert len(processed_images) == blocks
|
| 313 |
+
if use_thumbnail and len(processed_images) != 1:
|
| 314 |
+
thumbnail_img = image.resize((image_size, image_size))
|
| 315 |
+
processed_images.append(thumbnail_img)
|
| 316 |
+
return processed_images
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def load_image(image, input_size=448, min_num=1, max_num=12):
|
| 320 |
+
image = image.convert("RGB")
|
| 321 |
+
transform = build_transform(input_size=input_size)
|
| 322 |
+
images, target_aspect_ratio = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, min_num=min_num, max_num=max_num)
|
| 323 |
+
pixel_values = [transform(image) for image in images]
|
| 324 |
+
pixel_values = torch.stack(pixel_values)
|
| 325 |
+
return pixel_values, target_aspect_ratio
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def load_image2(image, input_size=448, min_num=1, max_num=12, target_aspect_ratio=None):
|
| 329 |
+
image = image.convert("RGB")
|
| 330 |
+
transform = build_transform(input_size=input_size)
|
| 331 |
+
images = dynamic_preprocess2(image, image_size=input_size, use_thumbnail=True, min_num=min_num, max_num=max_num, prior_aspect_ratio=target_aspect_ratio)
|
| 332 |
+
pixel_values = [transform(image) for image in images]
|
| 333 |
+
pixel_values = torch.stack(pixel_values)
|
| 334 |
+
return pixel_values
|
LaViDa/eval/lmms_eval/models/model_utils/__init__.py
ADDED
|
File without changes
|
LaViDa/eval/lmms_eval/models/model_utils/audio_processing.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
from librosa import resample
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def downsample_audio(audio_array: np.ndarray, original_sr: int, target_sr: int) -> np.ndarray:
|
| 6 |
+
audio_resample_array = resample(audio_array, orig_sr=original_sr, target_sr=target_sr)
|
| 7 |
+
return audio_resample_array
|
LaViDa/eval/lmms_eval/models/model_utils/load_video.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
from io import BytesIO
|
| 3 |
+
from random import sample
|
| 4 |
+
from typing import Optional, Tuple, Union
|
| 5 |
+
|
| 6 |
+
import av
|
| 7 |
+
import numpy as np
|
| 8 |
+
from av.codec.context import CodecContext
|
| 9 |
+
from decord import VideoReader, cpu
|
| 10 |
+
from PIL import Image
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def load_video_decord(video_path, max_frames_num):
|
| 14 |
+
if type(video_path) == str:
|
| 15 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 16 |
+
else:
|
| 17 |
+
vr = VideoReader(video_path[0], ctx=cpu(0))
|
| 18 |
+
total_frame_num = len(vr)
|
| 19 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 20 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 21 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 22 |
+
return spare_frames # (frames, height, width, channels)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
# This one is faster
|
| 26 |
+
def record_video_length_stream(container, indices):
|
| 27 |
+
frames = []
|
| 28 |
+
start_index = indices[0]
|
| 29 |
+
end_index = indices[-1]
|
| 30 |
+
for i, frame in enumerate(container.decode(video=0)):
|
| 31 |
+
if i > end_index:
|
| 32 |
+
break
|
| 33 |
+
if i >= start_index and i in indices:
|
| 34 |
+
frames.append(frame)
|
| 35 |
+
return frames
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# This one works for all types of video
|
| 39 |
+
def record_video_length_packet(container):
|
| 40 |
+
frames = []
|
| 41 |
+
# https://github.com/PyAV-Org/PyAV/issues/1269
|
| 42 |
+
# https://www.cnblogs.com/beyond-tester/p/17641872.html
|
| 43 |
+
# context = CodecContext.create("libvpx-vp9", "r")
|
| 44 |
+
for packet in container.demux(video=0):
|
| 45 |
+
for frame in packet.decode():
|
| 46 |
+
frames.append(frame)
|
| 47 |
+
return frames
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def load_video_stream(container, num_frm: int = 8, fps: float = None, force_include_last_frame=False):
|
| 51 |
+
# container = av.open(video_path)
|
| 52 |
+
total_frames = container.streams.video[0].frames
|
| 53 |
+
frame_rate = container.streams.video[0].average_rate
|
| 54 |
+
if fps is not None:
|
| 55 |
+
video_length = total_frames / frame_rate
|
| 56 |
+
num_frm = min(num_frm, int(video_length * fps))
|
| 57 |
+
sampled_frm = min(total_frames, num_frm)
|
| 58 |
+
indices = np.linspace(0, total_frames - 1, sampled_frm, dtype=int)
|
| 59 |
+
if force_include_last_frame:
|
| 60 |
+
last_frame = total_frames - 1
|
| 61 |
+
if last_frame not in indices:
|
| 62 |
+
indices = np.linspace(0, total_frames - 2, sampled_frm - 1, dtype=int)
|
| 63 |
+
indices = np.append(indices, last_frame)
|
| 64 |
+
|
| 65 |
+
return record_video_length_stream(container, indices)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def load_video_packet(container, num_frm: int = 8, fps: float = None):
|
| 69 |
+
frames = record_video_length_packet(container)
|
| 70 |
+
total_frames = len(frames)
|
| 71 |
+
frame_rate = container.streams.video[0].average_rate
|
| 72 |
+
if fps is not None:
|
| 73 |
+
video_length = total_frames / frame_rate
|
| 74 |
+
num_frm = min(num_frm, int(video_length * fps))
|
| 75 |
+
sampled_frm = min(total_frames, num_frm)
|
| 76 |
+
indices = np.linspace(0, total_frames - 1, sampled_frm, dtype=int)
|
| 77 |
+
|
| 78 |
+
# Append the last frame index if not already included
|
| 79 |
+
if total_frames - 1 not in indices:
|
| 80 |
+
indices = np.append(indices, total_frames - 1)
|
| 81 |
+
|
| 82 |
+
return [frames[i] for i in indices]
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def read_video_pyav(video_path: str, *, num_frm: int = 8, fps: float = None, format="rgb24", force_include_last_frame=False) -> np.ndarray:
|
| 86 |
+
"""
|
| 87 |
+
Read video using the PyAV library.
|
| 88 |
+
|
| 89 |
+
Args:
|
| 90 |
+
video_path (str): The path to the video file.
|
| 91 |
+
num_frm (int, optional): The maximum number of frames to extract. Defaults to 8.
|
| 92 |
+
fps (float, optional): The frames per second for extraction. If `None`, the maximum number of frames will be extracted. Defaults to None.
|
| 93 |
+
format (str, optional): The format of the extracted frames. Defaults to "rgb24".
|
| 94 |
+
|
| 95 |
+
Returns:
|
| 96 |
+
np.ndarray: A numpy array containing the extracted frames in RGB format.
|
| 97 |
+
"""
|
| 98 |
+
|
| 99 |
+
container = av.open(video_path)
|
| 100 |
+
|
| 101 |
+
if "webm" not in video_path and "mkv" not in video_path:
|
| 102 |
+
# For mp4, we try loading with stream first
|
| 103 |
+
try:
|
| 104 |
+
frames = load_video_stream(container, num_frm, fps, force_include_last_frame=force_include_last_frame)
|
| 105 |
+
except:
|
| 106 |
+
frames = record_video_length_packet(container)
|
| 107 |
+
else:
|
| 108 |
+
frames = record_video_length_packet(container)
|
| 109 |
+
|
| 110 |
+
return np.stack([x.to_ndarray(format=format) for x in frames])
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def read_video_pyav_pil(video_path: str, *, num_frm: int = 8, fps: float = None, format="rgb24", max_image_size: Optional[Union[Tuple[int, int], int]] = None, resize_strategy: str = "resize", force_include_last_frame=False):
|
| 114 |
+
frames = read_video_pyav(video_path, num_frm=num_frm, fps=fps, format=format, force_include_last_frame=force_include_last_frame)
|
| 115 |
+
pil_frames = []
|
| 116 |
+
for frame in frames:
|
| 117 |
+
img = Image.fromarray(frame)
|
| 118 |
+
if max_image_size:
|
| 119 |
+
if resize_strategy == "resize":
|
| 120 |
+
if isinstance(max_image_size, int):
|
| 121 |
+
max_image_size = (max_image_size, max_image_size)
|
| 122 |
+
img = img.resize(max_image_size)
|
| 123 |
+
elif resize_strategy == "thumbnail":
|
| 124 |
+
img.thumbnail(max_image_size)
|
| 125 |
+
else:
|
| 126 |
+
raise ValueError(f"Unknown resize strategy: {resize_strategy}")
|
| 127 |
+
pil_frames.append(img)
|
| 128 |
+
return pil_frames
|
| 129 |
+
# return [Image.fromarray(frame) for frame in frames]
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def read_video_pyav_base64(video_path: str, *, num_frm: int = 8, fps: Optional[float] = None, format="rgb24", img_format="PNG", max_image_size: Optional[Union[Tuple[int, int], int]] = None, resize_strategy: str = "resize"):
|
| 133 |
+
frames = read_video_pyav(video_path, num_frm=num_frm, fps=fps, format=format)
|
| 134 |
+
base64_frames = []
|
| 135 |
+
for frame in frames:
|
| 136 |
+
img = Image.fromarray(frame)
|
| 137 |
+
if max_image_size:
|
| 138 |
+
if resize_strategy == "resize":
|
| 139 |
+
if isinstance(max_image_size, int):
|
| 140 |
+
max_image_size = (max_image_size, max_image_size)
|
| 141 |
+
img = img.resize(max_image_size)
|
| 142 |
+
elif resize_strategy == "thumbnail":
|
| 143 |
+
img.thumbnail(max_image_size)
|
| 144 |
+
else:
|
| 145 |
+
raise ValueError(f"Unknown resize strategy: {resize_strategy}")
|
| 146 |
+
output_buffer = BytesIO()
|
| 147 |
+
img.save(output_buffer, format=img_format)
|
| 148 |
+
byte_data = output_buffer.getvalue()
|
| 149 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 150 |
+
base64_frames.append(base64_str)
|
| 151 |
+
return base64_frames
|
LaViDa/eval/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba Cloud.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
"""Generation support."""
|
| 7 |
+
import warnings
|
| 8 |
+
|
| 9 |
+
warnings.simplefilter("ignore", category=DeprecationWarning)
|
| 10 |
+
warnings.filterwarnings("ignore")
|
| 11 |
+
|
| 12 |
+
from typing import Iterable, List, Tuple, Union
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from loguru import logger
|
| 18 |
+
from transformers import PreTrainedTokenizer
|
| 19 |
+
from transformers.generation import LogitsProcessor
|
| 20 |
+
|
| 21 |
+
# Types.
|
| 22 |
+
HistoryType = List[Tuple[str, str]]
|
| 23 |
+
TokensType = List[int]
|
| 24 |
+
BatchTokensType = List[List[int]]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType:
|
| 28 |
+
for tokens in batch:
|
| 29 |
+
context_length = len(tokens)
|
| 30 |
+
if context_length < seq_length:
|
| 31 |
+
tokens.extend([pad_id] * (seq_length - context_length))
|
| 32 |
+
return batch
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def get_ltor_masks_and_position_ids(
|
| 36 |
+
data,
|
| 37 |
+
eod_token,
|
| 38 |
+
reset_position_ids,
|
| 39 |
+
reset_attention_mask,
|
| 40 |
+
eod_mask_loss,
|
| 41 |
+
):
|
| 42 |
+
"""Build masks and position id for left to right model."""
|
| 43 |
+
|
| 44 |
+
# Extract batch size and sequence length.
|
| 45 |
+
micro_batch_size, seq_length = data.size()
|
| 46 |
+
|
| 47 |
+
# Attention mask (lower triangular).
|
| 48 |
+
if reset_attention_mask:
|
| 49 |
+
att_mask_batch = micro_batch_size
|
| 50 |
+
else:
|
| 51 |
+
att_mask_batch = 1
|
| 52 |
+
attention_mask = torch.tril(torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)).view(att_mask_batch, 1, seq_length, seq_length)
|
| 53 |
+
|
| 54 |
+
# Loss mask.
|
| 55 |
+
loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device)
|
| 56 |
+
if eod_mask_loss:
|
| 57 |
+
loss_mask[data == eod_token] = 0.0
|
| 58 |
+
|
| 59 |
+
# Position ids.
|
| 60 |
+
position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
|
| 61 |
+
position_ids = position_ids.unsqueeze(0).expand_as(data)
|
| 62 |
+
# We need to clone as the ids will be modifed based on batch index.
|
| 63 |
+
if reset_position_ids:
|
| 64 |
+
position_ids = position_ids.clone()
|
| 65 |
+
|
| 66 |
+
if reset_position_ids or reset_attention_mask:
|
| 67 |
+
# Loop through the batches:
|
| 68 |
+
for b in range(micro_batch_size):
|
| 69 |
+
# Find indecies where EOD token is.
|
| 70 |
+
eod_index = position_ids[b, data[b] == eod_token]
|
| 71 |
+
# Detach indecies from positions if going to modify positions.
|
| 72 |
+
if reset_position_ids:
|
| 73 |
+
eod_index = eod_index.clone()
|
| 74 |
+
|
| 75 |
+
# Loop through EOD indecies:
|
| 76 |
+
prev_index = 0
|
| 77 |
+
for j in range(eod_index.size()[0]):
|
| 78 |
+
i = eod_index[j]
|
| 79 |
+
# Mask attention loss.
|
| 80 |
+
if reset_attention_mask:
|
| 81 |
+
attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0
|
| 82 |
+
# Reset positions.
|
| 83 |
+
if reset_position_ids:
|
| 84 |
+
position_ids[b, (i + 1) :] -= i + 1 - prev_index
|
| 85 |
+
prev_index = i + 1
|
| 86 |
+
|
| 87 |
+
# Convert attention mask to binary:
|
| 88 |
+
attention_mask = attention_mask < 0.5
|
| 89 |
+
|
| 90 |
+
return attention_mask, loss_mask, position_ids
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def get_batch(context_tokens: torch.LongTensor, eod_id: int):
|
| 94 |
+
"""Generate batch from context tokens."""
|
| 95 |
+
# Move to GPU.
|
| 96 |
+
tokens = context_tokens.contiguous().to(context_tokens.device)
|
| 97 |
+
# Get the attention mask and postition ids.
|
| 98 |
+
attention_mask, _, position_ids = get_ltor_masks_and_position_ids(
|
| 99 |
+
tokens,
|
| 100 |
+
eod_id,
|
| 101 |
+
reset_position_ids=False,
|
| 102 |
+
reset_attention_mask=False,
|
| 103 |
+
eod_mask_loss=False,
|
| 104 |
+
)
|
| 105 |
+
return tokens, attention_mask, position_ids
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def get_stop_words_ids(chat_format, tokenizer):
|
| 109 |
+
if chat_format == "raw":
|
| 110 |
+
stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]]
|
| 111 |
+
elif chat_format == "chatml":
|
| 112 |
+
stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]]
|
| 113 |
+
else:
|
| 114 |
+
raise NotImplementedError(f"Unknown chat format {chat_format!r}")
|
| 115 |
+
return stop_words_ids
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def make_context(
|
| 119 |
+
tokenizer: PreTrainedTokenizer,
|
| 120 |
+
query: str,
|
| 121 |
+
history: List[Tuple[str, str]] = None,
|
| 122 |
+
system: str = "",
|
| 123 |
+
max_window_size: int = 6144,
|
| 124 |
+
chat_format: str = "chatml",
|
| 125 |
+
):
|
| 126 |
+
if history is None:
|
| 127 |
+
history = []
|
| 128 |
+
|
| 129 |
+
if chat_format == "chatml":
|
| 130 |
+
im_start, im_end = "<|im_start|>", "<|im_end|>"
|
| 131 |
+
im_start_tokens = [tokenizer.im_start_id]
|
| 132 |
+
im_end_tokens = [tokenizer.im_end_id]
|
| 133 |
+
nl_tokens = tokenizer.encode("\n")
|
| 134 |
+
|
| 135 |
+
def _tokenize_str(role, content):
|
| 136 |
+
return f"{role}\n{content}", tokenizer.encode(role, allowed_special=set(tokenizer.IMAGE_ST)) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST))
|
| 137 |
+
|
| 138 |
+
system_text, system_tokens_part = _tokenize_str("system", system)
|
| 139 |
+
system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
|
| 140 |
+
|
| 141 |
+
raw_text = ""
|
| 142 |
+
context_tokens = []
|
| 143 |
+
|
| 144 |
+
for turn_query, turn_response in reversed(history):
|
| 145 |
+
query_text, query_tokens_part = _tokenize_str("user", turn_query)
|
| 146 |
+
query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
|
| 147 |
+
if turn_response is not None:
|
| 148 |
+
response_text, response_tokens_part = _tokenize_str("assistant", turn_response)
|
| 149 |
+
response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
|
| 150 |
+
|
| 151 |
+
next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
|
| 152 |
+
prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
|
| 153 |
+
else:
|
| 154 |
+
next_context_tokens = nl_tokens + query_tokens + nl_tokens
|
| 155 |
+
prev_chat = f"\n{im_start}{query_text}{im_end}\n"
|
| 156 |
+
|
| 157 |
+
current_context_size = len(system_tokens) + len(next_context_tokens) + len(context_tokens)
|
| 158 |
+
if current_context_size < max_window_size:
|
| 159 |
+
context_tokens = next_context_tokens + context_tokens
|
| 160 |
+
raw_text = prev_chat + raw_text
|
| 161 |
+
else:
|
| 162 |
+
break
|
| 163 |
+
|
| 164 |
+
context_tokens = system_tokens + context_tokens
|
| 165 |
+
raw_text = f"{im_start}{system_text}{im_end}" + raw_text
|
| 166 |
+
context_tokens += nl_tokens + im_start_tokens + _tokenize_str("user", query)[1] + im_end_tokens + nl_tokens + im_start_tokens + tokenizer.encode("assistant") + nl_tokens
|
| 167 |
+
raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
|
| 168 |
+
|
| 169 |
+
elif chat_format == "raw":
|
| 170 |
+
raw_text = query
|
| 171 |
+
context_tokens = tokenizer.encode(raw_text)
|
| 172 |
+
else:
|
| 173 |
+
raise NotImplementedError(f"Unknown chat format {chat_format!r}")
|
| 174 |
+
|
| 175 |
+
return raw_text, context_tokens
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _decode_default(
|
| 179 |
+
tokens: List[int],
|
| 180 |
+
*,
|
| 181 |
+
stop_words: List[str],
|
| 182 |
+
eod_words: List[str],
|
| 183 |
+
tokenizer: PreTrainedTokenizer,
|
| 184 |
+
raw_text_len: int,
|
| 185 |
+
verbose: bool = False,
|
| 186 |
+
return_end_reason: bool = False,
|
| 187 |
+
errors: str = "replace",
|
| 188 |
+
):
|
| 189 |
+
trim_decode_tokens = tokenizer.decode(tokens, errors=errors)[raw_text_len:]
|
| 190 |
+
if verbose:
|
| 191 |
+
print("\nRaw Generate: ", trim_decode_tokens)
|
| 192 |
+
|
| 193 |
+
end_reason = f"Gen length {len(tokens)}"
|
| 194 |
+
for stop_word in stop_words:
|
| 195 |
+
trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
|
| 196 |
+
for eod_word in eod_words:
|
| 197 |
+
if eod_word in trim_decode_tokens:
|
| 198 |
+
end_reason = f"Gen {eod_word!r}"
|
| 199 |
+
trim_decode_tokens = trim_decode_tokens.split(eod_word)[0]
|
| 200 |
+
trim_decode_tokens = trim_decode_tokens.strip()
|
| 201 |
+
if verbose:
|
| 202 |
+
print("\nEnd Reason:", end_reason)
|
| 203 |
+
print("\nGenerate: ", trim_decode_tokens)
|
| 204 |
+
|
| 205 |
+
if return_end_reason:
|
| 206 |
+
return trim_decode_tokens, end_reason
|
| 207 |
+
else:
|
| 208 |
+
return trim_decode_tokens
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _decode_chatml(
|
| 212 |
+
tokens: List[int], *, stop_words: List[str], eod_token_ids: List[int], tokenizer: PreTrainedTokenizer, raw_text_len: int, context_length: int, verbose: bool = False, return_end_reason: bool = False, errors: str = "replace"
|
| 213 |
+
):
|
| 214 |
+
end_reason = f"Gen length {len(tokens)}"
|
| 215 |
+
eod_token_idx = context_length
|
| 216 |
+
for eod_token_idx in range(context_length, len(tokens)):
|
| 217 |
+
if tokens[eod_token_idx] in eod_token_ids:
|
| 218 |
+
end_reason = f"Gen {tokenizer.decode([tokens[eod_token_idx]])!r}"
|
| 219 |
+
break
|
| 220 |
+
|
| 221 |
+
trim_decode_tokens = tokenizer.decode(tokens[:eod_token_idx], errors=errors)[raw_text_len:]
|
| 222 |
+
if verbose:
|
| 223 |
+
print("\nRaw Generate w/o EOD:", tokenizer.decode(tokens, errors=errors)[raw_text_len:])
|
| 224 |
+
print("\nRaw Generate:", trim_decode_tokens)
|
| 225 |
+
print("\nEnd Reason:", end_reason)
|
| 226 |
+
for stop_word in stop_words:
|
| 227 |
+
trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
|
| 228 |
+
trim_decode_tokens = trim_decode_tokens.strip()
|
| 229 |
+
if verbose:
|
| 230 |
+
print("\nGenerate:", trim_decode_tokens)
|
| 231 |
+
|
| 232 |
+
if return_end_reason:
|
| 233 |
+
return trim_decode_tokens, end_reason
|
| 234 |
+
else:
|
| 235 |
+
return trim_decode_tokens
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def decode_tokens(
|
| 239 |
+
tokens: Union[torch.LongTensor, TokensType],
|
| 240 |
+
tokenizer: PreTrainedTokenizer,
|
| 241 |
+
raw_text_len: int,
|
| 242 |
+
context_length: int,
|
| 243 |
+
chat_format: str,
|
| 244 |
+
verbose: bool = False,
|
| 245 |
+
return_end_reason: bool = False,
|
| 246 |
+
errors: str = "replace",
|
| 247 |
+
) -> str:
|
| 248 |
+
if torch.is_tensor(tokens):
|
| 249 |
+
tokens = tokens.cpu().numpy().tolist()
|
| 250 |
+
|
| 251 |
+
if chat_format == "chatml":
|
| 252 |
+
return _decode_chatml(
|
| 253 |
+
tokens,
|
| 254 |
+
stop_words=[],
|
| 255 |
+
eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id],
|
| 256 |
+
tokenizer=tokenizer,
|
| 257 |
+
raw_text_len=raw_text_len,
|
| 258 |
+
context_length=context_length,
|
| 259 |
+
verbose=verbose,
|
| 260 |
+
return_end_reason=return_end_reason,
|
| 261 |
+
errors=errors,
|
| 262 |
+
)
|
| 263 |
+
elif chat_format == "raw":
|
| 264 |
+
return _decode_default(
|
| 265 |
+
tokens,
|
| 266 |
+
stop_words=["<|endoftext|>"],
|
| 267 |
+
eod_words=["<|endoftext|>"],
|
| 268 |
+
tokenizer=tokenizer,
|
| 269 |
+
raw_text_len=raw_text_len,
|
| 270 |
+
verbose=verbose,
|
| 271 |
+
return_end_reason=return_end_reason,
|
| 272 |
+
errors=errors,
|
| 273 |
+
)
|
| 274 |
+
else:
|
| 275 |
+
raise NotImplementedError(f"Unknown chat format {chat_format!r}")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
class StopWordsLogitsProcessor(LogitsProcessor):
|
| 279 |
+
"""
|
| 280 |
+
:class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration.
|
| 281 |
+
Args:
|
| 282 |
+
stop_words_ids (:obj:`List[List[int]]`):
|
| 283 |
+
List of list of token ids of stop ids. In order to get the tokens of the words
|
| 284 |
+
that should not appear in the generated text, use :obj:`tokenizer(bad_word,
|
| 285 |
+
add_prefix_space=True).input_ids`.
|
| 286 |
+
eos_token_id (:obj:`int`):
|
| 287 |
+
The id of the `end-of-sequence` token.
|
| 288 |
+
"""
|
| 289 |
+
|
| 290 |
+
def __init__(self, stop_words_ids: Iterable[Iterable[int]], eos_token_id: int):
|
| 291 |
+
if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0:
|
| 292 |
+
raise ValueError(f"`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}.")
|
| 293 |
+
if any(not isinstance(bad_word_ids, list) for bad_word_ids in stop_words_ids):
|
| 294 |
+
raise ValueError(f"`stop_words_ids` has to be a list of lists, but is {stop_words_ids}.")
|
| 295 |
+
if any(any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in stop_word_ids) for stop_word_ids in stop_words_ids):
|
| 296 |
+
raise ValueError(f"Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}.")
|
| 297 |
+
|
| 298 |
+
self.stop_words_ids = list(filter(lambda bad_token_seq: bad_token_seq != [eos_token_id], stop_words_ids))
|
| 299 |
+
self.eos_token_id = eos_token_id
|
| 300 |
+
for stop_token_seq in self.stop_words_ids:
|
| 301 |
+
assert len(stop_token_seq) > 0, "Stop words token sequences {} cannot have an empty list".format(stop_words_ids)
|
| 302 |
+
|
| 303 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
|
| 304 |
+
stopped_samples = self._calc_stopped_samples(input_ids)
|
| 305 |
+
for i, should_stop in enumerate(stopped_samples):
|
| 306 |
+
if should_stop:
|
| 307 |
+
scores[i, self.eos_token_id] = float(2**15)
|
| 308 |
+
return scores
|
| 309 |
+
|
| 310 |
+
def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool:
|
| 311 |
+
if len(tokens) == 0:
|
| 312 |
+
# if bad word tokens is just one token always ban it
|
| 313 |
+
return True
|
| 314 |
+
elif len(tokens) > len(prev_tokens):
|
| 315 |
+
# if bad word tokens are longer then prev input_ids they can't be equal
|
| 316 |
+
return False
|
| 317 |
+
elif prev_tokens[-len(tokens) :].tolist() == tokens:
|
| 318 |
+
# if tokens match
|
| 319 |
+
return True
|
| 320 |
+
else:
|
| 321 |
+
return False
|
| 322 |
+
|
| 323 |
+
def _calc_stopped_samples(self, prev_input_ids: Iterable[int]) -> Iterable[int]:
|
| 324 |
+
stopped_samples = []
|
| 325 |
+
for prev_input_ids_slice in prev_input_ids:
|
| 326 |
+
match = False
|
| 327 |
+
for stop_token_seq in self.stop_words_ids:
|
| 328 |
+
if self._tokens_match(prev_input_ids_slice, stop_token_seq):
|
| 329 |
+
# if tokens do not match continue
|
| 330 |
+
match = True
|
| 331 |
+
break
|
| 332 |
+
stopped_samples.append(match)
|
| 333 |
+
|
| 334 |
+
return stopped_samples
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float("Inf")):
|
| 338 |
+
"""This function has been mostly taken from huggingface conversational
|
| 339 |
+
ai code at
|
| 340 |
+
https://medium.com/huggingface/how-to-build-a-state-of-the-art-
|
| 341 |
+
conversational-ai-with-transfer-learning-2d818ac26313"""
|
| 342 |
+
|
| 343 |
+
if top_k > 0:
|
| 344 |
+
# Remove all tokens with a probability less than the
|
| 345 |
+
# last token of the top-k
|
| 346 |
+
indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
|
| 347 |
+
logits[indices_to_remove] = filter_value
|
| 348 |
+
|
| 349 |
+
if top_p > 0.0:
|
| 350 |
+
# Cconvert to 1D
|
| 351 |
+
sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
|
| 352 |
+
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
|
| 353 |
+
|
| 354 |
+
# Remove tokens with cumulative probability above the threshold
|
| 355 |
+
sorted_indices_to_remove = cumulative_probs > top_p
|
| 356 |
+
# Shift the indices to the right to keep also the first token
|
| 357 |
+
# above the threshold
|
| 358 |
+
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
|
| 359 |
+
sorted_indices_to_remove[..., 0] = 0
|
| 360 |
+
for i in range(sorted_indices.size(0)):
|
| 361 |
+
indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]]
|
| 362 |
+
logits[i][indices_to_remove] = filter_value
|
| 363 |
+
|
| 364 |
+
return logits
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def switch(val1, val2, boolean):
|
| 368 |
+
boolean = boolean.type_as(val1)
|
| 369 |
+
return (1 - boolean) * val1 + boolean * val2
|
LaViDa/eval/lmms_eval/models/mplug_owl_video.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datetime import timedelta
|
| 2 |
+
from typing import List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 6 |
+
from accelerate.state import AcceleratorState
|
| 7 |
+
from loguru import logger
|
| 8 |
+
from tqdm import tqdm
|
| 9 |
+
from transformers import AutoTokenizer
|
| 10 |
+
|
| 11 |
+
from lmms_eval.api.instance import Instance
|
| 12 |
+
from lmms_eval.api.model import lmms
|
| 13 |
+
from lmms_eval.api.registry import register_model
|
| 14 |
+
from lmms_eval.models.mplug_owl_video.modeling_mplug_owl import (
|
| 15 |
+
MplugOwlForConditionalGeneration,
|
| 16 |
+
)
|
| 17 |
+
from lmms_eval.models.mplug_owl_video.processing_mplug_owl import (
|
| 18 |
+
MplugOwlImageProcessor,
|
| 19 |
+
MplugOwlProcessor,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
eval_logger = logger
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@register_model("mplug_owl_video")
|
| 26 |
+
class mplug_Owl(lmms):
|
| 27 |
+
def __init__(
|
| 28 |
+
self,
|
| 29 |
+
pretrained: str = "MAGAer13/mplug-owl-llama-7b-video",
|
| 30 |
+
device: Optional[str] = "cuda:0",
|
| 31 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 32 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 33 |
+
device_map="cuda:0",
|
| 34 |
+
num_frames: Union[str, int] = 4,
|
| 35 |
+
**kwargs,
|
| 36 |
+
) -> None:
|
| 37 |
+
"""
|
| 38 |
+
Install instructions:
|
| 39 |
+
1. Install lmms-eval
|
| 40 |
+
cd lmms-eval
|
| 41 |
+
pip install -e .;
|
| 42 |
+
2. Install other packages with restricted versions
|
| 43 |
+
pip install av sentencepiece protobuf==3.20 transformers==4.28.1 einops;
|
| 44 |
+
"""
|
| 45 |
+
super().__init__()
|
| 46 |
+
|
| 47 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 48 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 49 |
+
if accelerator.num_processes > 1:
|
| 50 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 51 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 52 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 53 |
+
self._device = torch.device(device)
|
| 54 |
+
self.device_map = device_map
|
| 55 |
+
else:
|
| 56 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 57 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 58 |
+
|
| 59 |
+
# import pdb; pdb.set_trace()
|
| 60 |
+
# This is very slow. Their issue, not mine
|
| 61 |
+
# Also, keep transformers in version 4.28.1
|
| 62 |
+
# They put a Config object inside a config object, this is not acceptable
|
| 63 |
+
# for transformers == 4.39.1, object type not serializable
|
| 64 |
+
# Protobuf needs to be in 3.20.x otherwise error
|
| 65 |
+
# ヽ(`Д´)ノ
|
| 66 |
+
self._model = MplugOwlForConditionalGeneration.from_pretrained(
|
| 67 |
+
pretrained,
|
| 68 |
+
torch_dtype=torch.bfloat16,
|
| 69 |
+
)
|
| 70 |
+
self.image_processor = MplugOwlImageProcessor.from_pretrained(pretrained)
|
| 71 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
|
| 72 |
+
self.processor = MplugOwlProcessor(self.image_processor, self.tokenizer)
|
| 73 |
+
self.model.eval()
|
| 74 |
+
self.batch_size_per_gpu = batch_size
|
| 75 |
+
self.num_frames = num_frames
|
| 76 |
+
|
| 77 |
+
self.model.to(self.device)
|
| 78 |
+
|
| 79 |
+
if accelerator.num_processes > 1:
|
| 80 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 81 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 82 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 83 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 84 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 85 |
+
kwargs = {
|
| 86 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 87 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 88 |
+
}
|
| 89 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 90 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 91 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 92 |
+
self._model = accelerator.prepare(self.model)
|
| 93 |
+
else:
|
| 94 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 95 |
+
self.accelerator = accelerator
|
| 96 |
+
if self.accelerator.is_local_main_process:
|
| 97 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 98 |
+
self._rank = self.accelerator.local_process_index
|
| 99 |
+
self._world_size = self.accelerator.num_processes
|
| 100 |
+
else:
|
| 101 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 102 |
+
self.model.to(self._device)
|
| 103 |
+
self._rank = 0
|
| 104 |
+
self._world_size = 1
|
| 105 |
+
|
| 106 |
+
@property
|
| 107 |
+
def config(self):
|
| 108 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 109 |
+
return self._config
|
| 110 |
+
|
| 111 |
+
@property
|
| 112 |
+
def tokenizer(self):
|
| 113 |
+
return self._tokenizer
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def model(self):
|
| 117 |
+
# returns the model, unwrapping it if using Accelerate
|
| 118 |
+
if hasattr(self, "accelerator"):
|
| 119 |
+
return self.accelerator.unwrap_model(self._model)
|
| 120 |
+
else:
|
| 121 |
+
return self._model
|
| 122 |
+
|
| 123 |
+
@property
|
| 124 |
+
def eot_token_id(self):
|
| 125 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 126 |
+
return self.tokenizer.eos_token_id
|
| 127 |
+
|
| 128 |
+
@property
|
| 129 |
+
def max_length(self):
|
| 130 |
+
return self._max_length
|
| 131 |
+
|
| 132 |
+
@property
|
| 133 |
+
def batch_size(self):
|
| 134 |
+
return self.batch_size_per_gpu
|
| 135 |
+
|
| 136 |
+
@property
|
| 137 |
+
def device(self):
|
| 138 |
+
return self._device
|
| 139 |
+
|
| 140 |
+
@property
|
| 141 |
+
def rank(self):
|
| 142 |
+
return self._rank
|
| 143 |
+
|
| 144 |
+
@property
|
| 145 |
+
def world_size(self):
|
| 146 |
+
return self._world_size
|
| 147 |
+
|
| 148 |
+
def flatten(self, input):
|
| 149 |
+
new_list = []
|
| 150 |
+
for i in input:
|
| 151 |
+
for j in i:
|
| 152 |
+
new_list.append(j)
|
| 153 |
+
return new_list
|
| 154 |
+
|
| 155 |
+
def format_prompt(self, question):
|
| 156 |
+
prompts = [f" <|video|> Question : {question} Answer : "]
|
| 157 |
+
return prompts
|
| 158 |
+
|
| 159 |
+
def generate_until(self, requests) -> List[str]:
|
| 160 |
+
res = []
|
| 161 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 162 |
+
|
| 163 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 164 |
+
# encode, pad, and truncate contexts for this batch
|
| 165 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 166 |
+
visuals = self.flatten(visuals)
|
| 167 |
+
inputs = self.processor(text=self.format_prompt(contexts), videos=visuals, num_frames=self.num_frames, return_tensors="pt")
|
| 168 |
+
pixel_values_videos = inputs["video_pixel_values"]
|
| 169 |
+
if pixel_values_videos.shape[2] != self.num_frames:
|
| 170 |
+
empty_frames = torch.zeros((1, pixel_values_videos.shape[1], self.num_frames - pixel_values_videos.shape[2], *pixel_values_videos.shape[3:]), dtype=pixel_values_videos.dtype)
|
| 171 |
+
pixel_values_videos = torch.cat([pixel_values_videos, empty_frames], dim=2)
|
| 172 |
+
inputs["video_pixel_values"] = pixel_values_videos
|
| 173 |
+
inputs = {k: v.bfloat16() if v.dtype == torch.float else v for k, v in inputs.items()}
|
| 174 |
+
inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
|
| 175 |
+
|
| 176 |
+
if "max_new_tokens" in gen_kwargs:
|
| 177 |
+
gen_kwargs["max_length"] = gen_kwargs["max_new_tokens"]
|
| 178 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 179 |
+
gen_kwargs["max_length"] = 128
|
| 180 |
+
if "do_sample" not in gen_kwargs:
|
| 181 |
+
gen_kwargs["do_sample"] = False
|
| 182 |
+
if "top_k" not in gen_kwargs:
|
| 183 |
+
gen_kwargs["top_k"] = 1
|
| 184 |
+
|
| 185 |
+
generate_kwargs = {"do_sample": gen_kwargs["do_sample"], "top_k": gen_kwargs["top_k"], "max_length": gen_kwargs["max_length"]}
|
| 186 |
+
|
| 187 |
+
with torch.no_grad():
|
| 188 |
+
outputs = self.model.generate(**inputs, **generate_kwargs)
|
| 189 |
+
sentence = self.tokenizer.decode(outputs.tolist()[0], skip_special_tokens=True)
|
| 190 |
+
pbar.update(1)
|
| 191 |
+
res.append(sentence)
|
| 192 |
+
pbar.close()
|
| 193 |
+
return res
|
| 194 |
+
|
| 195 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 196 |
+
return super().loglikelihood(requests)
|
| 197 |
+
|
| 198 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 199 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/mplug_owl_video/__init__.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from typing import TYPE_CHECKING
|
| 15 |
+
|
| 16 |
+
from transformers.utils import (
|
| 17 |
+
OptionalDependencyNotAvailable,
|
| 18 |
+
_LazyModule,
|
| 19 |
+
is_tokenizers_available,
|
| 20 |
+
is_torch_available,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
_import_structure = {
|
| 24 |
+
"configuration_mplug_owl": ["MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP", "MplugOwlConfig"],
|
| 25 |
+
"processing_mplug_owl": ["MplugOwlImageProcessor", "MplugOwlProcessor"],
|
| 26 |
+
"tokenization_mplug_owl": ["MplugOwlTokenizer"],
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
try:
|
| 30 |
+
if not is_tokenizers_available():
|
| 31 |
+
raise OptionalDependencyNotAvailable()
|
| 32 |
+
except OptionalDependencyNotAvailable:
|
| 33 |
+
pass
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
try:
|
| 37 |
+
if not is_torch_available():
|
| 38 |
+
raise OptionalDependencyNotAvailable()
|
| 39 |
+
except OptionalDependencyNotAvailable:
|
| 40 |
+
pass
|
| 41 |
+
else:
|
| 42 |
+
_import_structure["modeling_mplug_owl"] = [
|
| 43 |
+
"MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST",
|
| 44 |
+
"MplugOwlForConditionalGeneration",
|
| 45 |
+
"MplugOwlModel",
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
if TYPE_CHECKING:
|
| 50 |
+
from .configuration_mplug_owl import (
|
| 51 |
+
MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
| 52 |
+
MplugOwlConfig,
|
| 53 |
+
)
|
| 54 |
+
from .tokenization_mplug_owl import MplugOwlTokenizer
|
| 55 |
+
|
| 56 |
+
try:
|
| 57 |
+
if not is_tokenizers_available():
|
| 58 |
+
raise OptionalDependencyNotAvailable()
|
| 59 |
+
except OptionalDependencyNotAvailable:
|
| 60 |
+
pass
|
| 61 |
+
|
| 62 |
+
try:
|
| 63 |
+
if not is_torch_available():
|
| 64 |
+
raise OptionalDependencyNotAvailable()
|
| 65 |
+
except OptionalDependencyNotAvailable:
|
| 66 |
+
pass
|
| 67 |
+
else:
|
| 68 |
+
from .modeling_mplug_owl import (
|
| 69 |
+
MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST,
|
| 70 |
+
MplugOwlForConditionalGeneration,
|
| 71 |
+
MplugOwlModel,
|
| 72 |
+
MplugOwlPreTrainedModel,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
else:
|
| 77 |
+
import sys
|
| 78 |
+
|
| 79 |
+
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
|
| 80 |
+
|
| 81 |
+
from .configuration_mplug_owl import *
|
| 82 |
+
from .modeling_mplug_owl import *
|
| 83 |
+
from .processing_mplug_owl import *
|
| 84 |
+
from .tokenization_mplug_owl import *
|
LaViDa/eval/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2022 x-plug and The HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
""" MplugOwl model configuration """
|
| 16 |
+
import copy
|
| 17 |
+
import os
|
| 18 |
+
from typing import Union
|
| 19 |
+
|
| 20 |
+
from loguru import logger
|
| 21 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 22 |
+
from transformers.models.auto import CONFIG_MAPPING
|
| 23 |
+
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
|
| 24 |
+
|
| 25 |
+
MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
|
| 26 |
+
"MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/config.json",
|
| 27 |
+
# See all MplugOwl models at https://huggingface.co/models?filter=mplug_owl
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class MplugOwlVisionConfig(PretrainedConfig):
|
| 32 |
+
r"""
|
| 33 |
+
This is the configuration class to store the configuration of a [`MplugOwlVisionModel`]. It is used to instantiate a
|
| 34 |
+
mPLUG-Owl vision encoder according to the specified arguments, defining the model architecture. Instantiating a
|
| 35 |
+
configuration defaults will yield a similar configuration to that of the mPLUG-Owl
|
| 36 |
+
[x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b) architecture.
|
| 37 |
+
|
| 38 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 39 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 40 |
+
|
| 41 |
+
Args:
|
| 42 |
+
hidden_size (`int`, *optional*, defaults to 768):
|
| 43 |
+
Dimensionality of the encoder layers and the pooler layer.
|
| 44 |
+
intermediate_size (`int`, *optional*, defaults to 3072):
|
| 45 |
+
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
|
| 46 |
+
num_hidden_layers (`int`, *optional*, defaults to 12):
|
| 47 |
+
Number of hidden layers in the Transformer encoder.
|
| 48 |
+
num_attention_heads (`int`, *optional*, defaults to 12):
|
| 49 |
+
Number of attention heads for each attention layer in the Transformer encoder.
|
| 50 |
+
image_size (`int`, *optional*, defaults to 224):
|
| 51 |
+
The size (resolution) of each image.
|
| 52 |
+
patch_size (`int`, *optional*, defaults to 32):
|
| 53 |
+
The size (resolution) of each patch.
|
| 54 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
|
| 55 |
+
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
|
| 56 |
+
`"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
|
| 57 |
+
layer_norm_eps (`float`, *optional*, defaults to 1e-5):
|
| 58 |
+
The epsilon used by the layer normalization layers.
|
| 59 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 60 |
+
The dropout ratio for the attention probabilities.
|
| 61 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 62 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 63 |
+
initializer_factor (`float`, *optional*, defaults to 1):
|
| 64 |
+
A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
|
| 65 |
+
testing).
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
```"""
|
| 69 |
+
|
| 70 |
+
model_type = "mplug_owl_vision_model"
|
| 71 |
+
|
| 72 |
+
def __init__(
|
| 73 |
+
self,
|
| 74 |
+
hidden_size=1024,
|
| 75 |
+
intermediate_size=4096,
|
| 76 |
+
projection_dim=768,
|
| 77 |
+
num_hidden_layers=24,
|
| 78 |
+
num_attention_heads=16,
|
| 79 |
+
num_channels=3,
|
| 80 |
+
image_size=224,
|
| 81 |
+
patch_size=14,
|
| 82 |
+
hidden_act="quick_gelu",
|
| 83 |
+
layer_norm_eps=1e-6,
|
| 84 |
+
attention_dropout=0.0,
|
| 85 |
+
initializer_range=0.02,
|
| 86 |
+
initializer_factor=1.0,
|
| 87 |
+
use_flash_attn=False,
|
| 88 |
+
**kwargs,
|
| 89 |
+
):
|
| 90 |
+
super().__init__(**kwargs)
|
| 91 |
+
self.hidden_size = hidden_size
|
| 92 |
+
self.intermediate_size = intermediate_size
|
| 93 |
+
self.projection_dim = projection_dim
|
| 94 |
+
self.num_hidden_layers = num_hidden_layers
|
| 95 |
+
self.num_attention_heads = num_attention_heads
|
| 96 |
+
self.num_channels = num_channels
|
| 97 |
+
self.patch_size = patch_size
|
| 98 |
+
self.image_size = image_size
|
| 99 |
+
self.initializer_range = initializer_range
|
| 100 |
+
self.initializer_factor = initializer_factor
|
| 101 |
+
self.attention_dropout = attention_dropout
|
| 102 |
+
self.layer_norm_eps = layer_norm_eps
|
| 103 |
+
self.hidden_act = hidden_act
|
| 104 |
+
self.use_flash_attn = use_flash_attn
|
| 105 |
+
|
| 106 |
+
@classmethod
|
| 107 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
|
| 108 |
+
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
|
| 109 |
+
|
| 110 |
+
# get the vision config dict if we are loading from MplugOwlConfig
|
| 111 |
+
if config_dict.get("model_type") == "mplug-owl":
|
| 112 |
+
config_dict = config_dict["vision_config"]
|
| 113 |
+
|
| 114 |
+
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
|
| 115 |
+
logger.warning(f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")
|
| 116 |
+
|
| 117 |
+
return cls.from_dict(config_dict, **kwargs)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class MplugOwlVisualAbstractorConfig(PretrainedConfig):
|
| 121 |
+
model_type = "mplug_owl_visual_abstract"
|
| 122 |
+
|
| 123 |
+
def __init__(
|
| 124 |
+
self,
|
| 125 |
+
hidden_size=1024, #
|
| 126 |
+
num_hidden_layers=6, #
|
| 127 |
+
num_attention_heads=16, #
|
| 128 |
+
intermediate_size=4096, #
|
| 129 |
+
attention_probs_dropout_prob=0.1, #
|
| 130 |
+
initializer_range=0.02,
|
| 131 |
+
layer_norm_eps=1e-6, #
|
| 132 |
+
encoder_hidden_size=1024, #
|
| 133 |
+
**kwargs,
|
| 134 |
+
):
|
| 135 |
+
super().__init__(**kwargs)
|
| 136 |
+
self.hidden_size = hidden_size
|
| 137 |
+
self.num_hidden_layers = num_hidden_layers
|
| 138 |
+
self.num_attention_heads = num_attention_heads
|
| 139 |
+
self.intermediate_size = intermediate_size
|
| 140 |
+
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
| 141 |
+
self.initializer_range = initializer_range
|
| 142 |
+
self.layer_norm_eps = layer_norm_eps
|
| 143 |
+
self.encoder_hidden_size = encoder_hidden_size
|
| 144 |
+
|
| 145 |
+
@classmethod
|
| 146 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
|
| 147 |
+
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
|
| 148 |
+
|
| 149 |
+
# get the visual_abstractor config dict if we are loading from MplugOwlConfig
|
| 150 |
+
if config_dict.get("model_type") == "mplug-owl":
|
| 151 |
+
config_dict = config_dict["abstractor_config"]
|
| 152 |
+
|
| 153 |
+
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
|
| 154 |
+
logger.warning(f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")
|
| 155 |
+
|
| 156 |
+
return cls.from_dict(config_dict, **kwargs)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
class MplugOwlConfig(PretrainedConfig):
|
| 160 |
+
r"""
|
| 161 |
+
[`MplugOwlConfig`] is the configuration class to store the configuration of a [`MplugOwlForConditionalGeneration`]. It is
|
| 162 |
+
used to instantiate a mPLUG-Owl model according to the specified arguments, defining the vision model, Q-Former model
|
| 163 |
+
and language model configs. Instantiating a configuration with the defaults will yield a similar configuration to
|
| 164 |
+
that of the mPLUG-Owl [x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b) architecture.
|
| 165 |
+
|
| 166 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 167 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 168 |
+
|
| 169 |
+
Args:
|
| 170 |
+
vision_config (`dict`, *optional*):
|
| 171 |
+
Dictionary of configuration options used to initialize [`MplugOwlVisionConfig`].
|
| 172 |
+
visual_abstractor_config (`dict`, *optional*):
|
| 173 |
+
Dictionary of configuration options used to initialize [`MplugOwlVisualAbstractorConfig`].
|
| 174 |
+
text_config (`dict`, *optional*):
|
| 175 |
+
Dictionary of configuration options used to initialize any [`PretrainedConfig`].
|
| 176 |
+
num_query_tokens (`int`, *optional*, defaults to 32):
|
| 177 |
+
The number of query tokens passed through the Transformer.
|
| 178 |
+
|
| 179 |
+
kwargs (*optional*):
|
| 180 |
+
Dictionary of keyword arguments.
|
| 181 |
+
|
| 182 |
+
Example:
|
| 183 |
+
|
| 184 |
+
```python
|
| 185 |
+
>>> from transformers import (
|
| 186 |
+
... MplugOwlVisionConfig,
|
| 187 |
+
... MplugOwlVisualAbstractorConfig,
|
| 188 |
+
... OPTConfig,
|
| 189 |
+
... MplugOwlConfig,
|
| 190 |
+
... MplugOwlForConditionalGeneration,
|
| 191 |
+
... )
|
| 192 |
+
|
| 193 |
+
>>> # Initializing a MplugOwlConfig with x-plug/x_plug-llama-7b style configuration
|
| 194 |
+
>>> configuration = MplugOwlConfig()
|
| 195 |
+
|
| 196 |
+
>>> # Initializing a MplugOwlForConditionalGeneration (with random weights) from the x-plug/x_plug-llama-7b style configuration
|
| 197 |
+
>>> model = MplugOwlForConditionalGeneration(configuration)
|
| 198 |
+
|
| 199 |
+
>>> # Accessing the model configuration
|
| 200 |
+
>>> configuration = model.config
|
| 201 |
+
|
| 202 |
+
>>> # We can also initialize a MplugOwlConfig from a MplugOwlVisionConfig, MplugOwlVisualAbstractorConfig and any PretrainedConfig
|
| 203 |
+
|
| 204 |
+
>>> # Initializing mPLUG-Owl vision, mPLUG-Owl Q-Former and language model configurations
|
| 205 |
+
>>> vision_config = MplugOwlVisionConfig()
|
| 206 |
+
>>> visual_abstractor_config = MplugOwlVisualAbstractorConfig()
|
| 207 |
+
>>> text_config = OPTConfig()
|
| 208 |
+
|
| 209 |
+
>>> config = MplugOwlConfig.from_text_vision_configs(vision_config, visual_abstractor_config, text_config)
|
| 210 |
+
```"""
|
| 211 |
+
|
| 212 |
+
model_type = "mplug-owl"
|
| 213 |
+
is_composition = True
|
| 214 |
+
|
| 215 |
+
def __init__(self, vision_config=None, visual_abstractor_config=None, text_config=None, num_query_tokens=64, **kwargs):
|
| 216 |
+
super().__init__(**kwargs)
|
| 217 |
+
if vision_config is None:
|
| 218 |
+
vision_config = MplugOwlVisionConfig().to_dict()
|
| 219 |
+
logger.info("vision_config is None.")
|
| 220 |
+
|
| 221 |
+
if visual_abstractor_config is None:
|
| 222 |
+
visual_abstractor_config = {}
|
| 223 |
+
logger.info("abstractor_config is None. ")
|
| 224 |
+
|
| 225 |
+
if text_config is None:
|
| 226 |
+
# we use LLAMA 7b by default
|
| 227 |
+
from ..llama.configuration_llama import LlamaConfig
|
| 228 |
+
|
| 229 |
+
text_config = LlamaConfig(pad_token_id=2).to_dict()
|
| 230 |
+
logger.info("text_config is None.")
|
| 231 |
+
|
| 232 |
+
self.vision_config = MplugOwlVisionConfig(**vision_config)
|
| 233 |
+
self.visual_abstractor_config = MplugOwlVisualAbstractorConfig(**visual_abstractor_config)
|
| 234 |
+
# self.visual_abstractor_config.layer_norm_eps = 1e-6
|
| 235 |
+
text_model_type = text_config["model_type"] if "model_type" in text_config else "llama"
|
| 236 |
+
self.text_config = CONFIG_MAPPING[text_model_type](**text_config)
|
| 237 |
+
|
| 238 |
+
self.tie_word_embeddings = self.text_config.tie_word_embeddings
|
| 239 |
+
self.is_encoder_decoder = self.text_config.is_encoder_decoder
|
| 240 |
+
|
| 241 |
+
self.num_query_tokens = num_query_tokens
|
| 242 |
+
# self.visual_abstractor_config.encoder_hidden_size = self.vision_config.hidden_size
|
| 243 |
+
self.use_decoder_only_language_model = self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
|
| 244 |
+
self.initializer_factor = 1.0
|
| 245 |
+
self.initializer_range = 0.02
|
| 246 |
+
|
| 247 |
+
for attr in dir(self.text_config):
|
| 248 |
+
if not hasattr(self, attr):
|
| 249 |
+
setattr(self, attr, getattr(self.text_config, attr))
|
| 250 |
+
|
| 251 |
+
@classmethod
|
| 252 |
+
def from_vision_visual_abstractor_text_configs(
|
| 253 |
+
cls,
|
| 254 |
+
vision_config: MplugOwlVisionConfig,
|
| 255 |
+
visual_abstractor_config: MplugOwlVisualAbstractorConfig,
|
| 256 |
+
text_config: PretrainedConfig,
|
| 257 |
+
**kwargs,
|
| 258 |
+
):
|
| 259 |
+
r"""
|
| 260 |
+
Instantiate a [`MplugOwlConfig`] (or a derived class) from a mPLUG-Owl vision model, Q-Former and language model
|
| 261 |
+
configurations.
|
| 262 |
+
|
| 263 |
+
Returns:
|
| 264 |
+
[`MplugOwlConfig`]: An instance of a configuration object
|
| 265 |
+
"""
|
| 266 |
+
|
| 267 |
+
return cls(
|
| 268 |
+
vision_config=vision_config.to_dict(),
|
| 269 |
+
visual_abstractor_config=visual_abstractor_config.to_dict(),
|
| 270 |
+
text_config=text_config.to_dict(),
|
| 271 |
+
**kwargs,
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
def to_dict(self):
|
| 275 |
+
"""
|
| 276 |
+
Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
|
| 277 |
+
|
| 278 |
+
Returns:
|
| 279 |
+
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
|
| 280 |
+
"""
|
| 281 |
+
output = copy.deepcopy(self.__dict__)
|
| 282 |
+
output["vision_config"] = self.vision_config.to_dict()
|
| 283 |
+
output["visual_abstractor_config"] = self.visual_abstractor_config.to_dict()
|
| 284 |
+
output["text_config"] = self.text_config.to_dict()
|
| 285 |
+
output["model_type"] = self.__class__.model_type
|
| 286 |
+
return output
|
LaViDa/eval/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py
ADDED
|
@@ -0,0 +1,1850 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2022 x-plug The HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
""" PyTorch MplugOwl model. """
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
from typing import Any, Optional, Tuple, Union
|
| 19 |
+
|
| 20 |
+
from loguru import logger
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from flash_attn.flash_attn_interface import flash_attn_unpadded_func
|
| 24 |
+
|
| 25 |
+
flash_attn_func = flash_attn_unpadded_func
|
| 26 |
+
except:
|
| 27 |
+
flash_attn_func = None
|
| 28 |
+
logger.warning("Error importing flash_attn in mplug_owl. Please install flash-attn first.")
|
| 29 |
+
|
| 30 |
+
import math
|
| 31 |
+
from dataclasses import dataclass
|
| 32 |
+
from typing import Any, Optional, Tuple, Union
|
| 33 |
+
|
| 34 |
+
import einops
|
| 35 |
+
import torch
|
| 36 |
+
import torch.utils.checkpoint
|
| 37 |
+
from torch import nn
|
| 38 |
+
from transformers.modeling_outputs import (
|
| 39 |
+
BaseModelOutput,
|
| 40 |
+
BaseModelOutputWithPastAndCrossAttentions,
|
| 41 |
+
BaseModelOutputWithPooling,
|
| 42 |
+
)
|
| 43 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 44 |
+
from transformers.models.auto import AutoModelForCausalLM
|
| 45 |
+
from transformers.pytorch_utils import (
|
| 46 |
+
find_pruneable_heads_and_indices,
|
| 47 |
+
prune_linear_layer,
|
| 48 |
+
)
|
| 49 |
+
from transformers.utils import (
|
| 50 |
+
ModelOutput,
|
| 51 |
+
add_start_docstrings,
|
| 52 |
+
add_start_docstrings_to_model_forward,
|
| 53 |
+
replace_return_docstrings,
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
from .configuration_mplug_owl import (
|
| 57 |
+
MplugOwlConfig,
|
| 58 |
+
MplugOwlVisionConfig,
|
| 59 |
+
MplugOwlVisualAbstractorConfig,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
_CHECKPOINT_FOR_DOC = "MAGAer13/mplug-owl-llama-7b"
|
| 63 |
+
_CONFIG_FOR_DOC = "MplugOwlConfig"
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
| 67 |
+
"MAGAer13/mplug-owl-llama-7b",
|
| 68 |
+
# See all MplugOwl models at https://huggingface.co/models?filter=mplug_owl
|
| 69 |
+
]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
@dataclass
|
| 73 |
+
class MplugOwlForConditionalGenerationModelOutput(ModelOutput):
|
| 74 |
+
"""
|
| 75 |
+
Class defining the outputs of [`MPlugOwlForConditionalGeneration`].
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
|
| 79 |
+
Language modeling loss from the language model.
|
| 80 |
+
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
|
| 81 |
+
Prediction scores of the language modeling head of the language model.
|
| 82 |
+
vision_outputs (`BaseModelOutputWithPooling`):
|
| 83 |
+
Outputs of the vision encoder.
|
| 84 |
+
|
| 85 |
+
language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
|
| 86 |
+
Outputs of the language model.
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
loss: Optional[Tuple[torch.FloatTensor]] = None
|
| 90 |
+
logits: Optional[Tuple[torch.FloatTensor]] = None
|
| 91 |
+
vision_outputs: Optional[torch.FloatTensor] = None
|
| 92 |
+
language_model_outputs: Optional[Tuple[torch.FloatTensor]] = None
|
| 93 |
+
|
| 94 |
+
def to_tuple(self) -> Tuple[Any]:
|
| 95 |
+
return tuple(self[k] if k not in ["vision_outputs", "language_model_outputs"] else getattr(self, k).to_tuple() for k in self.keys())
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def get_ltor_masks_and_position_ids_from_embeddings(data):
|
| 99 |
+
"""Build masks and position id for left to right model."""
|
| 100 |
+
|
| 101 |
+
# Extract batch size and sequence length.
|
| 102 |
+
micro_batch_size, seq_length = data.size()[:2]
|
| 103 |
+
|
| 104 |
+
# Attention mask (lower triangular).
|
| 105 |
+
att_mask_batch = 1
|
| 106 |
+
attention_mask = torch.tril(torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)).view(att_mask_batch, 1, seq_length, seq_length)
|
| 107 |
+
|
| 108 |
+
# Loss mask.
|
| 109 |
+
loss_mask = torch.ones(data.size()[:2], dtype=torch.float, device=data.device)
|
| 110 |
+
|
| 111 |
+
# Position ids.
|
| 112 |
+
position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
|
| 113 |
+
position_ids = position_ids.unsqueeze(0).expand_as(data[..., 0])
|
| 114 |
+
|
| 115 |
+
# Convert attention mask to binary:
|
| 116 |
+
attention_mask = attention_mask < 0.5
|
| 117 |
+
|
| 118 |
+
return attention_mask, loss_mask, position_ids
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
class MplugOwlVisionEmbeddings(nn.Module):
|
| 122 |
+
def __init__(self, config: MplugOwlVisionConfig):
|
| 123 |
+
super().__init__()
|
| 124 |
+
self.config = config
|
| 125 |
+
self.hidden_size = config.hidden_size
|
| 126 |
+
self.image_size = config.image_size
|
| 127 |
+
self.patch_size = config.patch_size
|
| 128 |
+
|
| 129 |
+
self.cls_token = nn.Parameter(torch.randn(1, 1, self.hidden_size))
|
| 130 |
+
|
| 131 |
+
self.patch_embed = nn.Conv2d(
|
| 132 |
+
in_channels=3,
|
| 133 |
+
out_channels=self.hidden_size,
|
| 134 |
+
kernel_size=self.patch_size,
|
| 135 |
+
stride=self.patch_size,
|
| 136 |
+
bias=False,
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
self.num_patches = (self.image_size // self.patch_size) ** 2
|
| 140 |
+
|
| 141 |
+
self.position_embedding = nn.Parameter(torch.randn(1, self.num_patches + 1, self.hidden_size))
|
| 142 |
+
|
| 143 |
+
self.pre_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
|
| 144 |
+
|
| 145 |
+
def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
|
| 146 |
+
# [B, C, T, H, W] or [B, C, H, W]
|
| 147 |
+
batch_size = pixel_values.size(0)
|
| 148 |
+
T = pixel_values.size(2) if pixel_values.dim() > 4 else 1
|
| 149 |
+
if T > 1:
|
| 150 |
+
pixel_values = einops.rearrange(pixel_values, "b c t h w -> (b t) c h w")
|
| 151 |
+
image_embeds = self.patch_embed(pixel_values)
|
| 152 |
+
image_embeds = image_embeds.flatten(2).transpose(1, 2)
|
| 153 |
+
|
| 154 |
+
class_embeds = self.cls_token.expand(batch_size * T, 1, -1).to(image_embeds.dtype)
|
| 155 |
+
embeddings = torch.cat([class_embeds, image_embeds], dim=1)
|
| 156 |
+
embeddings = embeddings + self.position_embedding[:, : embeddings.size(1)].to(image_embeds.dtype)
|
| 157 |
+
embeddings = self.pre_layernorm(embeddings)
|
| 158 |
+
embeddings = einops.rearrange(embeddings, "(b t) n d -> b t n d", b=batch_size)
|
| 159 |
+
return embeddings
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class LayerNormFp32(nn.LayerNorm):
|
| 163 |
+
"""Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back)."""
|
| 164 |
+
|
| 165 |
+
def __init__(self, *args, **kwargs):
|
| 166 |
+
super().__init__(*args, **kwargs)
|
| 167 |
+
|
| 168 |
+
def forward(self, x: torch.Tensor):
|
| 169 |
+
output = torch.nn.functional.layer_norm(
|
| 170 |
+
x.float(),
|
| 171 |
+
self.normalized_shape,
|
| 172 |
+
self.weight.float() if self.weight is not None else None,
|
| 173 |
+
self.bias.float() if self.bias is not None else None,
|
| 174 |
+
self.eps,
|
| 175 |
+
)
|
| 176 |
+
return output.type_as(x)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class QuickGELU(nn.Module):
|
| 180 |
+
def forward(self, x: torch.Tensor):
|
| 181 |
+
return x * torch.sigmoid(1.702 * x)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class MplugOwlVisionLocalTemporal(nn.Module):
|
| 185 |
+
def __init__(self, config):
|
| 186 |
+
super(MplugOwlVisionLocalTemporal, self).__init__()
|
| 187 |
+
|
| 188 |
+
self.image_size = config.image_size
|
| 189 |
+
self.patch_size = config.patch_size
|
| 190 |
+
self.num_patches = 1 + (self.image_size // self.patch_size) ** 2
|
| 191 |
+
self.hidden_size = config.hidden_size
|
| 192 |
+
d_bottleneck = self.hidden_size // 2
|
| 193 |
+
|
| 194 |
+
self.ln = LayerNormFp32(self.hidden_size)
|
| 195 |
+
self.down_proj = nn.Conv3d(self.hidden_size, d_bottleneck, kernel_size=1, stride=1, padding=0)
|
| 196 |
+
self.conv = nn.Conv3d(d_bottleneck, d_bottleneck, kernel_size=(3, 1, 1), stride=1, padding=(1, 0, 0), groups=d_bottleneck)
|
| 197 |
+
self.up_proj = nn.Conv3d(d_bottleneck, self.hidden_size, kernel_size=1, stride=1, padding=0)
|
| 198 |
+
|
| 199 |
+
nn.init.constant_(self.up_proj.weight, 0)
|
| 200 |
+
nn.init.constant_(self.up_proj.bias, 0)
|
| 201 |
+
|
| 202 |
+
self.activation_func = QuickGELU()
|
| 203 |
+
|
| 204 |
+
def forward(self, x):
|
| 205 |
+
# [b, t, s, c]
|
| 206 |
+
T = x.size(1)
|
| 207 |
+
H = int((self.num_patches - 1) ** 0.5)
|
| 208 |
+
cls_token, x = x[:, :, 0:1], x[:, :, 1:]
|
| 209 |
+
x = self.ln(x)
|
| 210 |
+
x = einops.rearrange(x, "b t (h w) c -> b c t h w", h=H)
|
| 211 |
+
x = self.down_proj(x)
|
| 212 |
+
if self.conv.weight.dtype == torch.bfloat16:
|
| 213 |
+
x = torch.nn.functional.conv3d(x.half(), self.conv.weight.half(), bias=self.conv.bias.half(), stride=1, padding=(1, 0, 0), groups=self.conv.weight.shape[0]).to(cls_token.dtype)
|
| 214 |
+
else:
|
| 215 |
+
x = self.conv(x)
|
| 216 |
+
x = self.activation_func(x)
|
| 217 |
+
x = self.up_proj(x)
|
| 218 |
+
x = einops.rearrange(x, "b c t h w -> b t (h w) c")
|
| 219 |
+
x = torch.cat([cls_token, x], dim=2)
|
| 220 |
+
return x
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class MplugOwlVisionAttention(nn.Module):
|
| 224 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
| 225 |
+
|
| 226 |
+
def __init__(self, config):
|
| 227 |
+
super().__init__()
|
| 228 |
+
self.config = config
|
| 229 |
+
self.hidden_size = config.hidden_size
|
| 230 |
+
self.num_heads = config.num_attention_heads
|
| 231 |
+
self.head_dim = self.hidden_size // self.num_heads
|
| 232 |
+
if self.head_dim * self.num_heads != self.hidden_size:
|
| 233 |
+
raise ValueError(f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" f" {self.num_heads}).")
|
| 234 |
+
self.scale = self.head_dim**-0.5
|
| 235 |
+
self.dropout = nn.Dropout(config.attention_dropout)
|
| 236 |
+
|
| 237 |
+
self.query_key_value = nn.Linear(self.hidden_size, 3 * self.hidden_size)
|
| 238 |
+
self.dense = nn.Linear(self.hidden_size, self.hidden_size)
|
| 239 |
+
|
| 240 |
+
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
| 241 |
+
return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
|
| 242 |
+
|
| 243 |
+
def forward(
|
| 244 |
+
self,
|
| 245 |
+
hidden_states: torch.Tensor,
|
| 246 |
+
head_mask: Optional[torch.Tensor] = None,
|
| 247 |
+
output_attentions: Optional[bool] = False,
|
| 248 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 249 |
+
"""Input shape: Batch x Time x Channel"""
|
| 250 |
+
|
| 251 |
+
bsz, seq_len, embed_dim = hidden_states.size()
|
| 252 |
+
|
| 253 |
+
mixed_qkv = self.query_key_value(hidden_states)
|
| 254 |
+
|
| 255 |
+
mixed_qkv = mixed_qkv.reshape(bsz, seq_len, self.num_heads, 3, embed_dim // self.num_heads).permute(3, 0, 2, 1, 4) # [3, b, np, sq, hn]
|
| 256 |
+
query_states, key_states, value_states = (
|
| 257 |
+
mixed_qkv[0],
|
| 258 |
+
mixed_qkv[1],
|
| 259 |
+
mixed_qkv[2],
|
| 260 |
+
)
|
| 261 |
+
# if self.config.use_flash_attn and flash_attn_func is not None:
|
| 262 |
+
if False:
|
| 263 |
+
# [b*sq, np, hn]
|
| 264 |
+
query_states = query_states.permute(0, 2, 1, 3).contiguous()
|
| 265 |
+
query_states = query_states.view(query_states.size(0) * query_states.size(1), query_states.size(2), -1)
|
| 266 |
+
|
| 267 |
+
key_states = key_states.permute(0, 2, 1, 3).contiguous()
|
| 268 |
+
key_states = key_states.view(key_states.size(0) * key_states.size(1), key_states.size(2), -1)
|
| 269 |
+
|
| 270 |
+
value_states = value_states.permute(0, 2, 1, 3).contiguous()
|
| 271 |
+
value_states = value_states.view(value_states.size(0) * value_states.size(1), value_states.size(2), -1)
|
| 272 |
+
|
| 273 |
+
cu_seqlens = torch.arange(0, (bsz + 1) * seq_len, step=seq_len, dtype=torch.int32, device=query_states.device)
|
| 274 |
+
|
| 275 |
+
context_layer = flash_attn_func(
|
| 276 |
+
query_states,
|
| 277 |
+
key_states,
|
| 278 |
+
value_states,
|
| 279 |
+
cu_seqlens,
|
| 280 |
+
cu_seqlens,
|
| 281 |
+
seq_len,
|
| 282 |
+
seq_len,
|
| 283 |
+
self.dropout if self.training else 0.0,
|
| 284 |
+
softmax_scale=self.scale,
|
| 285 |
+
causal=False,
|
| 286 |
+
return_attn_probs=False,
|
| 287 |
+
)
|
| 288 |
+
# [b*sq, np, hn] => [b, sq, np, hn]
|
| 289 |
+
context_layer = context_layer.view(bsz, seq_len, context_layer.size(1), context_layer.size(2))
|
| 290 |
+
else:
|
| 291 |
+
# Take the dot product between "query" and "key" to get the raw attention scores.
|
| 292 |
+
attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2))
|
| 293 |
+
|
| 294 |
+
attention_scores = attention_scores * self.scale
|
| 295 |
+
|
| 296 |
+
# Normalize the attention scores to probabilities.
|
| 297 |
+
attention_probs = torch.softmax(attention_scores, dim=-1)
|
| 298 |
+
|
| 299 |
+
# This is actually dropping out entire tokens to attend to, which might
|
| 300 |
+
# seem a bit unusual, but is taken from the original Transformer paper.
|
| 301 |
+
attention_probs = self.dropout(attention_probs)
|
| 302 |
+
|
| 303 |
+
# Mask heads if we want to
|
| 304 |
+
if head_mask is not None:
|
| 305 |
+
attention_probs = attention_probs * head_mask
|
| 306 |
+
|
| 307 |
+
context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3)
|
| 308 |
+
|
| 309 |
+
new_context_layer_shape = context_layer.size()[:-2] + (self.hidden_size,)
|
| 310 |
+
context_layer = context_layer.reshape(new_context_layer_shape)
|
| 311 |
+
|
| 312 |
+
output = self.dense(context_layer)
|
| 313 |
+
|
| 314 |
+
outputs = (output, attention_probs) if output_attentions else (output, None)
|
| 315 |
+
|
| 316 |
+
return outputs
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
class MplugOwlMLP(nn.Module):
|
| 320 |
+
def __init__(self, config):
|
| 321 |
+
super().__init__()
|
| 322 |
+
self.config = config
|
| 323 |
+
self.activation_fn = QuickGELU()
|
| 324 |
+
self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
|
| 325 |
+
self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
|
| 326 |
+
|
| 327 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 328 |
+
hidden_states = self.fc1(hidden_states)
|
| 329 |
+
hidden_states = self.activation_fn(hidden_states)
|
| 330 |
+
hidden_states = self.fc2(hidden_states)
|
| 331 |
+
return hidden_states
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
class MplugOwlVisionEncoderLayer(nn.Module):
|
| 335 |
+
def __init__(self, config: MplugOwlVisionConfig):
|
| 336 |
+
super().__init__()
|
| 337 |
+
self.hidden_size = config.hidden_size
|
| 338 |
+
self.temporal = MplugOwlVisionLocalTemporal(config)
|
| 339 |
+
self.self_attn = MplugOwlVisionAttention(config)
|
| 340 |
+
self.input_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
|
| 341 |
+
self.mlp = MplugOwlMLP(config)
|
| 342 |
+
self.post_attention_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
|
| 343 |
+
|
| 344 |
+
def forward(
|
| 345 |
+
self,
|
| 346 |
+
hidden_states: torch.Tensor,
|
| 347 |
+
attention_mask: torch.Tensor,
|
| 348 |
+
output_attentions: Optional[bool] = False,
|
| 349 |
+
) -> Tuple[torch.FloatTensor]:
|
| 350 |
+
"""
|
| 351 |
+
Args:
|
| 352 |
+
hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, time, seq_len, embed_dim)`
|
| 353 |
+
attention_mask (`torch.FloatTensor`): attention mask of size
|
| 354 |
+
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
|
| 355 |
+
`(config.encoder_attention_heads,)`.
|
| 356 |
+
output_attentions (`bool`, *optional*):
|
| 357 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 358 |
+
returned tensors for more detail.
|
| 359 |
+
"""
|
| 360 |
+
B, T = hidden_states.size(0), hidden_states.size(1)
|
| 361 |
+
if T > 1:
|
| 362 |
+
hidden_states = hidden_states + self.temporal(hidden_states)
|
| 363 |
+
hidden_states = einops.rearrange(hidden_states, "b t n d -> (b t) n d")
|
| 364 |
+
|
| 365 |
+
residual = hidden_states
|
| 366 |
+
|
| 367 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 368 |
+
hidden_states, attn_weights = self.self_attn(
|
| 369 |
+
hidden_states=hidden_states,
|
| 370 |
+
head_mask=attention_mask,
|
| 371 |
+
output_attentions=output_attentions,
|
| 372 |
+
)
|
| 373 |
+
hidden_states = hidden_states + residual
|
| 374 |
+
residual = hidden_states
|
| 375 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 376 |
+
hidden_states = self.mlp(hidden_states)
|
| 377 |
+
|
| 378 |
+
hidden_states = hidden_states + residual
|
| 379 |
+
hidden_states = einops.rearrange(hidden_states, "(b t) n d -> b t n d", b=B)
|
| 380 |
+
|
| 381 |
+
outputs = (hidden_states,)
|
| 382 |
+
|
| 383 |
+
if output_attentions:
|
| 384 |
+
outputs += (attn_weights,)
|
| 385 |
+
|
| 386 |
+
return outputs
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
class MplugOwlPreTrainedModel(PreTrainedModel):
|
| 390 |
+
"""
|
| 391 |
+
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
|
| 392 |
+
models.
|
| 393 |
+
"""
|
| 394 |
+
|
| 395 |
+
config_class = MplugOwlConfig
|
| 396 |
+
base_model_prefix = "mplug_owl"
|
| 397 |
+
supports_gradient_checkpointing = True
|
| 398 |
+
_keys_to_ignore_on_load_missing = [
|
| 399 |
+
r"position_ids",
|
| 400 |
+
r"language_model.encoder.embed_tokens.weight",
|
| 401 |
+
r"language_model.decoder.embed_tokens.weight",
|
| 402 |
+
r"language_model.lm_head.weight",
|
| 403 |
+
]
|
| 404 |
+
_no_split_modules = [
|
| 405 |
+
"MplugOwlVisionEncoderLayer",
|
| 406 |
+
"LlamaDecoderLayer",
|
| 407 |
+
"MplugOwlVisualAbstractorLayer",
|
| 408 |
+
"LlamaForCausalLM",
|
| 409 |
+
"Parameter",
|
| 410 |
+
]
|
| 411 |
+
_keep_in_fp32_modules = ["wo"]
|
| 412 |
+
|
| 413 |
+
def _init_weights(self, module):
|
| 414 |
+
"""Initialize the weights"""
|
| 415 |
+
factor = self.config.initializer_range
|
| 416 |
+
if isinstance(module, nn.Conv2d) or isinstance(module, nn.Embedding) or isinstance(module, nn.Linear):
|
| 417 |
+
module.weight.data.normal_(mean=0.0, std=factor)
|
| 418 |
+
if hasattr(module, "bias") and module.bias is not None:
|
| 419 |
+
module.bias.data.zero_()
|
| 420 |
+
|
| 421 |
+
if isinstance(module, MplugOwlVisionEmbeddings):
|
| 422 |
+
if hasattr(self.config, "vision_config"):
|
| 423 |
+
factor = self.config.vision_config.initializer_range
|
| 424 |
+
nn.init.trunc_normal_(module.position_embedding, mean=0.0, std=factor)
|
| 425 |
+
nn.init.trunc_normal_(module.cls_token, mean=0.0, std=factor)
|
| 426 |
+
|
| 427 |
+
elif isinstance(module, nn.LayerNorm):
|
| 428 |
+
module.bias.data.zero_()
|
| 429 |
+
module.weight.data.fill_(1.0)
|
| 430 |
+
elif isinstance(module, nn.Linear) and module.bias is not None:
|
| 431 |
+
module.bias.data.zero_()
|
| 432 |
+
elif isinstance(module, nn.Parameter):
|
| 433 |
+
raise ValueError
|
| 434 |
+
nn.init.trunc_normal_(module.data, mean=0.0, std=factor)
|
| 435 |
+
|
| 436 |
+
def _set_gradient_checkpointing(self, module, value=False):
|
| 437 |
+
if isinstance(module, MplugOwlVisionEncoder):
|
| 438 |
+
module.gradient_checkpointing = value
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
MPLUG_OWL_START_DOCSTRING = r"""
|
| 442 |
+
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
| 443 |
+
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
| 444 |
+
etc.)
|
| 445 |
+
|
| 446 |
+
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
| 447 |
+
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
| 448 |
+
and behavior.
|
| 449 |
+
|
| 450 |
+
Parameters:
|
| 451 |
+
config ([`MplugOwlConfig`]): Model configuration class with all the parameters of the model.
|
| 452 |
+
Initializing with a config file does not load the weights associated with the model, only the
|
| 453 |
+
configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
| 454 |
+
"""
|
| 455 |
+
|
| 456 |
+
MPLUG_OWL_VISION_INPUTS_DOCSTRING = r"""
|
| 457 |
+
Args:
|
| 458 |
+
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
|
| 459 |
+
Pixel values. Pixel values can be obtained using [`MplugOwlProcessor`]. See [`MplugOwlProcessor.__call__`] for
|
| 460 |
+
details.
|
| 461 |
+
output_attentions (`bool`, *optional*):
|
| 462 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 463 |
+
tensors for more detail.
|
| 464 |
+
output_hidden_states (`bool`, *optional*):
|
| 465 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 466 |
+
more detail.
|
| 467 |
+
return_dict (`bool`, *optional*):
|
| 468 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 469 |
+
"""
|
| 470 |
+
|
| 471 |
+
MPLUG_OWL_TEXT_INPUTS_DOCSTRING = r"""
|
| 472 |
+
Args:
|
| 473 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
| 474 |
+
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
|
| 475 |
+
it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 476 |
+
[`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
|
| 477 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 478 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 479 |
+
- 1 for tokens that are **not masked**,
|
| 480 |
+
- 0 for tokens that are **masked**.
|
| 481 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 482 |
+
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
|
| 483 |
+
Indices of decoder input sequence tokens in the vocabulary.
|
| 484 |
+
|
| 485 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 486 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
| 487 |
+
|
| 488 |
+
[What are decoder input IDs?](../glossary#decoder-input-ids)
|
| 489 |
+
|
| 490 |
+
T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
|
| 491 |
+
is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
|
| 492 |
+
|
| 493 |
+
To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
|
| 494 |
+
Training](./t5#training).
|
| 495 |
+
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
|
| 496 |
+
Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
|
| 497 |
+
be used by default.
|
| 498 |
+
output_attentions (`bool`, *optional*):
|
| 499 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 500 |
+
tensors for more detail.
|
| 501 |
+
output_hidden_states (`bool`, *optional*):
|
| 502 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 503 |
+
more detail.
|
| 504 |
+
return_dict (`bool`, *optional*):
|
| 505 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 506 |
+
"""
|
| 507 |
+
|
| 508 |
+
MPLUG_OWL_INPUTS_DOCSTRING = r"""
|
| 509 |
+
Args:
|
| 510 |
+
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
|
| 511 |
+
Pixel values. Pixel values can be obtained using [`MplugOwlProcessor`]. See [`MplugOwlProcessor.__call__`] for
|
| 512 |
+
details.
|
| 513 |
+
|
| 514 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 515 |
+
Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be
|
| 516 |
+
provided to serve as text prompt, which the language model can continue.
|
| 517 |
+
|
| 518 |
+
Indices can be obtained using [`MplugOwlProcessor`]. See [`MplugOwlProcessor.__call__`] for details.
|
| 519 |
+
|
| 520 |
+
[What are input IDs?](../glossary#input-ids)
|
| 521 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 522 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 523 |
+
|
| 524 |
+
- 1 for tokens that are **not masked**,
|
| 525 |
+
- 0 for tokens that are **masked**.
|
| 526 |
+
|
| 527 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 528 |
+
|
| 529 |
+
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
|
| 530 |
+
Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an
|
| 531 |
+
encoder-decoder language model (like T5) is used.
|
| 532 |
+
|
| 533 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 534 |
+
[`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids)
|
| 535 |
+
|
| 536 |
+
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
|
| 537 |
+
Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
|
| 538 |
+
be used by default.
|
| 539 |
+
|
| 540 |
+
Only relevant in case an encoder-decoder language model (like T5) is used.
|
| 541 |
+
|
| 542 |
+
output_attentions (`bool`, *optional*):
|
| 543 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 544 |
+
tensors for more detail.
|
| 545 |
+
output_hidden_states (`bool`, *optional*):
|
| 546 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 547 |
+
more detail.
|
| 548 |
+
return_dict (`bool`, *optional*):
|
| 549 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 550 |
+
"""
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
class MplugOwlVisionEncoder(nn.Module):
|
| 554 |
+
"""
|
| 555 |
+
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
|
| 556 |
+
[`MplugOwlVisionEncoderLayer`].
|
| 557 |
+
|
| 558 |
+
Args:
|
| 559 |
+
config (`MplugOwlVisionConfig`):
|
| 560 |
+
The corresponding vision configuration for the `MplugOwlEncoder`.
|
| 561 |
+
"""
|
| 562 |
+
|
| 563 |
+
def __init__(self, config: MplugOwlVisionConfig):
|
| 564 |
+
super().__init__()
|
| 565 |
+
self.config = config
|
| 566 |
+
self.layers = nn.ModuleList([MplugOwlVisionEncoderLayer(config) for _ in range(config.num_hidden_layers)])
|
| 567 |
+
self.gradient_checkpointing = False
|
| 568 |
+
|
| 569 |
+
def forward(
|
| 570 |
+
self,
|
| 571 |
+
inputs_embeds,
|
| 572 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 573 |
+
output_attentions: Optional[bool] = None,
|
| 574 |
+
output_hidden_states: Optional[bool] = None,
|
| 575 |
+
return_dict: Optional[bool] = None,
|
| 576 |
+
) -> Union[Tuple, BaseModelOutput]:
|
| 577 |
+
r"""
|
| 578 |
+
Args:
|
| 579 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 580 |
+
Embedded representation of the inputs. Should be float, not int tokens.
|
| 581 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 582 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 583 |
+
|
| 584 |
+
- 1 for tokens that are **not masked**,
|
| 585 |
+
- 0 for tokens that are **masked**.
|
| 586 |
+
|
| 587 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 588 |
+
output_attentions (`bool`, *optional*):
|
| 589 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 590 |
+
returned tensors for more detail.
|
| 591 |
+
output_hidden_states (`bool`, *optional*):
|
| 592 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
|
| 593 |
+
for more detail.
|
| 594 |
+
return_dict (`bool`, *optional*):
|
| 595 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 596 |
+
"""
|
| 597 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 598 |
+
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 599 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 600 |
+
|
| 601 |
+
encoder_states = () if output_hidden_states else None
|
| 602 |
+
all_attentions = () if output_attentions else None
|
| 603 |
+
|
| 604 |
+
hidden_states = inputs_embeds
|
| 605 |
+
for idx, encoder_layer in enumerate(self.layers):
|
| 606 |
+
if output_hidden_states:
|
| 607 |
+
encoder_states = encoder_states + (hidden_states,)
|
| 608 |
+
if self.gradient_checkpointing and self.training:
|
| 609 |
+
|
| 610 |
+
def create_custom_forward(module):
|
| 611 |
+
def custom_forward(*inputs):
|
| 612 |
+
return module(*inputs, output_attentions)
|
| 613 |
+
|
| 614 |
+
return custom_forward
|
| 615 |
+
|
| 616 |
+
layer_outputs = torch.utils.checkpoint.checkpoint(
|
| 617 |
+
create_custom_forward(encoder_layer),
|
| 618 |
+
hidden_states,
|
| 619 |
+
attention_mask,
|
| 620 |
+
)
|
| 621 |
+
else:
|
| 622 |
+
layer_outputs = encoder_layer(
|
| 623 |
+
hidden_states,
|
| 624 |
+
attention_mask,
|
| 625 |
+
output_attentions=output_attentions,
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
hidden_states = layer_outputs[0]
|
| 629 |
+
|
| 630 |
+
if output_attentions:
|
| 631 |
+
all_attentions = all_attentions + (layer_outputs[1],)
|
| 632 |
+
|
| 633 |
+
if output_hidden_states:
|
| 634 |
+
encoder_states = encoder_states + (hidden_states,)
|
| 635 |
+
|
| 636 |
+
if not return_dict:
|
| 637 |
+
return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
|
| 638 |
+
return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions)
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
class MplugOwlVisionModel(MplugOwlPreTrainedModel):
|
| 642 |
+
main_input_name = "pixel_values"
|
| 643 |
+
config_class = MplugOwlVisionConfig
|
| 644 |
+
|
| 645 |
+
def __init__(self, config: MplugOwlVisionConfig):
|
| 646 |
+
super().__init__(config)
|
| 647 |
+
self.config = config
|
| 648 |
+
self.hidden_size = config.hidden_size
|
| 649 |
+
|
| 650 |
+
self.embeddings = MplugOwlVisionEmbeddings(config)
|
| 651 |
+
self.encoder = MplugOwlVisionEncoder(config)
|
| 652 |
+
self.post_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
|
| 653 |
+
|
| 654 |
+
self.post_init()
|
| 655 |
+
|
| 656 |
+
@add_start_docstrings_to_model_forward(MPLUG_OWL_VISION_INPUTS_DOCSTRING)
|
| 657 |
+
@replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=MplugOwlVisionConfig)
|
| 658 |
+
def forward(
|
| 659 |
+
self,
|
| 660 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 661 |
+
output_attentions: Optional[bool] = None,
|
| 662 |
+
output_hidden_states: Optional[bool] = None,
|
| 663 |
+
return_dict: Optional[bool] = None,
|
| 664 |
+
) -> Union[Tuple, BaseModelOutputWithPooling]:
|
| 665 |
+
r"""
|
| 666 |
+
Returns:
|
| 667 |
+
|
| 668 |
+
"""
|
| 669 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 670 |
+
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 671 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 672 |
+
|
| 673 |
+
if pixel_values is None:
|
| 674 |
+
raise ValueError("You have to specify pixel_values")
|
| 675 |
+
|
| 676 |
+
hidden_states = self.embeddings(pixel_values) # [B, T, N, D]
|
| 677 |
+
|
| 678 |
+
encoder_outputs = self.encoder(
|
| 679 |
+
inputs_embeds=hidden_states,
|
| 680 |
+
output_attentions=output_attentions,
|
| 681 |
+
output_hidden_states=output_hidden_states,
|
| 682 |
+
return_dict=return_dict,
|
| 683 |
+
)
|
| 684 |
+
|
| 685 |
+
last_hidden_state = encoder_outputs[0]
|
| 686 |
+
last_hidden_state = self.post_layernorm(last_hidden_state)
|
| 687 |
+
|
| 688 |
+
pooled_output = last_hidden_state[:, :, 0, :].mean(1)
|
| 689 |
+
pooled_output = self.post_layernorm(pooled_output)
|
| 690 |
+
|
| 691 |
+
if not return_dict:
|
| 692 |
+
return (last_hidden_state, pooled_output) + encoder_outputs[1:]
|
| 693 |
+
|
| 694 |
+
return BaseModelOutputWithPooling(
|
| 695 |
+
last_hidden_state=last_hidden_state,
|
| 696 |
+
pooler_output=pooled_output,
|
| 697 |
+
hidden_states=encoder_outputs.hidden_states,
|
| 698 |
+
attentions=encoder_outputs.attentions,
|
| 699 |
+
)
|
| 700 |
+
|
| 701 |
+
def get_input_embeddings(self):
|
| 702 |
+
return self.embeddings
|
| 703 |
+
|
| 704 |
+
|
| 705 |
+
class MplugOwlVisualAbstractorMLP(nn.Module):
|
| 706 |
+
def __init__(self, config: MplugOwlVisualAbstractorConfig):
|
| 707 |
+
super().__init__()
|
| 708 |
+
self.config = config
|
| 709 |
+
in_features = config.hidden_size
|
| 710 |
+
hidden_features = config.intermediate_size
|
| 711 |
+
if hidden_features != 2816:
|
| 712 |
+
hidden_features = int(2 * hidden_features / 3)
|
| 713 |
+
multiple_of = 256
|
| 714 |
+
hidden_features = multiple_of * ((hidden_features + multiple_of - 1) // multiple_of)
|
| 715 |
+
self.act = nn.SiLU()
|
| 716 |
+
|
| 717 |
+
self.w1 = nn.Linear(in_features, hidden_features)
|
| 718 |
+
self.w2 = nn.Linear(hidden_features, in_features)
|
| 719 |
+
self.w3 = nn.Linear(in_features, hidden_features)
|
| 720 |
+
self.ffn_ln = LayerNormFp32(hidden_features, eps=config.layer_norm_eps)
|
| 721 |
+
|
| 722 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 723 |
+
hidden_states = self.act(self.w1(hidden_states)) * self.w3(hidden_states)
|
| 724 |
+
hidden_states = self.ffn_ln(hidden_states)
|
| 725 |
+
hidden_states = self.w2(hidden_states)
|
| 726 |
+
return hidden_states
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
class MplugOwlVisualAbstractorMultiHeadAttention(nn.Module):
|
| 730 |
+
def __init__(self, config: MplugOwlVisualAbstractorConfig):
|
| 731 |
+
super().__init__()
|
| 732 |
+
self.config = config
|
| 733 |
+
if config.hidden_size % config.num_attention_heads != 0:
|
| 734 |
+
raise ValueError("The hidden size (%d) is not a multiple of the number of attention heads (%d)" % (config.hidden_size, config.num_attention_heads))
|
| 735 |
+
|
| 736 |
+
self.num_attention_heads = config.num_attention_heads
|
| 737 |
+
self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
|
| 738 |
+
self.all_head_size = self.num_attention_heads * self.attention_head_size
|
| 739 |
+
|
| 740 |
+
self.query = nn.Linear(config.hidden_size, self.all_head_size)
|
| 741 |
+
self.key = nn.Linear(config.encoder_hidden_size, self.all_head_size)
|
| 742 |
+
self.value = nn.Linear(config.encoder_hidden_size, self.all_head_size)
|
| 743 |
+
|
| 744 |
+
self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
|
| 745 |
+
self.save_attention = False
|
| 746 |
+
|
| 747 |
+
def save_attn_gradients(self, attn_gradients):
|
| 748 |
+
self.attn_gradients = attn_gradients
|
| 749 |
+
|
| 750 |
+
def get_attn_gradients(self):
|
| 751 |
+
return self.attn_gradients
|
| 752 |
+
|
| 753 |
+
def save_attention_map(self, attention_map):
|
| 754 |
+
self.attention_map = attention_map
|
| 755 |
+
|
| 756 |
+
def get_attention_map(self):
|
| 757 |
+
return self.attention_map
|
| 758 |
+
|
| 759 |
+
def transpose_for_scores(self, x):
|
| 760 |
+
new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
|
| 761 |
+
x = x.view(*new_x_shape)
|
| 762 |
+
return x.permute(0, 2, 1, 3)
|
| 763 |
+
|
| 764 |
+
def forward(
|
| 765 |
+
self,
|
| 766 |
+
hidden_states,
|
| 767 |
+
attention_mask=None,
|
| 768 |
+
head_mask=None,
|
| 769 |
+
encoder_hidden_states=None,
|
| 770 |
+
encoder_attention_mask=None,
|
| 771 |
+
past_key_value=None,
|
| 772 |
+
output_attentions=False,
|
| 773 |
+
):
|
| 774 |
+
# If this is instantiated as a cross-attention module, the keys
|
| 775 |
+
# and values come from an encoder; the attention mask needs to be
|
| 776 |
+
# such that the encoder's padding tokens are not attended to.
|
| 777 |
+
key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
|
| 778 |
+
value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
|
| 779 |
+
attention_mask = encoder_attention_mask
|
| 780 |
+
|
| 781 |
+
mixed_query_layer = self.query(hidden_states)
|
| 782 |
+
|
| 783 |
+
query_layer = self.transpose_for_scores(mixed_query_layer)
|
| 784 |
+
|
| 785 |
+
past_key_value = (key_layer, value_layer)
|
| 786 |
+
|
| 787 |
+
# Take the dot product between "query" and "key" to get the raw attention scores.
|
| 788 |
+
attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
|
| 789 |
+
|
| 790 |
+
attention_scores = attention_scores / math.sqrt(self.attention_head_size)
|
| 791 |
+
|
| 792 |
+
if attention_mask is not None:
|
| 793 |
+
# Apply the attention mask is (precomputed for all layers in BertModel forward() function)
|
| 794 |
+
attention_scores = attention_scores + attention_mask
|
| 795 |
+
|
| 796 |
+
# Normalize the attention scores to probabilities.
|
| 797 |
+
attention_probs = nn.Softmax(dim=-1)(attention_scores)
|
| 798 |
+
|
| 799 |
+
if self.save_attention:
|
| 800 |
+
self.save_attention_map(attention_probs)
|
| 801 |
+
attention_probs.register_hook(self.save_attn_gradients)
|
| 802 |
+
|
| 803 |
+
# This is actually dropping out entire tokens to attend to, which might
|
| 804 |
+
# seem a bit unusual, but is taken from the original Transformer paper.
|
| 805 |
+
attention_probs_dropped = self.dropout(attention_probs)
|
| 806 |
+
|
| 807 |
+
# Mask heads if we want to
|
| 808 |
+
if head_mask is not None:
|
| 809 |
+
attention_probs_dropped = attention_probs_dropped * head_mask
|
| 810 |
+
|
| 811 |
+
context_layer = torch.matmul(attention_probs_dropped, value_layer)
|
| 812 |
+
|
| 813 |
+
context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
|
| 814 |
+
new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
|
| 815 |
+
context_layer = context_layer.view(*new_context_layer_shape)
|
| 816 |
+
|
| 817 |
+
outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
|
| 818 |
+
|
| 819 |
+
outputs = outputs + (past_key_value,)
|
| 820 |
+
return outputs
|
| 821 |
+
|
| 822 |
+
|
| 823 |
+
class MplugOwlVisualAbstractorCrossOutput(nn.Module):
|
| 824 |
+
def __init__(self, config: MplugOwlVisualAbstractorConfig):
|
| 825 |
+
super().__init__()
|
| 826 |
+
dim = config.hidden_size
|
| 827 |
+
self.out_proj = nn.Linear(dim, dim, bias=True)
|
| 828 |
+
self.norm2 = LayerNormFp32(dim)
|
| 829 |
+
self.mlp = MplugOwlVisualAbstractorMLP(config)
|
| 830 |
+
|
| 831 |
+
def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor:
|
| 832 |
+
input_tensor = input_tensor + self.out_proj(hidden_states)
|
| 833 |
+
input_tensor = input_tensor + self.mlp(self.norm2(input_tensor))
|
| 834 |
+
return input_tensor
|
| 835 |
+
|
| 836 |
+
|
| 837 |
+
class MplugOwlVisualAbstractorAttention(nn.Module):
|
| 838 |
+
def __init__(self, config: MplugOwlVisualAbstractorConfig):
|
| 839 |
+
super().__init__()
|
| 840 |
+
self.attention = MplugOwlVisualAbstractorMultiHeadAttention(config)
|
| 841 |
+
self.output = MplugOwlVisualAbstractorCrossOutput(config)
|
| 842 |
+
self.pruned_heads = set()
|
| 843 |
+
self.norm1 = LayerNormFp32(config.hidden_size)
|
| 844 |
+
self.normk = LayerNormFp32(config.hidden_size)
|
| 845 |
+
|
| 846 |
+
def prune_heads(self, heads):
|
| 847 |
+
if len(heads) == 0:
|
| 848 |
+
return
|
| 849 |
+
heads, index = find_pruneable_heads_and_indices(heads, self.attention.num_attention_heads, self.attention.attention_head_size, self.pruned_heads)
|
| 850 |
+
|
| 851 |
+
# Prune linear layers
|
| 852 |
+
self.attention.query = prune_linear_layer(self.attention.query, index)
|
| 853 |
+
self.attention.key = prune_linear_layer(self.attention.key, index)
|
| 854 |
+
self.attention.value = prune_linear_layer(self.attention.value, index)
|
| 855 |
+
self.output.dense = prune_linear_layer(self.output.out_proj, index, dim=1)
|
| 856 |
+
|
| 857 |
+
# Update hyper params and store pruned heads
|
| 858 |
+
self.attention.num_attention_heads = self.attention.num_attention_heads - len(heads)
|
| 859 |
+
self.attention.all_head_size = self.attention.attention_head_size * self.attention.num_attention_heads
|
| 860 |
+
self.pruned_heads = self.pruned_heads.union(heads)
|
| 861 |
+
|
| 862 |
+
def forward(
|
| 863 |
+
self,
|
| 864 |
+
hidden_states: torch.Tensor,
|
| 865 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 866 |
+
head_mask: Optional[torch.FloatTensor] = None,
|
| 867 |
+
encoder_hidden_states: Optional[torch.FloatTensor] = None,
|
| 868 |
+
encoder_attention_mask: Optional[torch.FloatTensor] = None,
|
| 869 |
+
past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
|
| 870 |
+
output_attentions: Optional[bool] = False,
|
| 871 |
+
) -> Tuple[torch.Tensor]:
|
| 872 |
+
# HACK we apply norm on q and k
|
| 873 |
+
hidden_states = self.norm1(hidden_states)
|
| 874 |
+
encoder_hidden_states = self.normk(encoder_hidden_states)
|
| 875 |
+
encoder_hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1)
|
| 876 |
+
encoder_attention_mask = torch.cat([attention_mask, encoder_attention_mask], dim=-1)
|
| 877 |
+
self_outputs = self.attention(
|
| 878 |
+
hidden_states,
|
| 879 |
+
attention_mask,
|
| 880 |
+
head_mask,
|
| 881 |
+
encoder_hidden_states,
|
| 882 |
+
encoder_attention_mask,
|
| 883 |
+
past_key_value,
|
| 884 |
+
output_attentions,
|
| 885 |
+
)
|
| 886 |
+
attention_output = self.output(self_outputs[0], hidden_states)
|
| 887 |
+
# add attentions if we output them
|
| 888 |
+
outputs = (attention_output,) + self_outputs[1:]
|
| 889 |
+
return outputs
|
| 890 |
+
|
| 891 |
+
|
| 892 |
+
class MplugOwlVisualAbstractorLayer(nn.Module):
|
| 893 |
+
def __init__(self, config, layer_idx):
|
| 894 |
+
super().__init__()
|
| 895 |
+
self.chunk_size_feed_forward = config.chunk_size_feed_forward
|
| 896 |
+
self.seq_len_dim = 1
|
| 897 |
+
|
| 898 |
+
self.layer_idx = layer_idx
|
| 899 |
+
|
| 900 |
+
self.crossattention = MplugOwlVisualAbstractorAttention(config)
|
| 901 |
+
self.has_cross_attention = True
|
| 902 |
+
|
| 903 |
+
def forward(
|
| 904 |
+
self,
|
| 905 |
+
hidden_states,
|
| 906 |
+
attention_mask=None,
|
| 907 |
+
head_mask=None,
|
| 908 |
+
encoder_hidden_states=None,
|
| 909 |
+
encoder_attention_mask=None,
|
| 910 |
+
output_attentions=False,
|
| 911 |
+
):
|
| 912 |
+
if encoder_hidden_states is None:
|
| 913 |
+
raise ValueError("encoder_hidden_states must be given for cross-attention layers")
|
| 914 |
+
cross_attention_outputs = self.crossattention(
|
| 915 |
+
hidden_states,
|
| 916 |
+
attention_mask,
|
| 917 |
+
head_mask,
|
| 918 |
+
encoder_hidden_states,
|
| 919 |
+
encoder_attention_mask,
|
| 920 |
+
output_attentions=output_attentions,
|
| 921 |
+
)
|
| 922 |
+
query_attention_output = cross_attention_outputs[0]
|
| 923 |
+
|
| 924 |
+
outputs = (query_attention_output,)
|
| 925 |
+
return outputs
|
| 926 |
+
|
| 927 |
+
|
| 928 |
+
class MplugOwlVisualAbstractorEncoder(nn.Module):
|
| 929 |
+
def __init__(self, config):
|
| 930 |
+
super().__init__()
|
| 931 |
+
self.config = config
|
| 932 |
+
self.layers = nn.ModuleList([MplugOwlVisualAbstractorLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)])
|
| 933 |
+
self.gradient_checkpointing = False
|
| 934 |
+
|
| 935 |
+
def forward(
|
| 936 |
+
self,
|
| 937 |
+
hidden_states,
|
| 938 |
+
attention_mask=None,
|
| 939 |
+
head_mask=None,
|
| 940 |
+
encoder_hidden_states=None,
|
| 941 |
+
encoder_attention_mask=None,
|
| 942 |
+
past_key_values=None,
|
| 943 |
+
output_attentions=False,
|
| 944 |
+
output_hidden_states=False,
|
| 945 |
+
return_dict=True,
|
| 946 |
+
):
|
| 947 |
+
all_hidden_states = () if output_hidden_states else None
|
| 948 |
+
|
| 949 |
+
for i in range(self.config.num_hidden_layers):
|
| 950 |
+
layer_module = self.layers[i]
|
| 951 |
+
if output_hidden_states:
|
| 952 |
+
all_hidden_states = all_hidden_states + (hidden_states,)
|
| 953 |
+
|
| 954 |
+
layer_head_mask = head_mask[i] if head_mask is not None else None
|
| 955 |
+
past_key_value = past_key_values[i] if past_key_values is not None else None
|
| 956 |
+
|
| 957 |
+
if getattr(self.config, "gradient_checkpointing", False) and self.training:
|
| 958 |
+
|
| 959 |
+
def create_custom_forward(module):
|
| 960 |
+
def custom_forward(*inputs):
|
| 961 |
+
return module(*inputs, past_key_value, output_attentions)
|
| 962 |
+
|
| 963 |
+
return custom_forward
|
| 964 |
+
|
| 965 |
+
layer_outputs = torch.utils.checkpoint.checkpoint(
|
| 966 |
+
create_custom_forward(layer_module),
|
| 967 |
+
hidden_states,
|
| 968 |
+
attention_mask,
|
| 969 |
+
layer_head_mask,
|
| 970 |
+
encoder_hidden_states,
|
| 971 |
+
encoder_attention_mask,
|
| 972 |
+
)
|
| 973 |
+
else:
|
| 974 |
+
layer_outputs = layer_module(
|
| 975 |
+
hidden_states,
|
| 976 |
+
attention_mask,
|
| 977 |
+
layer_head_mask,
|
| 978 |
+
encoder_hidden_states,
|
| 979 |
+
encoder_attention_mask,
|
| 980 |
+
output_attentions,
|
| 981 |
+
)
|
| 982 |
+
|
| 983 |
+
hidden_states = layer_outputs[0]
|
| 984 |
+
|
| 985 |
+
return BaseModelOutput(
|
| 986 |
+
last_hidden_state=hidden_states,
|
| 987 |
+
)
|
| 988 |
+
|
| 989 |
+
|
| 990 |
+
class MplugOwlVisualAbstractorModel(MplugOwlPreTrainedModel):
|
| 991 |
+
def __init__(self, config: MplugOwlVisualAbstractorConfig, language_hidden_size):
|
| 992 |
+
super().__init__(config)
|
| 993 |
+
self.config = config
|
| 994 |
+
|
| 995 |
+
self.encoder = MplugOwlVisualAbstractorEncoder(config)
|
| 996 |
+
self.visual_fc = torch.nn.Linear(config.hidden_size, language_hidden_size)
|
| 997 |
+
self.temporal_visual_fc = torch.nn.Linear(config.hidden_size, language_hidden_size)
|
| 998 |
+
self.vit_eos = torch.nn.Parameter(torch.randn(1, 1, language_hidden_size))
|
| 999 |
+
nn.init.trunc_normal_(self.vit_eos, mean=0.0, std=self.config.initializer_range)
|
| 1000 |
+
self.post_init()
|
| 1001 |
+
|
| 1002 |
+
def _prune_heads(self, heads_to_prune):
|
| 1003 |
+
"""
|
| 1004 |
+
Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
|
| 1005 |
+
class PreTrainedModel
|
| 1006 |
+
"""
|
| 1007 |
+
for layer, heads in heads_to_prune.items():
|
| 1008 |
+
self.encoder.layer[layer].attention.prune_heads(heads)
|
| 1009 |
+
|
| 1010 |
+
def get_extended_attention_mask(
|
| 1011 |
+
self,
|
| 1012 |
+
attention_mask: torch.Tensor,
|
| 1013 |
+
input_shape: Tuple[int],
|
| 1014 |
+
device: torch.device,
|
| 1015 |
+
) -> torch.Tensor:
|
| 1016 |
+
"""
|
| 1017 |
+
Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
|
| 1018 |
+
|
| 1019 |
+
Arguments:
|
| 1020 |
+
attention_mask (`torch.Tensor`):
|
| 1021 |
+
Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
|
| 1022 |
+
input_shape (`Tuple[int]`):
|
| 1023 |
+
The shape of the input to the model.
|
| 1024 |
+
device: (`torch.device`):
|
| 1025 |
+
The device of the input to the model.
|
| 1026 |
+
|
| 1027 |
+
Returns:
|
| 1028 |
+
`torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
|
| 1029 |
+
"""
|
| 1030 |
+
# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
|
| 1031 |
+
# ourselves in which case we just need to make it broadcastable to all heads.
|
| 1032 |
+
if attention_mask.dim() == 3:
|
| 1033 |
+
extended_attention_mask = attention_mask[:, None, :, :]
|
| 1034 |
+
elif attention_mask.dim() == 2:
|
| 1035 |
+
# Provided a padding mask of dimensions [batch_size, seq_length]
|
| 1036 |
+
# - the model is an encoder, so make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
|
| 1037 |
+
extended_attention_mask = attention_mask[:, None, None, :]
|
| 1038 |
+
else:
|
| 1039 |
+
raise ValueError("Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(input_shape, attention_mask.shape))
|
| 1040 |
+
|
| 1041 |
+
# Since attention_mask is 1.0 for positions we want to attend and 0.0 for
|
| 1042 |
+
# masked positions, this operation will create a tensor which is 0.0 for
|
| 1043 |
+
# positions we want to attend and -10000.0 for masked positions.
|
| 1044 |
+
# Since we are adding it to the raw scores before the softmax, this is
|
| 1045 |
+
# effectively the same as removing these entirely.
|
| 1046 |
+
extended_attention_mask = extended_attention_mask.to(dtype=self.dtype) # fp16 compatibility
|
| 1047 |
+
extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
|
| 1048 |
+
return extended_attention_mask
|
| 1049 |
+
|
| 1050 |
+
def forward(
|
| 1051 |
+
self,
|
| 1052 |
+
query_embeds,
|
| 1053 |
+
temporal_query_embeds=None,
|
| 1054 |
+
attention_mask=None,
|
| 1055 |
+
head_mask=None,
|
| 1056 |
+
encoder_hidden_states=None,
|
| 1057 |
+
encoder_attention_mask=None,
|
| 1058 |
+
past_key_values=None,
|
| 1059 |
+
output_attentions=None,
|
| 1060 |
+
output_hidden_states=None,
|
| 1061 |
+
return_dict=None,
|
| 1062 |
+
):
|
| 1063 |
+
r"""
|
| 1064 |
+
encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`):
|
| 1065 |
+
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
|
| 1066 |
+
the model is configured as a decoder.
|
| 1067 |
+
encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, `optional`):
|
| 1068 |
+
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
|
| 1069 |
+
the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:
|
| 1070 |
+
- 1 for tokens that are **not masked**,
|
| 1071 |
+
- 0 for tokens that are **masked**.
|
| 1072 |
+
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of:
|
| 1073 |
+
shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and
|
| 1074 |
+
value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are
|
| 1075 |
+
used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key
|
| 1076 |
+
value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape
|
| 1077 |
+
`(batch_size, sequence_length)`.
|
| 1078 |
+
"""
|
| 1079 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1080 |
+
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1081 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1082 |
+
|
| 1083 |
+
T = encoder_hidden_states.size(1)
|
| 1084 |
+
if T == 1 or temporal_query_embeds is None:
|
| 1085 |
+
embedding_output = query_embeds
|
| 1086 |
+
else:
|
| 1087 |
+
embedding_output = torch.cat([query_embeds, temporal_query_embeds], dim=1)
|
| 1088 |
+
input_shape = embedding_output.size()[:-1]
|
| 1089 |
+
batch_size, seq_length = input_shape
|
| 1090 |
+
device = embedding_output.device
|
| 1091 |
+
|
| 1092 |
+
encoder_hidden_states = einops.rearrange(encoder_hidden_states, "b t n d -> b (t n) d")
|
| 1093 |
+
|
| 1094 |
+
# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
|
| 1095 |
+
# ourselves in which case we just need to make it broadcastable to all heads.
|
| 1096 |
+
if attention_mask is None:
|
| 1097 |
+
attention_mask = torch.ones((embedding_output.shape[0], embedding_output.shape[1]), dtype=torch.long, device=embedding_output.device)
|
| 1098 |
+
extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, device)
|
| 1099 |
+
|
| 1100 |
+
# If a 2D or 3D attention mask is provided for the cross-attention
|
| 1101 |
+
# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
|
| 1102 |
+
if encoder_hidden_states is not None:
|
| 1103 |
+
if type(encoder_hidden_states) == list:
|
| 1104 |
+
encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
|
| 1105 |
+
else:
|
| 1106 |
+
(
|
| 1107 |
+
encoder_batch_size,
|
| 1108 |
+
encoder_sequence_length,
|
| 1109 |
+
_,
|
| 1110 |
+
) = encoder_hidden_states.size()
|
| 1111 |
+
encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
|
| 1112 |
+
|
| 1113 |
+
if type(encoder_attention_mask) == list:
|
| 1114 |
+
encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
|
| 1115 |
+
elif encoder_attention_mask is None:
|
| 1116 |
+
encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
|
| 1117 |
+
encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
|
| 1118 |
+
else:
|
| 1119 |
+
encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
|
| 1120 |
+
else:
|
| 1121 |
+
encoder_extended_attention_mask = None
|
| 1122 |
+
|
| 1123 |
+
# Prepare head mask if needed
|
| 1124 |
+
# 1.0 in head_mask indicate we keep the head
|
| 1125 |
+
# attention_probs has shape bsz x n_heads x N x N
|
| 1126 |
+
# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
|
| 1127 |
+
# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
|
| 1128 |
+
head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
|
| 1129 |
+
|
| 1130 |
+
encoder_outputs = self.encoder(
|
| 1131 |
+
embedding_output,
|
| 1132 |
+
attention_mask=extended_attention_mask,
|
| 1133 |
+
head_mask=head_mask,
|
| 1134 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 1135 |
+
encoder_attention_mask=encoder_extended_attention_mask,
|
| 1136 |
+
past_key_values=past_key_values,
|
| 1137 |
+
output_attentions=output_attentions,
|
| 1138 |
+
output_hidden_states=output_hidden_states,
|
| 1139 |
+
return_dict=return_dict,
|
| 1140 |
+
)
|
| 1141 |
+
sequence_output = encoder_outputs[0]
|
| 1142 |
+
pooled_output = sequence_output[:, 0, :]
|
| 1143 |
+
|
| 1144 |
+
if T == 1 or temporal_query_embeds is None:
|
| 1145 |
+
temporal_sequence_output = None
|
| 1146 |
+
else:
|
| 1147 |
+
temporal_sequence_output = sequence_output[:, query_embeds.size(1) :]
|
| 1148 |
+
sequence_output = sequence_output[:, : query_embeds.size(1)]
|
| 1149 |
+
|
| 1150 |
+
sequence_output = self.visual_fc(sequence_output)
|
| 1151 |
+
if temporal_sequence_output is not None:
|
| 1152 |
+
sequence_output += self.temporal_visual_fc(temporal_sequence_output)
|
| 1153 |
+
sequence_output = torch.cat([sequence_output, self.vit_eos.repeat(sequence_output.shape[0], 1, 1)], dim=1)
|
| 1154 |
+
|
| 1155 |
+
return BaseModelOutputWithPooling(
|
| 1156 |
+
last_hidden_state=sequence_output,
|
| 1157 |
+
pooler_output=pooled_output,
|
| 1158 |
+
hidden_states=encoder_outputs.hidden_states,
|
| 1159 |
+
)
|
| 1160 |
+
|
| 1161 |
+
|
| 1162 |
+
@add_start_docstrings(
|
| 1163 |
+
"""
|
| 1164 |
+
mPLUG-Owl Model for generating text and image features. The model consists of a vision encoder, Querying Transformer
|
| 1165 |
+
(Q-Former) and a language model.
|
| 1166 |
+
""",
|
| 1167 |
+
MPLUG_OWL_START_DOCSTRING,
|
| 1168 |
+
)
|
| 1169 |
+
class MplugOwlModel(MplugOwlPreTrainedModel):
|
| 1170 |
+
config_class = MplugOwlConfig
|
| 1171 |
+
main_input_name = "pixel_values"
|
| 1172 |
+
|
| 1173 |
+
def __init__(self, config: MplugOwlConfig, *inputs, **kwargs):
|
| 1174 |
+
super().__init__(config, *inputs, **kwargs)
|
| 1175 |
+
|
| 1176 |
+
self.vision_model = MplugOwlVisionModel(config.vision_config)
|
| 1177 |
+
|
| 1178 |
+
self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
|
| 1179 |
+
self.temporal_query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
|
| 1180 |
+
self.abstractor = MplugOwlVisualAbstractorModel(config.visual_abstractor_config, config.text_config.hidden_size)
|
| 1181 |
+
|
| 1182 |
+
# if config.use_decoder_only_language_model:
|
| 1183 |
+
# from llama.modeling_llama import LlamaForCausalLM
|
| 1184 |
+
language_model = AutoModelForCausalLM.from_config(config.text_config)
|
| 1185 |
+
# else:
|
| 1186 |
+
# language_model = AutoModelForSeq2SeqLM.from_config(config.text_config)
|
| 1187 |
+
self.language_model = language_model
|
| 1188 |
+
|
| 1189 |
+
# Initialize weights and apply final processing
|
| 1190 |
+
self.post_init()
|
| 1191 |
+
|
| 1192 |
+
def get_input_embeddings(self):
|
| 1193 |
+
return self.language_model.get_input_embeddings()
|
| 1194 |
+
|
| 1195 |
+
def set_input_embeddings(self, value):
|
| 1196 |
+
self.language_model.set_input_embeddings(value)
|
| 1197 |
+
|
| 1198 |
+
def set_output_embeddings(self, new_embeddings):
|
| 1199 |
+
self.language_model.set_output_embeddings(new_embeddings)
|
| 1200 |
+
|
| 1201 |
+
def get_output_embeddings(self) -> nn.Module:
|
| 1202 |
+
return self.language_model.get_output_embeddings()
|
| 1203 |
+
|
| 1204 |
+
def get_encoder(self):
|
| 1205 |
+
return self.language_model.get_encoder()
|
| 1206 |
+
|
| 1207 |
+
def get_decoder(self):
|
| 1208 |
+
return self.language_model.get_decoder()
|
| 1209 |
+
|
| 1210 |
+
def _tie_weights(self):
|
| 1211 |
+
if not self.config.use_decoder_only_language_model:
|
| 1212 |
+
self.language_model.encoder.embed_tokens = self.language_model.shared
|
| 1213 |
+
self.language_model.decoder.embed_tokens = self.language_model.shared
|
| 1214 |
+
|
| 1215 |
+
def get_text_features(
|
| 1216 |
+
self,
|
| 1217 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 1218 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1219 |
+
decoder_input_ids: Optional[torch.Tensor] = None,
|
| 1220 |
+
decoder_attention_mask: Optional[torch.Tensor] = None,
|
| 1221 |
+
labels: Optional[torch.Tensor] = None,
|
| 1222 |
+
output_attentions: Optional[bool] = None,
|
| 1223 |
+
output_hidden_states: Optional[bool] = None,
|
| 1224 |
+
return_dict: Optional[bool] = None,
|
| 1225 |
+
):
|
| 1226 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1227 |
+
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1228 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1229 |
+
|
| 1230 |
+
if self.config.use_decoder_only_language_model:
|
| 1231 |
+
text_outputs = self.language_model(
|
| 1232 |
+
input_ids=input_ids,
|
| 1233 |
+
attention_mask=attention_mask,
|
| 1234 |
+
output_attentions=output_attentions,
|
| 1235 |
+
output_hidden_states=output_hidden_states,
|
| 1236 |
+
return_dict=return_dict,
|
| 1237 |
+
)
|
| 1238 |
+
else:
|
| 1239 |
+
inputs_embeds = self.language_model.get_input_embeddings()(input_ids)
|
| 1240 |
+
|
| 1241 |
+
text_outputs = self.language_model(
|
| 1242 |
+
inputs_embeds=inputs_embeds,
|
| 1243 |
+
attention_mask=attention_mask,
|
| 1244 |
+
decoder_input_ids=decoder_input_ids,
|
| 1245 |
+
decoder_attention_mask=decoder_attention_mask,
|
| 1246 |
+
output_attentions=output_attentions,
|
| 1247 |
+
output_hidden_states=output_hidden_states,
|
| 1248 |
+
return_dict=return_dict,
|
| 1249 |
+
labels=labels,
|
| 1250 |
+
)
|
| 1251 |
+
|
| 1252 |
+
return text_outputs
|
| 1253 |
+
|
| 1254 |
+
def get_image_features(
|
| 1255 |
+
self,
|
| 1256 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 1257 |
+
output_attentions: Optional[bool] = None,
|
| 1258 |
+
output_hidden_states: Optional[bool] = None,
|
| 1259 |
+
return_dict: Optional[bool] = None,
|
| 1260 |
+
):
|
| 1261 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1262 |
+
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1263 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1264 |
+
|
| 1265 |
+
vision_outputs = self.vision_model(
|
| 1266 |
+
pixel_values=pixel_values,
|
| 1267 |
+
output_attentions=output_attentions,
|
| 1268 |
+
output_hidden_states=output_hidden_states,
|
| 1269 |
+
return_dict=return_dict,
|
| 1270 |
+
)
|
| 1271 |
+
|
| 1272 |
+
return vision_outputs
|
| 1273 |
+
|
| 1274 |
+
|
| 1275 |
+
def get_media_indices(my_list):
|
| 1276 |
+
if isinstance(my_list, torch.Tensor):
|
| 1277 |
+
my_list = my_list.cpu().tolist()
|
| 1278 |
+
result = []
|
| 1279 |
+
for i in range(len(my_list)):
|
| 1280 |
+
if i == 0 and my_list[i] < 0:
|
| 1281 |
+
result.append(i)
|
| 1282 |
+
elif my_list[i] != my_list[i - 1] and my_list[i] < 0:
|
| 1283 |
+
result.append(i)
|
| 1284 |
+
return result
|
| 1285 |
+
|
| 1286 |
+
|
| 1287 |
+
def get_media_types(tensors, positions):
|
| 1288 |
+
if isinstance(tensors, torch.Tensor):
|
| 1289 |
+
tensors = tensors.cpu().tolist()
|
| 1290 |
+
result = []
|
| 1291 |
+
for pos in positions:
|
| 1292 |
+
result.append(tensors[pos])
|
| 1293 |
+
return result
|
| 1294 |
+
|
| 1295 |
+
|
| 1296 |
+
@add_start_docstrings(
|
| 1297 |
+
"""
|
| 1298 |
+
mPLUG-Owl Model for generating text given an image and an optional text prompt.
|
| 1299 |
+
""",
|
| 1300 |
+
MPLUG_OWL_START_DOCSTRING,
|
| 1301 |
+
)
|
| 1302 |
+
class MplugOwlForConditionalGeneration(MplugOwlPreTrainedModel):
|
| 1303 |
+
config_class = MplugOwlConfig
|
| 1304 |
+
main_input_name = "pixel_values"
|
| 1305 |
+
|
| 1306 |
+
def __init__(self, config: MplugOwlConfig):
|
| 1307 |
+
super().__init__(config)
|
| 1308 |
+
|
| 1309 |
+
self.vision_model = MplugOwlVisionModel(config.vision_config)
|
| 1310 |
+
|
| 1311 |
+
self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
|
| 1312 |
+
self.temporal_query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
|
| 1313 |
+
self.abstractor = MplugOwlVisualAbstractorModel(config.visual_abstractor_config, config.text_config.hidden_size)
|
| 1314 |
+
|
| 1315 |
+
# if config.use_decoder_only_language_model:
|
| 1316 |
+
# from llama.modeling_llama import LlamaForCausalLM
|
| 1317 |
+
language_model = AutoModelForCausalLM.from_config(config.text_config)
|
| 1318 |
+
# else:
|
| 1319 |
+
# language_model = AutoModelForSeq2SeqLM.from_config(config.text_config)
|
| 1320 |
+
self.language_model = language_model
|
| 1321 |
+
|
| 1322 |
+
# Initialize weights and apply final processing
|
| 1323 |
+
self.post_init()
|
| 1324 |
+
self.main_input_name = "input_ids"
|
| 1325 |
+
from transformers import GenerationConfig
|
| 1326 |
+
|
| 1327 |
+
self.generation_config = GenerationConfig(max_length=512, do_sample=True, top_k=3, pad_token_id=0, unk_token_id=0, bos_token_id=1, eos_token_id=2)
|
| 1328 |
+
|
| 1329 |
+
# Hack Bloom
|
| 1330 |
+
if config.text_config.model_type == "bloom":
|
| 1331 |
+
bound_method = bloom_forward.__get__(self.language_model.transformer, self.language_model.transformer.__class__)
|
| 1332 |
+
setattr(self.language_model.transformer, "forward", bound_method)
|
| 1333 |
+
|
| 1334 |
+
def get_input_embeddings(self):
|
| 1335 |
+
return self.language_model.get_input_embeddings()
|
| 1336 |
+
|
| 1337 |
+
def set_input_embeddings(self, value):
|
| 1338 |
+
self.language_model.set_input_embeddings(value)
|
| 1339 |
+
|
| 1340 |
+
def set_output_embeddings(self, new_embeddings):
|
| 1341 |
+
self.language_model.set_output_embeddings(new_embeddings)
|
| 1342 |
+
|
| 1343 |
+
def get_output_embeddings(self) -> nn.Module:
|
| 1344 |
+
return self.language_model.get_output_embeddings()
|
| 1345 |
+
|
| 1346 |
+
def get_encoder(self):
|
| 1347 |
+
return self.language_model.get_encoder()
|
| 1348 |
+
|
| 1349 |
+
def get_decoder(self):
|
| 1350 |
+
return self.language_model.get_decoder()
|
| 1351 |
+
|
| 1352 |
+
def _tie_weights(self):
|
| 1353 |
+
if not self.config.use_decoder_only_language_model:
|
| 1354 |
+
self.language_model.encoder.embed_tokens = self.language_model.shared
|
| 1355 |
+
self.language_model.decoder.embed_tokens = self.language_model.shared
|
| 1356 |
+
|
| 1357 |
+
def _preprocess_accelerate(self):
|
| 1358 |
+
r"""
|
| 1359 |
+
Some pre-processing hacks to make the model `accelerate` compatible. Check
|
| 1360 |
+
https://github.com/huggingface/transformers/pull/21707 for more details.
|
| 1361 |
+
"""
|
| 1362 |
+
hf_device_map = self.hf_device_map
|
| 1363 |
+
|
| 1364 |
+
if len(hf_device_map) > 1 and "language_model" not in hf_device_map and torch.cuda.device_count() > 1:
|
| 1365 |
+
# warn users about unexpected behavior when using multi-GPU + mPLUG-Owl + `accelerate`.
|
| 1366 |
+
logger.warning(
|
| 1367 |
+
"The `language_model` is not in the `hf_device_map` dictionary and you are running your script"
|
| 1368 |
+
" in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`."
|
| 1369 |
+
" Please pass a `device_map` that contains `language_model` to remove this warning."
|
| 1370 |
+
" Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for"
|
| 1371 |
+
" more details on creating a `device_map` for large models.",
|
| 1372 |
+
)
|
| 1373 |
+
|
| 1374 |
+
if hasattr(self.language_model, "_hf_hook"):
|
| 1375 |
+
self.language_model._hf_hook.io_same_device = True # For `generate` compatibility
|
| 1376 |
+
|
| 1377 |
+
@add_start_docstrings_to_model_forward(MPLUG_OWL_INPUTS_DOCSTRING)
|
| 1378 |
+
@replace_return_docstrings(output_type=MplugOwlForConditionalGenerationModelOutput, config_class=MplugOwlVisionConfig)
|
| 1379 |
+
def forward(
|
| 1380 |
+
self,
|
| 1381 |
+
pixel_values: torch.FloatTensor,
|
| 1382 |
+
video_pixel_values: torch.FloatTensor,
|
| 1383 |
+
input_ids: torch.FloatTensor,
|
| 1384 |
+
num_images,
|
| 1385 |
+
num_videos,
|
| 1386 |
+
non_padding_mask: Optional[torch.LongTensor] = None,
|
| 1387 |
+
non_media_mask: Optional[torch.LongTensor] = None,
|
| 1388 |
+
prompt_mask: Optional[torch.LongTensor] = None,
|
| 1389 |
+
attention_mask: Optional[torch.LongTensor] = None,
|
| 1390 |
+
decoder_input_ids: Optional[torch.LongTensor] = None,
|
| 1391 |
+
decoder_attention_mask: Optional[torch.LongTensor] = None,
|
| 1392 |
+
output_attentions: Optional[bool] = None,
|
| 1393 |
+
output_hidden_states: Optional[bool] = None,
|
| 1394 |
+
labels: Optional[torch.LongTensor] = None,
|
| 1395 |
+
return_dict: Optional[bool] = None,
|
| 1396 |
+
) -> Union[Tuple, MplugOwlForConditionalGenerationModelOutput]:
|
| 1397 |
+
r"""
|
| 1398 |
+
Returns:
|
| 1399 |
+
|
| 1400 |
+
Examples:
|
| 1401 |
+
|
| 1402 |
+
Image captioning (without providing a text prompt):
|
| 1403 |
+
|
| 1404 |
+
```python
|
| 1405 |
+
>>> from PIL import Image
|
| 1406 |
+
>>> import requests
|
| 1407 |
+
>>> from transformers import MplugOwlProcessor, MplugOwlForConditionalGeneration
|
| 1408 |
+
>>> import torch
|
| 1409 |
+
|
| 1410 |
+
>>> device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 1411 |
+
|
| 1412 |
+
>>> processor = MplugOwlProcessor.from_pretrained("x-plug/x_plug-llama-7b")
|
| 1413 |
+
>>> model = MplugOwlForConditionalGeneration.from_pretrained(
|
| 1414 |
+
... "x-plug/x_plug-llama-7b", torch_dtype=torch.float16
|
| 1415 |
+
... )
|
| 1416 |
+
>>> model.to(device) # doctest: +IGNORE_RESULT
|
| 1417 |
+
|
| 1418 |
+
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
| 1419 |
+
>>> image = Image.open(requests.get(url, stream=True).raw)
|
| 1420 |
+
|
| 1421 |
+
>>> inputs = processor(images=image, return_tensors="pt").to(device, torch.float16)
|
| 1422 |
+
|
| 1423 |
+
>>> generated_ids = model.generate(**inputs)
|
| 1424 |
+
>>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
|
| 1425 |
+
>>> print(generated_text)
|
| 1426 |
+
two cats laying on a couch
|
| 1427 |
+
```
|
| 1428 |
+
|
| 1429 |
+
Visual question answering (prompt = question):
|
| 1430 |
+
|
| 1431 |
+
```python
|
| 1432 |
+
>>> from PIL import Image
|
| 1433 |
+
>>> import requests
|
| 1434 |
+
>>> from transformers import MplugOwlProcessor, MplugOwlForConditionalGeneration
|
| 1435 |
+
>>> import torch
|
| 1436 |
+
|
| 1437 |
+
>>> device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 1438 |
+
|
| 1439 |
+
>>> processor = MplugOwlProcessor.from_pretrained("x-plug/x_plug-llama-7b")
|
| 1440 |
+
>>> model = MplugOwlForConditionalGeneration.from_pretrained(
|
| 1441 |
+
... "x-plug/x_plug-llama-7b", torch_dtype=torch.float16
|
| 1442 |
+
... )
|
| 1443 |
+
>>> model.to(device) # doctest: +IGNORE_RESULT
|
| 1444 |
+
|
| 1445 |
+
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
| 1446 |
+
>>> image = Image.open(requests.get(url, stream=True).raw)
|
| 1447 |
+
|
| 1448 |
+
>>> prompt = "Question: how many cats are there? Answer:"
|
| 1449 |
+
>>> inputs = processor(images=image, text=prompt, return_tensors="pt").to(device, torch.float16)
|
| 1450 |
+
|
| 1451 |
+
>>> generated_ids = model.generate(**inputs)
|
| 1452 |
+
>>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
|
| 1453 |
+
>>> print(generated_text)
|
| 1454 |
+
two
|
| 1455 |
+
```"""
|
| 1456 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1457 |
+
|
| 1458 |
+
# get text embedding
|
| 1459 |
+
text_tokens_ = input_ids.clone()
|
| 1460 |
+
batch_size = input_ids.shape[0]
|
| 1461 |
+
# labels = text_tokens_[:, 1:].clone().contiguous()
|
| 1462 |
+
|
| 1463 |
+
media_token_indices = [
|
| 1464 |
+
# [:-1] since we would not use the last token for embedding
|
| 1465 |
+
get_media_indices(text_tokens_[i][:-1])
|
| 1466 |
+
for i in range(batch_size)
|
| 1467 |
+
]
|
| 1468 |
+
|
| 1469 |
+
media_token_types = [get_media_types(text_tokens_[i][:-1], media_token_indices[i]) for i in range(batch_size)]
|
| 1470 |
+
|
| 1471 |
+
text_tokens_[text_tokens_ < 0] = 1 # Not used
|
| 1472 |
+
# text_tokens = text_tokens_[:, :-1].contiguous()
|
| 1473 |
+
text_embeds = self.get_input_embeddings()(text_tokens_) # Temporally Embedding
|
| 1474 |
+
|
| 1475 |
+
if pixel_values is not None:
|
| 1476 |
+
image_embeds = self.vision_model(pixel_values, return_dict=True).last_hidden_state
|
| 1477 |
+
|
| 1478 |
+
image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
|
| 1479 |
+
query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
|
| 1480 |
+
temporal_query_tokens = self.temporal_query_tokens.expand(image_embeds.shape[0], -1, -1)
|
| 1481 |
+
|
| 1482 |
+
query_features = self.abstractor(
|
| 1483 |
+
query_embeds=query_tokens,
|
| 1484 |
+
encoder_hidden_states=image_embeds,
|
| 1485 |
+
encoder_attention_mask=image_attention_mask,
|
| 1486 |
+
)["last_hidden_state"]
|
| 1487 |
+
img_seq_length = query_features.shape[1]
|
| 1488 |
+
|
| 1489 |
+
if video_pixel_values is not None:
|
| 1490 |
+
video_embeds = self.vision_model(video_pixel_values, return_dict=True).last_hidden_state
|
| 1491 |
+
|
| 1492 |
+
video_attention_mask = torch.ones(video_embeds.size()[:-1], dtype=torch.long, device=video_embeds.device)
|
| 1493 |
+
video_attention_mask = einops.rearrange(video_attention_mask, "b t n -> b (t n)")
|
| 1494 |
+
query_tokens = self.query_tokens.expand(video_embeds.shape[0], -1, -1)
|
| 1495 |
+
temporal_query_tokens = self.temporal_query_tokens.expand(video_embeds.shape[0], -1, -1)
|
| 1496 |
+
|
| 1497 |
+
video_query_features = self.abstractor(
|
| 1498 |
+
query_embeds=query_tokens,
|
| 1499 |
+
temporal_query_embeds=temporal_query_tokens,
|
| 1500 |
+
encoder_hidden_states=video_embeds,
|
| 1501 |
+
encoder_attention_mask=video_attention_mask,
|
| 1502 |
+
)["last_hidden_state"]
|
| 1503 |
+
vid_seq_length = video_query_features.shape[1]
|
| 1504 |
+
|
| 1505 |
+
num_images_per_sample = num_images.long().cpu().tolist()
|
| 1506 |
+
num_videos_per_sample = num_videos.long().cpu().tolist()
|
| 1507 |
+
|
| 1508 |
+
text_chunk_embeds = []
|
| 1509 |
+
img_idx = 0
|
| 1510 |
+
for b in range(batch_size):
|
| 1511 |
+
start = 0
|
| 1512 |
+
result = []
|
| 1513 |
+
if len(media_token_indices[b]) > 0:
|
| 1514 |
+
for i, pos in enumerate(media_token_indices[b]):
|
| 1515 |
+
if pos > start:
|
| 1516 |
+
result.append(text_embeds[b, start:pos])
|
| 1517 |
+
result.append(query_features[img_idx + i])
|
| 1518 |
+
start = pos + img_seq_length
|
| 1519 |
+
if start < text_embeds.shape[1]:
|
| 1520 |
+
result.append(text_embeds[b, start:])
|
| 1521 |
+
|
| 1522 |
+
img_idx += num_images_per_sample[b]
|
| 1523 |
+
text_chunk_embeds.append(torch.cat(result, dim=0))
|
| 1524 |
+
|
| 1525 |
+
# Actual Input Embeddings
|
| 1526 |
+
input_embeds = torch.stack(text_chunk_embeds, dim=0)
|
| 1527 |
+
|
| 1528 |
+
# if pixel_values is None and self.language_model.is_gradient_checkpointing:
|
| 1529 |
+
# # Hack here when gradient checkpoint is enable.
|
| 1530 |
+
# # Keep the compute graph static
|
| 1531 |
+
# image_embeds = self.vision_model(torch.zeros(1,3,224,224,device=input_embeds.device,dtype=input_embeds.dtype), return_dict=True).last_hidden_state
|
| 1532 |
+
# query_tokens = self.query_tokens.expand(
|
| 1533 |
+
# image_embeds.shape[0], -1, -1)
|
| 1534 |
+
# query_features = self.abstractor(query_embeds=query_tokens,
|
| 1535 |
+
# encoder_hidden_states=image_embeds,)['last_hidden_state']
|
| 1536 |
+
|
| 1537 |
+
# input_embeds = input_embeds + query_features.mean()*0
|
| 1538 |
+
|
| 1539 |
+
# Create causal mask and position ids
|
| 1540 |
+
_, loss_mask, position_ids = get_ltor_masks_and_position_ids_from_embeddings(input_embeds)
|
| 1541 |
+
|
| 1542 |
+
# Calculate the loss_mask
|
| 1543 |
+
non_padding_mask = non_padding_mask.long()
|
| 1544 |
+
non_media_mask = non_media_mask.long()
|
| 1545 |
+
prompt_mask = prompt_mask.long() # TODO How to deal with prompt mask
|
| 1546 |
+
# from icecream import ic
|
| 1547 |
+
# non_padding_mask = non_padding_mask[:,:-1]
|
| 1548 |
+
# non_media_mask = non_media_mask[:,:-1]
|
| 1549 |
+
# prompt_mask = prompt_mask[:,:-1]
|
| 1550 |
+
# attention_mask = attention_mask[:,:-1]
|
| 1551 |
+
loss_mask = loss_mask[:, :-1]
|
| 1552 |
+
|
| 1553 |
+
loss_mask = loss_mask * non_padding_mask * non_media_mask * prompt_mask
|
| 1554 |
+
labels[:, 1:][loss_mask != 1] = -100
|
| 1555 |
+
# Forward into GPT
|
| 1556 |
+
outputs = self.language_model(
|
| 1557 |
+
inputs_embeds=input_embeds,
|
| 1558 |
+
attention_mask=attention_mask,
|
| 1559 |
+
labels=labels,
|
| 1560 |
+
return_dict=return_dict,
|
| 1561 |
+
output_attentions=self.config.output_attentions,
|
| 1562 |
+
)
|
| 1563 |
+
# outputs.loss = (outputs.loss * loss_mask.view(-1)
|
| 1564 |
+
# ).sum()/loss_mask.sum()
|
| 1565 |
+
return outputs
|
| 1566 |
+
|
| 1567 |
+
@torch.no_grad()
|
| 1568 |
+
def generate(
|
| 1569 |
+
self,
|
| 1570 |
+
pixel_values: torch.FloatTensor = None,
|
| 1571 |
+
video_pixel_values: torch.FloatTensor = None,
|
| 1572 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 1573 |
+
attention_mask: Optional[torch.LongTensor] = None,
|
| 1574 |
+
isdecoder=True,
|
| 1575 |
+
**generate_kwargs,
|
| 1576 |
+
) -> torch.LongTensor:
|
| 1577 |
+
"""
|
| 1578 |
+
Overrides `generate` function to be able to use the model as a conditional generator.
|
| 1579 |
+
|
| 1580 |
+
Args:
|
| 1581 |
+
pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width)):
|
| 1582 |
+
Input images to be processed.
|
| 1583 |
+
input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
|
| 1584 |
+
The sequence used as a prompt for the generation.
|
| 1585 |
+
attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
|
| 1586 |
+
Mask to avoid performing attention on padding token indices
|
| 1587 |
+
|
| 1588 |
+
Returns:
|
| 1589 |
+
captions (list): A list of strings of length batch_size * num_captions.
|
| 1590 |
+
"""
|
| 1591 |
+
if input_ids is None:
|
| 1592 |
+
return self.language_model.generate(attention_mask=attention_mask, **generate_kwargs)
|
| 1593 |
+
|
| 1594 |
+
if attention_mask is None:
|
| 1595 |
+
attention_mask = input_ids.new_ones(*input_ids.shape)
|
| 1596 |
+
|
| 1597 |
+
batch_size = input_ids.size(0)
|
| 1598 |
+
media_token_indices = [get_media_indices(input_ids[i]) for i in range(batch_size)]
|
| 1599 |
+
media_token_types = [get_media_types(input_ids[i], media_token_indices[i]) for i in range(batch_size)]
|
| 1600 |
+
num_images_per_sample = [len([y for y in x if y == -1]) for x in media_token_types]
|
| 1601 |
+
num_videos_per_sample = [len([y for y in x if y < -1]) for x in media_token_types]
|
| 1602 |
+
input_ids = input_ids.clone() # prevent inplace modify
|
| 1603 |
+
input_ids[input_ids < 0] = 0 # Not used
|
| 1604 |
+
|
| 1605 |
+
if hasattr(self, "hf_device_map"):
|
| 1606 |
+
# preprocess for `accelerate`
|
| 1607 |
+
self._preprocess_accelerate()
|
| 1608 |
+
batch_size = input_ids.shape[0]
|
| 1609 |
+
# get text embedding
|
| 1610 |
+
inputs_embeds = self.get_input_embeddings()(input_ids)
|
| 1611 |
+
if hasattr(self.language_model, "transformer") and hasattr(self.language_model.transformer, "word_embeddings_layernorm"):
|
| 1612 |
+
inputs_embeds = self.language_model.transformer.word_embeddings_layernorm(inputs_embeds)
|
| 1613 |
+
# get visual embedding
|
| 1614 |
+
if pixel_values is not None:
|
| 1615 |
+
pixel_values = pixel_values.to(input_ids.device)
|
| 1616 |
+
with torch.no_grad():
|
| 1617 |
+
image_embeds = self.vision_model(pixel_values, return_dict=True).last_hidden_state
|
| 1618 |
+
image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
|
| 1619 |
+
query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
|
| 1620 |
+
query_outputs = self.abstractor(
|
| 1621 |
+
query_embeds=query_tokens,
|
| 1622 |
+
encoder_hidden_states=image_embeds,
|
| 1623 |
+
encoder_attention_mask=image_attention_mask,
|
| 1624 |
+
return_dict=True,
|
| 1625 |
+
)
|
| 1626 |
+
query_output = query_outputs["last_hidden_state"]
|
| 1627 |
+
image_embeds = query_output
|
| 1628 |
+
img_seq_length = image_embeds.shape[1]
|
| 1629 |
+
|
| 1630 |
+
if video_pixel_values is not None:
|
| 1631 |
+
video_pixel_values = video_pixel_values.to(input_ids.device)
|
| 1632 |
+
with torch.no_grad():
|
| 1633 |
+
video_embeds = self.vision_model(video_pixel_values, return_dict=True).last_hidden_state
|
| 1634 |
+
video_attention_mask = torch.ones(video_embeds.size()[:-1], dtype=torch.long, device=video_embeds.device)
|
| 1635 |
+
video_attention_mask = einops.rearrange(video_attention_mask, "b t n -> b (t n)")
|
| 1636 |
+
query_tokens = self.query_tokens.expand(video_embeds.shape[0], -1, -1)
|
| 1637 |
+
temporal_query_tokens = self.temporal_query_tokens.expand(video_embeds.shape[0], -1, -1)
|
| 1638 |
+
query_outputs = self.abstractor(
|
| 1639 |
+
query_embeds=query_tokens,
|
| 1640 |
+
temporal_query_embeds=temporal_query_tokens,
|
| 1641 |
+
encoder_hidden_states=video_embeds,
|
| 1642 |
+
encoder_attention_mask=video_attention_mask,
|
| 1643 |
+
return_dict=True,
|
| 1644 |
+
)
|
| 1645 |
+
query_output = query_outputs["last_hidden_state"]
|
| 1646 |
+
video_embeds = query_output
|
| 1647 |
+
vid_seq_length = video_embeds.shape[1]
|
| 1648 |
+
|
| 1649 |
+
# ===================
|
| 1650 |
+
# Get actual input embeddings
|
| 1651 |
+
# ===================
|
| 1652 |
+
text_chunk_embeds = []
|
| 1653 |
+
text_chunk_attns = []
|
| 1654 |
+
img_idx = 0
|
| 1655 |
+
vid_idx = 0
|
| 1656 |
+
|
| 1657 |
+
for b in range(batch_size):
|
| 1658 |
+
start = 0
|
| 1659 |
+
result = []
|
| 1660 |
+
result_attn = []
|
| 1661 |
+
for i, pos in enumerate(media_token_indices[b]):
|
| 1662 |
+
curr_image_idx, curr_video_idx = 0, 0
|
| 1663 |
+
if pos > start:
|
| 1664 |
+
result.append(inputs_embeds[b, start:pos])
|
| 1665 |
+
result_attn.append(attention_mask[b, start:pos])
|
| 1666 |
+
if media_token_types[b][i] == -1:
|
| 1667 |
+
result.append(image_embeds[img_idx + curr_image_idx])
|
| 1668 |
+
result_attn.append(torch.ones(image_embeds[img_idx + curr_image_idx].shape[0], device=inputs_embeds.device))
|
| 1669 |
+
start = pos + img_seq_length
|
| 1670 |
+
curr_image_idx += 1
|
| 1671 |
+
else:
|
| 1672 |
+
result.append(video_embeds[vid_idx + curr_video_idx])
|
| 1673 |
+
result_attn.append(torch.ones(video_embeds[img_idx + curr_video_idx].shape[0], device=inputs_embeds.device))
|
| 1674 |
+
start = pos + vid_seq_length
|
| 1675 |
+
curr_video_idx += 1
|
| 1676 |
+
if start < inputs_embeds.shape[1]:
|
| 1677 |
+
result.append(inputs_embeds[b, start:])
|
| 1678 |
+
result_attn.append(attention_mask[b, start:])
|
| 1679 |
+
|
| 1680 |
+
img_idx += num_images_per_sample[b]
|
| 1681 |
+
vid_idx += num_videos_per_sample[b]
|
| 1682 |
+
text_chunk_embeds.append(torch.cat(result, dim=0))
|
| 1683 |
+
text_chunk_attns.append(torch.cat(result_attn, dim=0))
|
| 1684 |
+
inputs_embeds = torch.stack(text_chunk_embeds, dim=0)
|
| 1685 |
+
attention_mask = torch.stack(text_chunk_attns, dim=0)
|
| 1686 |
+
|
| 1687 |
+
outputs = self.language_model.generate(
|
| 1688 |
+
inputs_embeds=inputs_embeds,
|
| 1689 |
+
# input_ids=input_ids,
|
| 1690 |
+
attention_mask=attention_mask,
|
| 1691 |
+
**generate_kwargs,
|
| 1692 |
+
)
|
| 1693 |
+
|
| 1694 |
+
return outputs
|
| 1695 |
+
|
| 1696 |
+
def prepare_inputs_for_generation(self, input_ids, pixel_values=None, video_pixel_values=None, past_key_values=None, attention_mask=None, **model_kwargs):
|
| 1697 |
+
input_shape = input_ids.shape
|
| 1698 |
+
# if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
|
| 1699 |
+
if attention_mask is None:
|
| 1700 |
+
attention_mask = input_ids.new_ones(input_shape)
|
| 1701 |
+
|
| 1702 |
+
# # cut decoder_input_ids if past_key_values is used
|
| 1703 |
+
# if past_key_values is not None:
|
| 1704 |
+
# input_ids = input_ids[:, -1:]
|
| 1705 |
+
|
| 1706 |
+
return {
|
| 1707 |
+
"input_ids": input_ids,
|
| 1708 |
+
"pixel_values": pixel_values,
|
| 1709 |
+
"video_pixel_values": video_pixel_values,
|
| 1710 |
+
"attention_mask": attention_mask,
|
| 1711 |
+
# "past_key_values": past_key_values,
|
| 1712 |
+
# "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None),
|
| 1713 |
+
# "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None),
|
| 1714 |
+
"is_decoder": True,
|
| 1715 |
+
}
|
| 1716 |
+
|
| 1717 |
+
|
| 1718 |
+
def bloom_forward(
|
| 1719 |
+
self,
|
| 1720 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 1721 |
+
past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
|
| 1722 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1723 |
+
head_mask: Optional[torch.LongTensor] = None,
|
| 1724 |
+
inputs_embeds: Optional[torch.LongTensor] = None,
|
| 1725 |
+
use_cache: Optional[bool] = None,
|
| 1726 |
+
output_attentions: Optional[bool] = None,
|
| 1727 |
+
output_hidden_states: Optional[bool] = None,
|
| 1728 |
+
return_dict: Optional[bool] = None,
|
| 1729 |
+
**deprecated_arguments,
|
| 1730 |
+
) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
|
| 1731 |
+
if deprecated_arguments.pop("position_ids", False) is not False:
|
| 1732 |
+
# `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None`
|
| 1733 |
+
warnings.warn(
|
| 1734 |
+
"`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore" " passing `position_ids`.",
|
| 1735 |
+
FutureWarning,
|
| 1736 |
+
)
|
| 1737 |
+
if len(deprecated_arguments) > 0:
|
| 1738 |
+
raise ValueError(f"Got unexpected arguments: {deprecated_arguments}")
|
| 1739 |
+
|
| 1740 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1741 |
+
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1742 |
+
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
| 1743 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1744 |
+
|
| 1745 |
+
if input_ids is not None and inputs_embeds is not None:
|
| 1746 |
+
raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
|
| 1747 |
+
elif input_ids is not None:
|
| 1748 |
+
batch_size, seq_length = input_ids.shape
|
| 1749 |
+
elif inputs_embeds is not None:
|
| 1750 |
+
batch_size, seq_length, _ = inputs_embeds.shape
|
| 1751 |
+
else:
|
| 1752 |
+
raise ValueError("You have to specify either input_ids or inputs_embeds")
|
| 1753 |
+
|
| 1754 |
+
if past_key_values is None:
|
| 1755 |
+
past_key_values = tuple([None] * len(self.h))
|
| 1756 |
+
|
| 1757 |
+
# Prepare head mask if needed
|
| 1758 |
+
# 1.0 in head_mask indicate we keep the head
|
| 1759 |
+
# attention_probs has shape batch_size x num_heads x N x N
|
| 1760 |
+
# head_mask has shape n_layer x batch x num_heads x N x N
|
| 1761 |
+
head_mask = self.get_head_mask(head_mask, self.config.n_layer)
|
| 1762 |
+
|
| 1763 |
+
if inputs_embeds is None:
|
| 1764 |
+
inputs_embeds = self.word_embeddings(input_ids)
|
| 1765 |
+
inputs_embeds = self.word_embeddings_layernorm(inputs_embeds)
|
| 1766 |
+
|
| 1767 |
+
hidden_states = inputs_embeds
|
| 1768 |
+
|
| 1769 |
+
presents = () if use_cache else None
|
| 1770 |
+
all_self_attentions = () if output_attentions else None
|
| 1771 |
+
all_hidden_states = () if output_hidden_states else None
|
| 1772 |
+
|
| 1773 |
+
if self.gradient_checkpointing and self.training:
|
| 1774 |
+
if use_cache:
|
| 1775 |
+
logger.warning_once("`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...")
|
| 1776 |
+
use_cache = False
|
| 1777 |
+
|
| 1778 |
+
# Compute alibi tensor: check build_alibi_tensor documentation
|
| 1779 |
+
seq_length_with_past = seq_length
|
| 1780 |
+
past_key_values_length = 0
|
| 1781 |
+
if past_key_values[0] is not None:
|
| 1782 |
+
past_key_values_length = past_key_values[0][0].shape[2]
|
| 1783 |
+
seq_length_with_past = seq_length_with_past + past_key_values_length
|
| 1784 |
+
if attention_mask is None:
|
| 1785 |
+
attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
|
| 1786 |
+
else:
|
| 1787 |
+
attention_mask = attention_mask.to(hidden_states.device)
|
| 1788 |
+
|
| 1789 |
+
alibi = self.build_alibi_tensor(attention_mask, self.num_heads, dtype=hidden_states.dtype)
|
| 1790 |
+
|
| 1791 |
+
causal_mask = self._prepare_attn_mask(
|
| 1792 |
+
attention_mask,
|
| 1793 |
+
input_shape=(batch_size, seq_length),
|
| 1794 |
+
past_key_values_length=past_key_values_length,
|
| 1795 |
+
)
|
| 1796 |
+
|
| 1797 |
+
for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
|
| 1798 |
+
if output_hidden_states:
|
| 1799 |
+
all_hidden_states = all_hidden_states + (hidden_states,)
|
| 1800 |
+
|
| 1801 |
+
if self.gradient_checkpointing and self.training:
|
| 1802 |
+
|
| 1803 |
+
def create_custom_forward(module):
|
| 1804 |
+
def custom_forward(*inputs):
|
| 1805 |
+
# None for past_key_value
|
| 1806 |
+
return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)
|
| 1807 |
+
|
| 1808 |
+
return custom_forward
|
| 1809 |
+
|
| 1810 |
+
outputs = torch.utils.checkpoint.checkpoint(
|
| 1811 |
+
create_custom_forward(block),
|
| 1812 |
+
hidden_states,
|
| 1813 |
+
alibi,
|
| 1814 |
+
causal_mask,
|
| 1815 |
+
layer_past,
|
| 1816 |
+
head_mask[i],
|
| 1817 |
+
)
|
| 1818 |
+
else:
|
| 1819 |
+
outputs = block(
|
| 1820 |
+
hidden_states,
|
| 1821 |
+
layer_past=layer_past,
|
| 1822 |
+
attention_mask=causal_mask,
|
| 1823 |
+
head_mask=head_mask[i],
|
| 1824 |
+
use_cache=use_cache,
|
| 1825 |
+
output_attentions=output_attentions,
|
| 1826 |
+
alibi=alibi,
|
| 1827 |
+
)
|
| 1828 |
+
|
| 1829 |
+
hidden_states = outputs[0]
|
| 1830 |
+
if use_cache is True:
|
| 1831 |
+
presents = presents + (outputs[1],)
|
| 1832 |
+
|
| 1833 |
+
if output_attentions:
|
| 1834 |
+
all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
|
| 1835 |
+
|
| 1836 |
+
# Add last hidden state
|
| 1837 |
+
hidden_states = self.ln_f(hidden_states)
|
| 1838 |
+
|
| 1839 |
+
if output_hidden_states:
|
| 1840 |
+
all_hidden_states = all_hidden_states + (hidden_states,)
|
| 1841 |
+
|
| 1842 |
+
if not return_dict:
|
| 1843 |
+
return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
|
| 1844 |
+
|
| 1845 |
+
return BaseModelOutputWithPastAndCrossAttentions(
|
| 1846 |
+
last_hidden_state=hidden_states,
|
| 1847 |
+
past_key_values=presents,
|
| 1848 |
+
hidden_states=all_hidden_states,
|
| 1849 |
+
attentions=all_self_attentions,
|
| 1850 |
+
)
|
LaViDa/eval/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
import torch.utils.checkpoint
|
| 6 |
+
from decord import VideoReader
|
| 7 |
+
from PIL import Image
|
| 8 |
+
from transformers.models.clip.image_processing_clip import CLIPImageProcessor
|
| 9 |
+
from transformers.processing_utils import ProcessorMixin
|
| 10 |
+
from transformers.tokenization_utils_base import BatchEncoding
|
| 11 |
+
|
| 12 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 13 |
+
|
| 14 |
+
from .tokenization_mplug_owl import MplugOwlTokenizer
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def get_index(num_frames, num_segments):
|
| 18 |
+
seg_size = float(num_frames - 1) / num_segments
|
| 19 |
+
start = int(seg_size / 2)
|
| 20 |
+
offsets = np.array([start + int(np.round(seg_size * idx)) for idx in range(num_segments)])
|
| 21 |
+
return offsets
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def load_video(path, num_frames=4):
|
| 25 |
+
"""vr = VideoReader(path, height=224, width=224)
|
| 26 |
+
total_frames = len(vr)
|
| 27 |
+
frame_indices = get_index(total_frames, num_frames)
|
| 28 |
+
images_group = list()
|
| 29 |
+
for frame_index in frame_indices:
|
| 30 |
+
img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
|
| 31 |
+
images_group.append(img)
|
| 32 |
+
return images_group"""
|
| 33 |
+
# Change a bit here from the original code
|
| 34 |
+
# I use pyav instead of decord because it is much more safer
|
| 35 |
+
# The operations here are the same, we load video and return a list of PIL Image
|
| 36 |
+
# Load video frames
|
| 37 |
+
video_frames = read_video_pyav(path, num_frm=num_frames)
|
| 38 |
+
target_h, target_w = 224, 224
|
| 39 |
+
# If image shape is not as target, resize it
|
| 40 |
+
if video_frames.shape[-3] != target_h or video_frames.shape[-2] != target_w:
|
| 41 |
+
video_frames = torch.from_numpy(video_frames).permute(0, 3, 1, 2).float()
|
| 42 |
+
video_frames = torch.nn.functional.interpolate(video_frames, size=(target_h, target_w))
|
| 43 |
+
video_frames = video_frames.permute(0, 2, 3, 1).to(torch.uint8).numpy()
|
| 44 |
+
video_frames = [Image.fromarray(frame) for frame in video_frames]
|
| 45 |
+
if len(video_frames) > num_frames:
|
| 46 |
+
video_frames = video_frames[:num_frames]
|
| 47 |
+
return video_frames
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class MplugOwlProcessor(ProcessorMixin):
|
| 51 |
+
attributes = []
|
| 52 |
+
tokenizer_class = "MplugOwlTokenizer"
|
| 53 |
+
|
| 54 |
+
def __init__(self, image_processor=None, tokenizer=None, **kwargs):
|
| 55 |
+
super().__init__(**kwargs)
|
| 56 |
+
self.tokens_to_generate = 0
|
| 57 |
+
self.image_processor = image_processor
|
| 58 |
+
self.tokenizer = tokenizer
|
| 59 |
+
self.add_BOS = True
|
| 60 |
+
|
| 61 |
+
def __call__(self, text=None, images=None, videos=None, num_frames=4, return_tensors=None, **kwargs):
|
| 62 |
+
if text is None and images is None:
|
| 63 |
+
raise ValueError("You have to specify either text or images. Both cannot be none.")
|
| 64 |
+
|
| 65 |
+
if text is not None:
|
| 66 |
+
encoding = tokenize_prompts(
|
| 67 |
+
prompts=text,
|
| 68 |
+
tokens_to_generate=self.tokens_to_generate,
|
| 69 |
+
add_BOS=self.add_BOS,
|
| 70 |
+
tokenizer=self.tokenizer,
|
| 71 |
+
ignore_dist=True,
|
| 72 |
+
**kwargs,
|
| 73 |
+
)
|
| 74 |
+
# encoding = self.tokenizer(text, return_tensors=return_tensors, **kwargs)
|
| 75 |
+
|
| 76 |
+
if images is not None:
|
| 77 |
+
image_features = self.image_processor(images, return_tensors=return_tensors, **kwargs)
|
| 78 |
+
|
| 79 |
+
if videos is not None:
|
| 80 |
+
video_features = []
|
| 81 |
+
for video in videos:
|
| 82 |
+
video_frames = load_video(video, num_frames)
|
| 83 |
+
video_feature = self.image_processor(video_frames, return_tensors=return_tensors, **kwargs)["pixel_values"]
|
| 84 |
+
video_features.append(video_feature)
|
| 85 |
+
video_features = torch.stack(video_features, dim=0)
|
| 86 |
+
video_features = video_features.permute(0, 2, 1, 3, 4)
|
| 87 |
+
|
| 88 |
+
if text is not None and images is not None:
|
| 89 |
+
encoding["pixel_values"] = image_features.pixel_values
|
| 90 |
+
return encoding
|
| 91 |
+
if text is not None and videos is not None:
|
| 92 |
+
encoding["video_pixel_values"] = video_features
|
| 93 |
+
return encoding
|
| 94 |
+
elif text is not None:
|
| 95 |
+
return encoding
|
| 96 |
+
elif images is not None:
|
| 97 |
+
return BatchEncoding(data=dict(**image_features), tensor_type=return_tensors)
|
| 98 |
+
else:
|
| 99 |
+
return BatchEncoding(data=dict(video_pixel_values=video_pixel_values), tensor_type=return_tensors)
|
| 100 |
+
|
| 101 |
+
def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
|
| 102 |
+
"""
|
| 103 |
+
This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
|
| 104 |
+
refer to the docstring of this method for more information.
|
| 105 |
+
"""
|
| 106 |
+
return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
|
| 107 |
+
|
| 108 |
+
def decode(self, skip_special_tokens=True, *args, **kwargs):
|
| 109 |
+
"""
|
| 110 |
+
This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
|
| 111 |
+
the docstring of this method for more information.
|
| 112 |
+
"""
|
| 113 |
+
return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class MplugOwlImageProcessor(CLIPImageProcessor):
|
| 117 |
+
pass
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def detokenize_generations(tokens_gpu_tensor, lengths_gpu_tensor, return_segments, tokenizer):
|
| 121 |
+
"""Detokenize the generated tokens."""
|
| 122 |
+
|
| 123 |
+
prompts_plus_generations = []
|
| 124 |
+
if return_segments:
|
| 125 |
+
prompts_plus_generations_segments = []
|
| 126 |
+
|
| 127 |
+
tokens = tokens_gpu_tensor.cpu().numpy().tolist()
|
| 128 |
+
lengths = lengths_gpu_tensor.cpu().numpy().tolist()
|
| 129 |
+
for sequence_tokens, length in zip(tokens, lengths):
|
| 130 |
+
sequence_tokens = sequence_tokens[:length]
|
| 131 |
+
prompts_plus_generations.append(tokenizer.detokenize(sequence_tokens))
|
| 132 |
+
if return_segments:
|
| 133 |
+
from tokenizers.decoders import Metaspace
|
| 134 |
+
|
| 135 |
+
if hasattr(tokenizer, "tokenizer"):
|
| 136 |
+
if isinstance(tokenizer.tokenizer.decoder, Metaspace):
|
| 137 |
+
words = tokenizer.tokenizer.decode(sequence_tokens)
|
| 138 |
+
else:
|
| 139 |
+
words = []
|
| 140 |
+
for token in sequence_tokens:
|
| 141 |
+
word = tokenizer.tokenizer.decoder[token]
|
| 142 |
+
word = bytearray([tokenizer.tokenizer.byte_decoder[c] for c in word]).decode("utf-8", errors="replace")
|
| 143 |
+
words.append(word)
|
| 144 |
+
prompts_plus_generations_segments.append(words)
|
| 145 |
+
else:
|
| 146 |
+
words = tokenizer.detokenize(sequence_tokens)
|
| 147 |
+
# else:
|
| 148 |
+
# words = []
|
| 149 |
+
# for token in sequence_tokens:
|
| 150 |
+
# word = tokenizer.tokenizer.decoder[token]
|
| 151 |
+
# word = bytearray(
|
| 152 |
+
# [tokenizer.tokenizer.byte_decoder[c] for c in word]).decode(
|
| 153 |
+
# 'utf-8', errors='replace')
|
| 154 |
+
# words.append(word)
|
| 155 |
+
prompts_plus_generations_segments.append(words)
|
| 156 |
+
|
| 157 |
+
if return_segments:
|
| 158 |
+
return tokens, prompts_plus_generations, prompts_plus_generations_segments
|
| 159 |
+
|
| 160 |
+
return tokens, prompts_plus_generations
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def tokenize_prompts(prompts=None, tokens_to_generate=None, add_BOS=None, rank=0, tokenizer=None, ignore_dist=False, **kwargs):
|
| 164 |
+
"""Tokenize prompts and make them avaiable on all ranks."""
|
| 165 |
+
|
| 166 |
+
# On all ranks set to None so we can pass them to functions
|
| 167 |
+
prompts_tokens_cuda_long_tensor = None
|
| 168 |
+
prompts_length_cuda_long_tensor = None
|
| 169 |
+
|
| 170 |
+
# On the specified rank, build the above.
|
| 171 |
+
attention_mask = None
|
| 172 |
+
if ignore_dist or torch.distributed.get_rank() == rank:
|
| 173 |
+
assert prompts is not None
|
| 174 |
+
assert tokens_to_generate is not None
|
| 175 |
+
# Tensor of tokens padded and their unpadded length.
|
| 176 |
+
prompts_tokens_cuda_long_tensor, prompts_length_cuda_long_tensor, attention_mask = _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS, tokenizer, **kwargs)
|
| 177 |
+
# We need the sizes of these tensors for the boradcast
|
| 178 |
+
[
|
| 179 |
+
prompts_tokens_cuda_long_tensor.size(0), # Batch size
|
| 180 |
+
prompts_tokens_cuda_long_tensor.size(1),
|
| 181 |
+
] # Sequence lenght
|
| 182 |
+
|
| 183 |
+
return {
|
| 184 |
+
"input_ids": prompts_tokens_cuda_long_tensor,
|
| 185 |
+
"attention_mask": attention_mask,
|
| 186 |
+
# "prompt_length": prompts_length_cuda_long_tensor,
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS, tokenizer, **kwargs):
|
| 191 |
+
"""Given a set of prompts and number of tokens to generate:
|
| 192 |
+
- tokenize prompts
|
| 193 |
+
- set the sequence length to be the max of length of prompts
|
| 194 |
+
plus the number of tokens we would like to generate
|
| 195 |
+
- pad all the sequences to this length so we can convert them
|
| 196 |
+
into a 2D tensor.
|
| 197 |
+
"""
|
| 198 |
+
|
| 199 |
+
# Tokenize all the prompts.
|
| 200 |
+
# if add_BOS:
|
| 201 |
+
# prompts_tokens = [[tokenizer.bos] + tokenizer.tokenize(prompt)
|
| 202 |
+
# for prompt in prompts]
|
| 203 |
+
# else:
|
| 204 |
+
# prompts_tokens = [tokenizer.tokenize(prompt) for prompt in prompts]
|
| 205 |
+
|
| 206 |
+
prompts_tokens = [_tokenize_prompt(prompt, tokenizer, add_BOS, **kwargs) for prompt in prompts]
|
| 207 |
+
|
| 208 |
+
# Now we have a list of list of tokens which each list has a different
|
| 209 |
+
# size. We want to extend this list to:
|
| 210 |
+
# - incorporate the tokens that need to be generated
|
| 211 |
+
# - make all the sequences equal length.
|
| 212 |
+
# Get the prompts length.
|
| 213 |
+
prompts_length = [len(prompt_tokens) for prompt_tokens in prompts_tokens]
|
| 214 |
+
# Get the max prompts length.
|
| 215 |
+
max_prompt_len = max(prompts_length)
|
| 216 |
+
# Number of tokens in the each sample of the batch.
|
| 217 |
+
samples_length = max_prompt_len + tokens_to_generate
|
| 218 |
+
# Now update the list of list to be of the same size: samples_length.
|
| 219 |
+
for prompt_tokens, prompt_length in zip(prompts_tokens, prompts_length):
|
| 220 |
+
padding_size = samples_length - prompt_length
|
| 221 |
+
prompt_tokens.extend([tokenizer.eos_token_id] * padding_size)
|
| 222 |
+
|
| 223 |
+
# Now we are in a structured format, we can convert to tensors.
|
| 224 |
+
prompts_tokens_tensor = torch.LongTensor(prompts_tokens)
|
| 225 |
+
prompts_length_tensor = torch.LongTensor(prompts_length)
|
| 226 |
+
attention_mask = torch.zeros(prompts_tokens_tensor.shape[:2])
|
| 227 |
+
for i, l in enumerate(prompts_length_tensor):
|
| 228 |
+
attention_mask[i, :l] = 1
|
| 229 |
+
return prompts_tokens_tensor, prompts_length_tensor, attention_mask
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _tokenize_prompt(prompt, tokenizer, add_BOS=False, media_info={"<image>": 65, "<|video|>": 65}, **kwargs):
|
| 233 |
+
media_tokens = {k: -int(i + 1) for i, k in enumerate(media_info.keys())}
|
| 234 |
+
media_lengths = media_info.copy()
|
| 235 |
+
|
| 236 |
+
if add_BOS:
|
| 237 |
+
prompt_chunk = [tokenizer.bos_token_id]
|
| 238 |
+
else:
|
| 239 |
+
prompt_chunk = []
|
| 240 |
+
|
| 241 |
+
# Pure Text
|
| 242 |
+
if all([media_token not in prompt for media_token in media_tokens.keys()]):
|
| 243 |
+
enc_chunk = prompt_chunk + tokenizer(prompt, add_special_tokens=False, **kwargs)["input_ids"]
|
| 244 |
+
|
| 245 |
+
# Multi-Modal Text
|
| 246 |
+
else:
|
| 247 |
+
enc_chunk = prompt_chunk
|
| 248 |
+
pattern = "|".join(map(re.escape, list(media_tokens.keys())))
|
| 249 |
+
chunk_strs = re.split(f"({pattern})", prompt)
|
| 250 |
+
chunk_strs = [x for x in chunk_strs if len(x) > 0]
|
| 251 |
+
for idx, chunk_str in enumerate(chunk_strs):
|
| 252 |
+
if chunk_str in media_tokens:
|
| 253 |
+
enc_chunk += [media_tokens[chunk_str]] * media_lengths[chunk_str]
|
| 254 |
+
else:
|
| 255 |
+
tmp_chunk = tokenizer(chunk_str, add_special_tokens=False)["input_ids"]
|
| 256 |
+
# if idx < len(chunk_strs) - 1: # Last chunk should not have eos
|
| 257 |
+
# tmp_chunk += [tokenizer.eod_id]
|
| 258 |
+
enc_chunk += tmp_chunk
|
| 259 |
+
return enc_chunk
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
if __name__ == "__main__":
|
| 263 |
+
pass
|
LaViDa/eval/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2022 x-plug and The HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
"""Tokenization classes for MplugOwl."""
|
| 16 |
+
|
| 17 |
+
from loguru import logger
|
| 18 |
+
from transformers.models.llama.tokenization_llama import LlamaTokenizer
|
| 19 |
+
|
| 20 |
+
VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
|
| 21 |
+
|
| 22 |
+
PRETRAINED_VOCAB_FILES_MAP = {
|
| 23 |
+
"vocab_file": {
|
| 24 |
+
"MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/vocab.txt",
|
| 25 |
+
},
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
| 29 |
+
"MAGAer13/mplug-owl-llama-7b": 2048,
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class MplugOwlTokenizer(LlamaTokenizer):
|
| 34 |
+
def __init__(
|
| 35 |
+
self,
|
| 36 |
+
vocab_file,
|
| 37 |
+
unk_token="<unk>",
|
| 38 |
+
bos_token="<s>",
|
| 39 |
+
eos_token="</s>",
|
| 40 |
+
pad_token="<unk>",
|
| 41 |
+
sp_model_kwargs=None,
|
| 42 |
+
add_bos_token=False,
|
| 43 |
+
add_eos_token=False,
|
| 44 |
+
clean_up_tokenization_spaces=False,
|
| 45 |
+
**kwargs,
|
| 46 |
+
):
|
| 47 |
+
super().__init__(
|
| 48 |
+
vocab_file,
|
| 49 |
+
unk_token,
|
| 50 |
+
bos_token,
|
| 51 |
+
eos_token,
|
| 52 |
+
pad_token,
|
| 53 |
+
sp_model_kwargs,
|
| 54 |
+
add_bos_token,
|
| 55 |
+
add_eos_token,
|
| 56 |
+
clean_up_tokenization_spaces,
|
| 57 |
+
**kwargs,
|
| 58 |
+
)
|
| 59 |
+
self.eod_id = self.eos_token_id
|
LaViDa/eval/lmms_eval/models/ola.py
ADDED
|
@@ -0,0 +1,709 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
os.environ["LOWRES_RESIZE"] = "384x32"
|
| 4 |
+
os.environ["HIGHRES_BASE"] = "0x32"
|
| 5 |
+
os.environ["VIDEO_RESIZE"] = "0x64"
|
| 6 |
+
os.environ["VIDEO_MAXRES"] = "480"
|
| 7 |
+
os.environ["VIDEO_MINRES"] = "288"
|
| 8 |
+
os.environ["MAXRES"] = "1536"
|
| 9 |
+
os.environ["MINRES"] = "0"
|
| 10 |
+
os.environ["FORCE_NO_DOWNSAMPLE"] = "1"
|
| 11 |
+
os.environ["LOAD_VISION_EARLY"] = "1"
|
| 12 |
+
os.environ["PAD2STRIDE"] = "1"
|
| 13 |
+
os.environ["USE_SPEECH"] = "1"
|
| 14 |
+
import copy
|
| 15 |
+
import logging
|
| 16 |
+
from datetime import timedelta
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import List, Optional, Tuple, Union
|
| 19 |
+
|
| 20 |
+
import librosa
|
| 21 |
+
import numpy as np
|
| 22 |
+
import PIL
|
| 23 |
+
import soundfile as sf
|
| 24 |
+
import torch
|
| 25 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 26 |
+
from accelerate.state import AcceleratorState
|
| 27 |
+
from decord import VideoReader, cpu
|
| 28 |
+
from tqdm import tqdm
|
| 29 |
+
from transformers import AutoConfig
|
| 30 |
+
|
| 31 |
+
from lmms_eval import utils
|
| 32 |
+
from lmms_eval.api.instance import Instance
|
| 33 |
+
from lmms_eval.api.model import lmms
|
| 34 |
+
from lmms_eval.api.registry import register_model
|
| 35 |
+
from lmms_eval.models.model_utils.audio_processing import downsample_audio
|
| 36 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 37 |
+
|
| 38 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 39 |
+
|
| 40 |
+
import sys
|
| 41 |
+
|
| 42 |
+
wd = Path(__file__).parent.parent.parent.parent.resolve()
|
| 43 |
+
sys.path.append(os.path.join(str(wd), "Ola"))
|
| 44 |
+
|
| 45 |
+
import whisper
|
| 46 |
+
from ola.constants import (
|
| 47 |
+
DEFAULT_IM_END_TOKEN,
|
| 48 |
+
DEFAULT_IM_START_TOKEN,
|
| 49 |
+
DEFAULT_IMAGE_TOKEN,
|
| 50 |
+
DEFAULT_SPEECH_TOKEN,
|
| 51 |
+
IMAGE_TOKEN_INDEX,
|
| 52 |
+
SPEECH_TOKEN_INDEX,
|
| 53 |
+
)
|
| 54 |
+
from ola.conversation import SeparatorStyle, conv_templates
|
| 55 |
+
from ola.datasets.preprocess import (
|
| 56 |
+
tokenizer_image_token,
|
| 57 |
+
tokenizer_speech_image_token,
|
| 58 |
+
tokenizer_speech_token,
|
| 59 |
+
)
|
| 60 |
+
from ola.mm_utils import (
|
| 61 |
+
KeywordsStoppingCriteria,
|
| 62 |
+
get_model_name_from_path,
|
| 63 |
+
process_anyres_highres_image,
|
| 64 |
+
process_anyres_video,
|
| 65 |
+
)
|
| 66 |
+
from ola.model.builder import load_pretrained_model
|
| 67 |
+
|
| 68 |
+
try:
|
| 69 |
+
from ola.model.language_model.ola_qwen import OlaConfigQwen
|
| 70 |
+
|
| 71 |
+
AutoConfig.register("ola_qwen", OlaConfigQwen)
|
| 72 |
+
except:
|
| 73 |
+
eval_logger.debug("")
|
| 74 |
+
from moviepy.video.io.VideoFileClip import VideoFileClip
|
| 75 |
+
|
| 76 |
+
if "USE_SPEECH" in os.environ:
|
| 77 |
+
USE_SPEECH = os.environ["USE_SPEECH"]
|
| 78 |
+
print("USE_SPEECH is set")
|
| 79 |
+
else:
|
| 80 |
+
USE_SPEECH = None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
@register_model("ola")
|
| 84 |
+
class Ola(lmms):
|
| 85 |
+
"""
|
| 86 |
+
How to run lmms-eval with Ola model:
|
| 87 |
+
|
| 88 |
+
1. Install Ola:
|
| 89 |
+
https://github.com/Ola-Omni/Ola?tab=readme-ov-file#installation
|
| 90 |
+
|
| 91 |
+
2. Download the pretrained weight from https://huggingface.co/THUdyh/Ola-7b
|
| 92 |
+
or skip this step to use the online weights directly
|
| 93 |
+
|
| 94 |
+
3.Download audio encoder from https://huggingface.co/THUdyh/Ola_speech_encoders/tree/main
|
| 95 |
+
and put the weights large-v3.pt and BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt
|
| 96 |
+
under llms-eval repository (ensure your current directory can see large-v3.pt and BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt)
|
| 97 |
+
|
| 98 |
+
The path for the project should be like this:
|
| 99 |
+
Folder/contains/lmms-eval/and/Ola
|
| 100 |
+
├── lmms-eval (current directory)
|
| 101 |
+
│ ├── lmms_eval/
|
| 102 |
+
│ ├── ...
|
| 103 |
+
│ ├── large-v3.pt
|
| 104 |
+
│ ├── BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt
|
| 105 |
+
├── Ola
|
| 106 |
+
│ ├── ...
|
| 107 |
+
|
| 108 |
+
4. Run the the command to start evaluate the modeL. For example:
|
| 109 |
+
```bash
|
| 110 |
+
python3 -m accelerate.commands.launch \
|
| 111 |
+
--num_processes=8 \
|
| 112 |
+
-m lmms_eval \
|
| 113 |
+
--model ola\
|
| 114 |
+
--tasks mme \
|
| 115 |
+
--batch_size 1 \
|
| 116 |
+
--log_samples \
|
| 117 |
+
--log_samples_suffix mme_ola \
|
| 118 |
+
--output_path ./logs/
|
| 119 |
+
```
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
def __init__(
|
| 123 |
+
self,
|
| 124 |
+
pretrained: str = "THUdyh/Ola-7b",
|
| 125 |
+
truncation: Optional[bool] = True,
|
| 126 |
+
device: Optional[str] = "cuda:0",
|
| 127 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 128 |
+
attn_implementation=(
|
| 129 |
+
"sdpa" if torch.__version__ >= "2.1.2" else "eager"
|
| 130 |
+
), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 131 |
+
device_map="",
|
| 132 |
+
conv_template="qwen_1_5",
|
| 133 |
+
use_cache=True,
|
| 134 |
+
truncate_context=False,
|
| 135 |
+
max_frames_num: int = 64,
|
| 136 |
+
mm_resampler_type: str = "spatial_pool",
|
| 137 |
+
overwrite: bool = True,
|
| 138 |
+
video_decode_backend: str = "decord",
|
| 139 |
+
**kwargs,
|
| 140 |
+
) -> None:
|
| 141 |
+
super().__init__()
|
| 142 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 143 |
+
|
| 144 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 145 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 146 |
+
if accelerator.num_processes > 1:
|
| 147 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 148 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 149 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 150 |
+
self._device = torch.device(device)
|
| 151 |
+
self.device_map = device_map
|
| 152 |
+
else:
|
| 153 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 154 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 155 |
+
|
| 156 |
+
self.pretrained = pretrained
|
| 157 |
+
self.model_name = get_model_name_from_path(pretrained)
|
| 158 |
+
self.video_decode_backend = video_decode_backend
|
| 159 |
+
# self._config = AutoConfig.from_pretrained(self.pretrained)
|
| 160 |
+
self.overwrite = overwrite
|
| 161 |
+
self.mm_resampler_type = mm_resampler_type
|
| 162 |
+
self.max_frames_num = int(max_frames_num)
|
| 163 |
+
if self.overwrite == True:
|
| 164 |
+
overwrite_config = {}
|
| 165 |
+
overwrite_config["patchify_video_feature"] = False
|
| 166 |
+
overwrite_config["attn_implementation"] = attn_implementation
|
| 167 |
+
|
| 168 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 169 |
+
|
| 170 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, device=self.device_map)
|
| 171 |
+
else:
|
| 172 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(
|
| 173 |
+
pretrained,
|
| 174 |
+
None,
|
| 175 |
+
device_map=self.device_map,
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
self._config = self._model.config
|
| 179 |
+
self.model.to(self.device).eval().bfloat16()
|
| 180 |
+
self.model.tie_weights()
|
| 181 |
+
self.truncation = truncation
|
| 182 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 183 |
+
self.conv_template = conv_template
|
| 184 |
+
self.use_cache = use_cache
|
| 185 |
+
self.truncate_context = truncate_context
|
| 186 |
+
if accelerator.num_processes > 1:
|
| 187 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 188 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 189 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 190 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 191 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 192 |
+
kwargs = {
|
| 193 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 194 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 195 |
+
}
|
| 196 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 197 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 198 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 199 |
+
self._model = accelerator.prepare(self.model)
|
| 200 |
+
else:
|
| 201 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 202 |
+
self.accelerator = accelerator
|
| 203 |
+
if self.accelerator.is_local_main_process:
|
| 204 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 205 |
+
self._rank = self.accelerator.local_process_index
|
| 206 |
+
self._world_size = self.accelerator.num_processes
|
| 207 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 208 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 209 |
+
self._rank = 0
|
| 210 |
+
self._world_size = 1
|
| 211 |
+
else:
|
| 212 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 213 |
+
self._rank = 0
|
| 214 |
+
self._world_size = 1
|
| 215 |
+
self.accelerator = accelerator
|
| 216 |
+
|
| 217 |
+
@property
|
| 218 |
+
def config(self):
|
| 219 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 220 |
+
return self._config
|
| 221 |
+
|
| 222 |
+
@property
|
| 223 |
+
def tokenizer(self):
|
| 224 |
+
return self._tokenizer
|
| 225 |
+
|
| 226 |
+
@property
|
| 227 |
+
def model(self):
|
| 228 |
+
# returns the model, unwrapping it if using Accelerate
|
| 229 |
+
if hasattr(self, "accelerator"):
|
| 230 |
+
return self.accelerator.unwrap_model(self._model)
|
| 231 |
+
else:
|
| 232 |
+
return self._model
|
| 233 |
+
|
| 234 |
+
@property
|
| 235 |
+
def eot_token_id(self):
|
| 236 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 237 |
+
return self.tokenizer.eos_token_id
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def max_length(self):
|
| 241 |
+
return self._max_length
|
| 242 |
+
|
| 243 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 244 |
+
if self.tokenizer.padding_side == "left":
|
| 245 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 246 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 247 |
+
if self.tokenizer.padding_side == "left":
|
| 248 |
+
input_ids = torch.flip(input_ids, [1])
|
| 249 |
+
return input_ids
|
| 250 |
+
|
| 251 |
+
@property
|
| 252 |
+
def batch_size(self):
|
| 253 |
+
return self.batch_size_per_gpu
|
| 254 |
+
|
| 255 |
+
@property
|
| 256 |
+
def device(self):
|
| 257 |
+
return self._device
|
| 258 |
+
|
| 259 |
+
@property
|
| 260 |
+
def rank(self):
|
| 261 |
+
return self._rank
|
| 262 |
+
|
| 263 |
+
@property
|
| 264 |
+
def world_size(self):
|
| 265 |
+
return self._world_size
|
| 266 |
+
|
| 267 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 268 |
+
""" """
|
| 269 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 270 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 271 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 272 |
+
if left_truncate_len:
|
| 273 |
+
encoding = encoding[-left_truncate_len:]
|
| 274 |
+
return encoding
|
| 275 |
+
|
| 276 |
+
def load_video(self, video_path, max_frames_num):
|
| 277 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 278 |
+
total_frame_num = len(vr)
|
| 279 |
+
fps = round(vr.get_avg_fps())
|
| 280 |
+
|
| 281 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 282 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 283 |
+
|
| 284 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 285 |
+
video = [PIL.Image.fromarray(frame) for frame in spare_frames]
|
| 286 |
+
return video, frame_idx
|
| 287 |
+
|
| 288 |
+
def tok_decode(self, tokens):
|
| 289 |
+
return self.tokenizer.decode(tokens)
|
| 290 |
+
|
| 291 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 292 |
+
res = []
|
| 293 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 294 |
+
|
| 295 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 296 |
+
# encode, pad, and truncate contexts for this batch
|
| 297 |
+
if type(doc_to_target) == str:
|
| 298 |
+
continuation = doc_to_target
|
| 299 |
+
else:
|
| 300 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 301 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 302 |
+
visuals = self.flatten(visuals)
|
| 303 |
+
videos = []
|
| 304 |
+
for visual in visuals:
|
| 305 |
+
video = self.load_video(visual, self.max_frames_num)
|
| 306 |
+
video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].bfloat16().to(self.device)
|
| 307 |
+
videos.append(video)
|
| 308 |
+
|
| 309 |
+
qs = contexts
|
| 310 |
+
if self.model.config.mm_use_im_start_end:
|
| 311 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 312 |
+
else:
|
| 313 |
+
qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 314 |
+
|
| 315 |
+
conv = conv_templates[self.conv_template].copy()
|
| 316 |
+
conv.append_message(conv.roles[0], qs)
|
| 317 |
+
conv.append_message(conv.roles[1], None)
|
| 318 |
+
prompt = conv.get_prompt()
|
| 319 |
+
|
| 320 |
+
contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 321 |
+
|
| 322 |
+
conv = conv_templates[self.conv_template].copy()
|
| 323 |
+
conv.append_message(conv.roles[0], qs)
|
| 324 |
+
conv.append_message(conv.roles[1], continuation)
|
| 325 |
+
prompt = conv.get_prompt()
|
| 326 |
+
|
| 327 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 328 |
+
|
| 329 |
+
labels = input_ids.clone()
|
| 330 |
+
# Context part no need to calculate for loss
|
| 331 |
+
labels[0, : contxt_id.shape[1]] = -100
|
| 332 |
+
|
| 333 |
+
with torch.inference_mode():
|
| 334 |
+
outputs = self.model(input_ids=input_ids, labels=labels, images=videos, modalities="video")
|
| 335 |
+
|
| 336 |
+
loss = outputs["loss"]
|
| 337 |
+
# loss = torch.exp(loss)
|
| 338 |
+
logits = outputs["logits"]
|
| 339 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 340 |
+
cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
|
| 341 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
|
| 342 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 343 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 344 |
+
pbar.update(1)
|
| 345 |
+
pbar.close()
|
| 346 |
+
return res
|
| 347 |
+
|
| 348 |
+
def flatten(self, input):
|
| 349 |
+
new_list = []
|
| 350 |
+
for i in input:
|
| 351 |
+
for j in i:
|
| 352 |
+
new_list.append(j)
|
| 353 |
+
return new_list
|
| 354 |
+
|
| 355 |
+
def extract_audio(self, videos_file_path):
|
| 356 |
+
my_clip = VideoFileClip(videos_file_path)
|
| 357 |
+
return my_clip.audio
|
| 358 |
+
|
| 359 |
+
def load_audio(self, audio_file_name):
|
| 360 |
+
CHUNK_LIM = 480000
|
| 361 |
+
import librosa
|
| 362 |
+
|
| 363 |
+
audio, samplerate = librosa.load(audio_file_name, sr=16000)
|
| 364 |
+
audio = audio.astype(np.float32)
|
| 365 |
+
|
| 366 |
+
if len(audio.shape) > 1:
|
| 367 |
+
audio = audio[:, 0]
|
| 368 |
+
|
| 369 |
+
speechs = []
|
| 370 |
+
speech_wavs = []
|
| 371 |
+
if len(audio) <= CHUNK_LIM:
|
| 372 |
+
audio = whisper.pad_or_trim(audio)
|
| 373 |
+
speechs.append(audio)
|
| 374 |
+
speech_wavs.append(torch.from_numpy(audio).unsqueeze(0))
|
| 375 |
+
else:
|
| 376 |
+
for i in range(0, len(audio), CHUNK_LIM):
|
| 377 |
+
chunk = audio[i : i + CHUNK_LIM]
|
| 378 |
+
if len(chunk) < CHUNK_LIM:
|
| 379 |
+
chunk = whisper.pad_or_trim(chunk)
|
| 380 |
+
speechs.append(chunk)
|
| 381 |
+
speech_wavs.append(torch.from_numpy(chunk).unsqueeze(0))
|
| 382 |
+
mels = []
|
| 383 |
+
for chunk in speechs:
|
| 384 |
+
chunk = whisper.log_mel_spectrogram(chunk, n_mels=128).permute(1, 0).unsqueeze(0)
|
| 385 |
+
mels.append(chunk)
|
| 386 |
+
|
| 387 |
+
mels = torch.cat(mels, dim=0)
|
| 388 |
+
speech_wavs = torch.cat(speech_wavs, dim=0)
|
| 389 |
+
if mels.shape[0] > 20:
|
| 390 |
+
mels = mels[:20]
|
| 391 |
+
speech_wavs = speech_wavs[:20]
|
| 392 |
+
|
| 393 |
+
speech_length = torch.LongTensor([mels.shape[1]] * mels.shape[0])
|
| 394 |
+
speech_chunks = torch.LongTensor([mels.shape[0]])
|
| 395 |
+
|
| 396 |
+
return mels, speech_length, speech_chunks, speech_wavs
|
| 397 |
+
|
| 398 |
+
def process_audio(self, audio_array, sampling_rate):
|
| 399 |
+
"""
|
| 400 |
+
Process audio array to format of Ola model
|
| 401 |
+
"""
|
| 402 |
+
audio = audio_array.astype(np.float32)
|
| 403 |
+
if len(audio.shape) > 1:
|
| 404 |
+
audio = audio[:, 0]
|
| 405 |
+
target_sr = 16000
|
| 406 |
+
CHUNK_LIM = 480000
|
| 407 |
+
|
| 408 |
+
if sampling_rate != target_sr:
|
| 409 |
+
speech_wav = librosa.resample(audio_array, orig_sr=sampling_rate, target_sr=target_sr).astype(np.float32)
|
| 410 |
+
else:
|
| 411 |
+
speech_wav = audio_array.astype(np.float32)
|
| 412 |
+
speechs = []
|
| 413 |
+
speech_wavs = []
|
| 414 |
+
|
| 415 |
+
if len(speech_wav) <= CHUNK_LIM:
|
| 416 |
+
speech = whisper.pad_or_trim(speech_wav)
|
| 417 |
+
speech_wav = whisper.pad_or_trim(speech_wav)
|
| 418 |
+
speechs.append(speech)
|
| 419 |
+
speech_wavs.append(torch.from_numpy(speech_wav).unsqueeze(0))
|
| 420 |
+
else:
|
| 421 |
+
for i in range(0, len(speech_wav), CHUNK_LIM):
|
| 422 |
+
chunk = speech_wav[i : i + CHUNK_LIM]
|
| 423 |
+
if len(chunk) < CHUNK_LIM:
|
| 424 |
+
chunk = whisper.pad_or_trim(chunk)
|
| 425 |
+
speechs.append(chunk)
|
| 426 |
+
speech_wavs.append(torch.from_numpy(chunk).unsqueeze(0))
|
| 427 |
+
mels = []
|
| 428 |
+
for chunk in speechs:
|
| 429 |
+
chunk = whisper.log_mel_spectrogram(chunk, n_mels=128).permute(1, 0).unsqueeze(0)
|
| 430 |
+
mels.append(chunk)
|
| 431 |
+
|
| 432 |
+
mels = torch.cat(mels, dim=0)
|
| 433 |
+
speech_wavs = torch.cat(speech_wavs, dim=0)
|
| 434 |
+
if mels.shape[0] > 25:
|
| 435 |
+
mels = mels[:25]
|
| 436 |
+
speech_wavs = speech_wavs[:25]
|
| 437 |
+
|
| 438 |
+
speech_length = torch.LongTensor([mels.shape[1]] * mels.shape[0])
|
| 439 |
+
speech_chunks = torch.LongTensor([mels.shape[0]])
|
| 440 |
+
return mels, speech_length, speech_chunks, speech_wavs
|
| 441 |
+
|
| 442 |
+
def generate_until(self, requests) -> List[str]:
|
| 443 |
+
MODALITY = None
|
| 444 |
+
res = []
|
| 445 |
+
|
| 446 |
+
def _collate(x):
|
| 447 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 448 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 449 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 450 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 451 |
+
# automatic adaptive batches much much easier to implement
|
| 452 |
+
# - any OOMs will happen right away rather than near the end
|
| 453 |
+
toks = self.tok_encode(x[0])
|
| 454 |
+
return -len(toks), x[0]
|
| 455 |
+
|
| 456 |
+
# we group requests by their generation_kwargs,
|
| 457 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 458 |
+
# in the same batch.
|
| 459 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 460 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 461 |
+
num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
|
| 462 |
+
pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
|
| 463 |
+
for chunk in chunks:
|
| 464 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 465 |
+
task = task[0]
|
| 466 |
+
split = split[0]
|
| 467 |
+
context = contexts[0]
|
| 468 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 469 |
+
visuals = self.flatten(visuals) # Len = 1. just an audio tho
|
| 470 |
+
|
| 471 |
+
speechs, speech_lengths, speech_wavs, speech_chunks = [], [], [], []
|
| 472 |
+
images, images_highres = [], [] # For dummy image passed in audio modality
|
| 473 |
+
image_sizes = []
|
| 474 |
+
image_tensor, image_highres_tensor = [], [] # For image
|
| 475 |
+
video_processed = [] # For video only
|
| 476 |
+
for visual in visuals:
|
| 477 |
+
if isinstance(visual, str): # For Video
|
| 478 |
+
if MODALITY is None:
|
| 479 |
+
MODALITY = "VIDEO"
|
| 480 |
+
# Process audio of video
|
| 481 |
+
try:
|
| 482 |
+
video, frame_idx = self.load_video(visual, self.max_frames_num)
|
| 483 |
+
except Exception as e:
|
| 484 |
+
eval_logger.info(f"{e}")
|
| 485 |
+
eval_logger.info(f"Video {visuals} can not load, check the source")
|
| 486 |
+
continue
|
| 487 |
+
audio = self.extract_audio(visual)
|
| 488 |
+
audio.write_audiofile(f"./video_audio_{self.rank}.wav")
|
| 489 |
+
video_audio_path = f"./video_audio_{self.rank}.wav"
|
| 490 |
+
speech, speech_length, speech_chunk, speech_wav = self.load_audio(video_audio_path)
|
| 491 |
+
speechs.append(speech.bfloat16().to(self.device))
|
| 492 |
+
speech_lengths.append(speech_length.to(self.device))
|
| 493 |
+
speech_chunks.append(speech_chunk.to(self.device))
|
| 494 |
+
speech_wavs.append(speech_wav.to(self.device))
|
| 495 |
+
os.remove(video_audio_path)
|
| 496 |
+
|
| 497 |
+
# Process images of video
|
| 498 |
+
for idx, frame in enumerate(video):
|
| 499 |
+
self._image_processor.do_resize = False
|
| 500 |
+
self._image_processor.do_center_crop = False
|
| 501 |
+
frame = process_anyres_video(frame, self._image_processor)
|
| 502 |
+
|
| 503 |
+
if frame_idx is not None and idx in frame_idx:
|
| 504 |
+
video_processed.append(frame.unsqueeze(0))
|
| 505 |
+
elif frame_idx is None:
|
| 506 |
+
video_processed.append(frame.unsqueeze(0))
|
| 507 |
+
|
| 508 |
+
if frame_idx is None:
|
| 509 |
+
frame_idx = np.arange(0, len(video_processed), dtype=int).tolist()
|
| 510 |
+
|
| 511 |
+
video_processed = torch.cat(video_processed, dim=0).bfloat16().to(self.device)
|
| 512 |
+
video_processed = (video_processed, video_processed)
|
| 513 |
+
|
| 514 |
+
video_data = (video_processed, (384, 384), "video")
|
| 515 |
+
|
| 516 |
+
elif isinstance(visual, PIL.Image.Image): # For Image
|
| 517 |
+
if MODALITY is None:
|
| 518 |
+
MODALITY = "IMAGE"
|
| 519 |
+
self._image_processor.do_resize = False
|
| 520 |
+
self._image_processor.do_center_crop = False
|
| 521 |
+
image_sizes.append(visual.size)
|
| 522 |
+
image_tensor_, image_highres_tensor_ = process_anyres_highres_image(visual, self._image_processor)
|
| 523 |
+
image_tensor.append(image_tensor_)
|
| 524 |
+
image_highres_tensor.append(image_highres_tensor_)
|
| 525 |
+
|
| 526 |
+
elif isinstance(visual, dict) and "array" in visual: # For Audio
|
| 527 |
+
if MODALITY is None:
|
| 528 |
+
MODALITY = "AUDIO"
|
| 529 |
+
mels, speech_length, speech_chunk, speech_wav = self.process_audio(visual["array"], visual["sampling_rate"])
|
| 530 |
+
speechs.append(mels.bfloat16().to(self.device))
|
| 531 |
+
speech_lengths.append(speech_length.to(self.device))
|
| 532 |
+
speech_chunks.append(speech_chunk.to(self.device))
|
| 533 |
+
speech_wavs.append(speech_wav.to(self.device))
|
| 534 |
+
|
| 535 |
+
# Processing dummy images, as required by model
|
| 536 |
+
images.append(torch.zeros(1, 3, 224, 224).to(dtype=torch.bfloat16, device=self.device, non_blocking=True))
|
| 537 |
+
images_highres.append(torch.zeros(1, 3, 224, 224).to(dtype=torch.bfloat16, device=self.device, non_blocking=True))
|
| 538 |
+
image_sizes.append((224, 224))
|
| 539 |
+
|
| 540 |
+
if not video_processed and MODALITY == "VIDEO":
|
| 541 |
+
# If video is not processed, skip the iteration
|
| 542 |
+
pbar.update(1)
|
| 543 |
+
continue
|
| 544 |
+
|
| 545 |
+
if MODALITY == "IMAGE":
|
| 546 |
+
if all(x.shape == image_tensor[0].shape for x in image_tensor):
|
| 547 |
+
image_tensor = torch.stack(image_tensor, dim=0)
|
| 548 |
+
if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor):
|
| 549 |
+
image_highres_tensor = torch.stack(image_highres_tensor, dim=0)
|
| 550 |
+
if type(image_tensor) is list:
|
| 551 |
+
image_tensor = [_image.bfloat16().to("cuda") for _image in image_tensor]
|
| 552 |
+
else:
|
| 553 |
+
image_tensor = image_tensor.bfloat16().to("cuda")
|
| 554 |
+
if type(image_highres_tensor) is list:
|
| 555 |
+
image_highres_tensor = [_image.bfloat16().to("cuda") for _image in image_highres_tensor]
|
| 556 |
+
else:
|
| 557 |
+
image_highres_tensor = image_highres_tensor.bfloat16().to("cuda")
|
| 558 |
+
|
| 559 |
+
# Processing dummy audio, as required by model
|
| 560 |
+
speechs.append(torch.zeros(1, 3000, 128).bfloat16().to("cuda"))
|
| 561 |
+
speech_lengths.append(torch.LongTensor([3000]).to("cuda"))
|
| 562 |
+
speech_wavs.append(torch.zeros([1, 480000]).to("cuda"))
|
| 563 |
+
speech_chunks.append(torch.LongTensor([1]).to("cuda"))
|
| 564 |
+
|
| 565 |
+
# we assume all gen kwargs in the batch are the same
|
| 566 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 567 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 568 |
+
|
| 569 |
+
# Set default values for until and max_new_tokens
|
| 570 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 571 |
+
|
| 572 |
+
# Update values from gen_kwargs if present
|
| 573 |
+
if "until" in gen_kwargs:
|
| 574 |
+
until = gen_kwargs.pop("until")
|
| 575 |
+
if isinstance(until, str):
|
| 576 |
+
until = [until]
|
| 577 |
+
elif not isinstance(until, list):
|
| 578 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 579 |
+
assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
|
| 580 |
+
# Okay be I am assuming bs always == 1
|
| 581 |
+
qs = list(contexts)[0]
|
| 582 |
+
if self.model.config.mm_use_im_start_end:
|
| 583 |
+
if MODALITY == "AUDIO":
|
| 584 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_SPEECH_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 585 |
+
elif MODALITY == "IMAGE":
|
| 586 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 587 |
+
elif MODALITY == "VIDEO":
|
| 588 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_SPEECH_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 589 |
+
else:
|
| 590 |
+
if MODALITY == "AUDIO":
|
| 591 |
+
qs = DEFAULT_SPEECH_TOKEN + "\n" + qs
|
| 592 |
+
elif MODALITY == "IMAGE":
|
| 593 |
+
qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 594 |
+
elif MODALITY == "VIDEO":
|
| 595 |
+
qs = DEFAULT_SPEECH_TOKEN + DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 596 |
+
|
| 597 |
+
conv = conv_templates[self.conv_template].copy()
|
| 598 |
+
conv.append_message(conv.roles[0], qs)
|
| 599 |
+
conv.append_message(conv.roles[1], None)
|
| 600 |
+
prompt = conv.get_prompt()
|
| 601 |
+
|
| 602 |
+
if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
|
| 603 |
+
eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{prompt}\n")
|
| 604 |
+
|
| 605 |
+
if MODALITY == "AUDIO":
|
| 606 |
+
input_ids = tokenizer_speech_token(prompt, self.tokenizer, SPEECH_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 607 |
+
elif MODALITY == "IMAGE":
|
| 608 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 609 |
+
elif MODALITY == "VIDEO":
|
| 610 |
+
input_ids = tokenizer_speech_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 611 |
+
pad_token_ids = 151643
|
| 612 |
+
attention_masks = input_ids.ne(pad_token_ids).long().to(self.device)
|
| 613 |
+
|
| 614 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 615 |
+
keywords = [stop_str]
|
| 616 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 617 |
+
|
| 618 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 619 |
+
gen_kwargs["max_new_tokens"] = 256
|
| 620 |
+
if "temperature" not in gen_kwargs:
|
| 621 |
+
gen_kwargs["temperature"] = 0
|
| 622 |
+
if "top_p" not in gen_kwargs:
|
| 623 |
+
gen_kwargs["top_p"] = None
|
| 624 |
+
if "num_beams" not in gen_kwargs:
|
| 625 |
+
gen_kwargs["num_beams"] = 1
|
| 626 |
+
|
| 627 |
+
try:
|
| 628 |
+
with torch.inference_mode():
|
| 629 |
+
if MODALITY == "AUDIO":
|
| 630 |
+
output_ids = self.model.generate(
|
| 631 |
+
input_ids,
|
| 632 |
+
images=images,
|
| 633 |
+
images_highres=images_highres,
|
| 634 |
+
image_sizes=image_sizes,
|
| 635 |
+
modalities=["text"],
|
| 636 |
+
speech=speechs,
|
| 637 |
+
speech_lengths=speech_lengths,
|
| 638 |
+
speech_chunks=speech_chunks,
|
| 639 |
+
speech_wav=speech_wavs,
|
| 640 |
+
attention_mask=attention_masks,
|
| 641 |
+
use_cache=True,
|
| 642 |
+
stopping_criteria=[stopping_criteria],
|
| 643 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 644 |
+
temperature=gen_kwargs["temperature"],
|
| 645 |
+
top_p=gen_kwargs["top_p"],
|
| 646 |
+
num_beams=gen_kwargs["num_beams"],
|
| 647 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 648 |
+
)
|
| 649 |
+
elif MODALITY == "IMAGE":
|
| 650 |
+
output_ids = self.model.generate(
|
| 651 |
+
inputs=input_ids,
|
| 652 |
+
images=image_tensor,
|
| 653 |
+
images_highres=image_highres_tensor,
|
| 654 |
+
image_sizes=image_sizes,
|
| 655 |
+
modalities=["image"],
|
| 656 |
+
speech=speechs,
|
| 657 |
+
speech_lengths=speech_lengths,
|
| 658 |
+
speech_chunks=speech_chunks,
|
| 659 |
+
speech_wav=speech_wavs,
|
| 660 |
+
attention_mask=attention_masks,
|
| 661 |
+
use_cache=True,
|
| 662 |
+
stopping_criteria=[stopping_criteria],
|
| 663 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 664 |
+
temperature=gen_kwargs["temperature"],
|
| 665 |
+
top_p=gen_kwargs["top_p"],
|
| 666 |
+
num_beams=gen_kwargs["num_beams"],
|
| 667 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 668 |
+
)
|
| 669 |
+
elif MODALITY == "VIDEO":
|
| 670 |
+
output_ids = self.model.generate(
|
| 671 |
+
inputs=input_ids,
|
| 672 |
+
images=video_data[0][0],
|
| 673 |
+
images_highres=video_data[0][1],
|
| 674 |
+
modalities=video_data[2],
|
| 675 |
+
speech=speechs,
|
| 676 |
+
speech_lengths=speech_lengths,
|
| 677 |
+
speech_chunks=speech_chunks,
|
| 678 |
+
speech_wav=speech_wavs,
|
| 679 |
+
attention_mask=attention_masks,
|
| 680 |
+
use_cache=True,
|
| 681 |
+
stopping_criteria=[stopping_criteria],
|
| 682 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 683 |
+
temperature=gen_kwargs["temperature"],
|
| 684 |
+
top_p=gen_kwargs["top_p"],
|
| 685 |
+
num_beams=gen_kwargs["num_beams"],
|
| 686 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 687 |
+
)
|
| 688 |
+
except Exception as e:
|
| 689 |
+
eval_logger.error(f"Error {e} in generating")
|
| 690 |
+
outputs = ""
|
| 691 |
+
res.append(outputs)
|
| 692 |
+
pbar.update(1)
|
| 693 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), outputs)
|
| 694 |
+
continue
|
| 695 |
+
outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
|
| 696 |
+
if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
|
| 697 |
+
eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{outputs}\n")
|
| 698 |
+
|
| 699 |
+
res.append(outputs)
|
| 700 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), outputs)
|
| 701 |
+
pbar.update(1)
|
| 702 |
+
# reorder this group of results back to original unsorted form
|
| 703 |
+
res = re_ords.get_original(res)
|
| 704 |
+
|
| 705 |
+
pbar.close()
|
| 706 |
+
return res
|
| 707 |
+
|
| 708 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 709 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/openai_compatible.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
from io import BytesIO
|
| 6 |
+
from typing import List, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import requests as url_requests
|
| 10 |
+
from accelerate import Accelerator, DistributedType
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
|
| 13 |
+
from lmms_eval.api.instance import Instance
|
| 14 |
+
from lmms_eval.api.model import lmms
|
| 15 |
+
from lmms_eval.api.registry import register_model
|
| 16 |
+
|
| 17 |
+
try:
|
| 18 |
+
from decord import VideoReader, cpu
|
| 19 |
+
except ImportError:
|
| 20 |
+
pass
|
| 21 |
+
|
| 22 |
+
from dotenv import find_dotenv, load_dotenv
|
| 23 |
+
from loguru import logger as eval_logger
|
| 24 |
+
from openai import OpenAI
|
| 25 |
+
from PIL import Image
|
| 26 |
+
|
| 27 |
+
load_dotenv(verbose=True)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@register_model("openai_compatible")
|
| 31 |
+
class OpenAICompatible(lmms):
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
model_version: str = "grok-2-latest",
|
| 35 |
+
timeout: int = 120,
|
| 36 |
+
max_retries: int = 5,
|
| 37 |
+
max_size_in_mb: int = 20,
|
| 38 |
+
continual_mode: bool = False,
|
| 39 |
+
response_persistent_folder: str = None,
|
| 40 |
+
**kwargs,
|
| 41 |
+
) -> None:
|
| 42 |
+
super().__init__()
|
| 43 |
+
self.model_version = model_version
|
| 44 |
+
self.timeout = timeout
|
| 45 |
+
self.max_retries = max_retries
|
| 46 |
+
self.max_size_in_mb = max_size_in_mb # some models have a limit on the size of the image
|
| 47 |
+
self.continual_mode = continual_mode
|
| 48 |
+
if self.continual_mode:
|
| 49 |
+
if response_persistent_folder is None:
|
| 50 |
+
raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
|
| 51 |
+
|
| 52 |
+
os.makedirs(response_persistent_folder, exist_ok=True)
|
| 53 |
+
self.response_persistent_folder = response_persistent_folder
|
| 54 |
+
self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
|
| 55 |
+
|
| 56 |
+
if os.path.exists(self.response_persistent_file):
|
| 57 |
+
with open(self.response_persistent_file, "r") as f:
|
| 58 |
+
self.response_cache = json.load(f)
|
| 59 |
+
self.cache_mode = "resume"
|
| 60 |
+
else:
|
| 61 |
+
self.response_cache = {}
|
| 62 |
+
self.cache_mode = "start"
|
| 63 |
+
|
| 64 |
+
self.client = OpenAI(api_key=os.getenv("OPENAI_COMPATIBLE_API_KEY"), base_url=os.getenv("OPENAI_COMPATIBLE_API_URL"))
|
| 65 |
+
|
| 66 |
+
accelerator = Accelerator()
|
| 67 |
+
# assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
|
| 68 |
+
if accelerator.num_processes > 1:
|
| 69 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 70 |
+
self.accelerator = accelerator
|
| 71 |
+
if self.accelerator.is_local_main_process:
|
| 72 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 73 |
+
self._rank = self.accelerator.local_process_index
|
| 74 |
+
self._world_size = self.accelerator.num_processes
|
| 75 |
+
else:
|
| 76 |
+
self.accelerator = accelerator
|
| 77 |
+
self._rank = self.accelerator.local_process_index
|
| 78 |
+
self._world_size = self.accelerator.num_processes
|
| 79 |
+
|
| 80 |
+
self.device = self.accelerator.device
|
| 81 |
+
|
| 82 |
+
# Function to encode the image
|
| 83 |
+
def encode_image(self, image: Union[Image.Image, str]):
|
| 84 |
+
max_size = self.max_size_in_mb * 1024 * 1024 # 20MB in bytes
|
| 85 |
+
if isinstance(image, str):
|
| 86 |
+
img = Image.open(image).convert("RGB")
|
| 87 |
+
else:
|
| 88 |
+
img = image.copy()
|
| 89 |
+
|
| 90 |
+
output_buffer = BytesIO()
|
| 91 |
+
img.save(output_buffer, format="PNG")
|
| 92 |
+
byte_data = output_buffer.getvalue()
|
| 93 |
+
|
| 94 |
+
# If image is too large, resize it while maintaining aspect ratio
|
| 95 |
+
while len(byte_data) > max_size and img.size[0] > 100 and img.size[1] > 100:
|
| 96 |
+
new_size = (int(img.size[0] * 0.75), int(img.size[1] * 0.75))
|
| 97 |
+
img = img.resize(new_size, Image.Resampling.LANCZOS)
|
| 98 |
+
|
| 99 |
+
output_buffer = BytesIO()
|
| 100 |
+
img.save(output_buffer, format="PNG")
|
| 101 |
+
byte_data = output_buffer.getvalue()
|
| 102 |
+
|
| 103 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 104 |
+
return base64_str
|
| 105 |
+
|
| 106 |
+
# Function to encode the video
|
| 107 |
+
def encode_video(self, video_path, for_get_frames_num):
|
| 108 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 109 |
+
total_frame_num = len(vr)
|
| 110 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int)
|
| 111 |
+
|
| 112 |
+
# Ensure the last frame is included
|
| 113 |
+
if total_frame_num - 1 not in uniform_sampled_frames:
|
| 114 |
+
uniform_sampled_frames = np.append(uniform_sampled_frames, total_frame_num - 1)
|
| 115 |
+
|
| 116 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 117 |
+
frames = vr.get_batch(frame_idx).asnumpy()
|
| 118 |
+
|
| 119 |
+
base64_frames = []
|
| 120 |
+
for frame in frames:
|
| 121 |
+
img = Image.fromarray(frame)
|
| 122 |
+
output_buffer = BytesIO()
|
| 123 |
+
img.save(output_buffer, format="PNG")
|
| 124 |
+
byte_data = output_buffer.getvalue()
|
| 125 |
+
base64_str = base64.b64encode(byte_data).decode("utf-8")
|
| 126 |
+
base64_frames.append(base64_str)
|
| 127 |
+
|
| 128 |
+
return base64_frames
|
| 129 |
+
|
| 130 |
+
def flatten(self, input):
|
| 131 |
+
new_list = []
|
| 132 |
+
for i in input:
|
| 133 |
+
for j in i:
|
| 134 |
+
new_list.append(j)
|
| 135 |
+
return new_list
|
| 136 |
+
|
| 137 |
+
def generate_until(self, requests) -> List[str]:
|
| 138 |
+
res = []
|
| 139 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 140 |
+
|
| 141 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 142 |
+
if self.continual_mode is True and self.cache_mode == "resume":
|
| 143 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 144 |
+
if doc_uuid in self.response_cache:
|
| 145 |
+
response_text = self.response_cache[doc_uuid]
|
| 146 |
+
if response_text:
|
| 147 |
+
res.append(response_text)
|
| 148 |
+
pbar.update(1)
|
| 149 |
+
continue
|
| 150 |
+
|
| 151 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 152 |
+
if None in visuals:
|
| 153 |
+
visuals = []
|
| 154 |
+
imgs = []
|
| 155 |
+
else:
|
| 156 |
+
visuals = self.flatten(visuals)
|
| 157 |
+
imgs = [] # multiple images or frames for video
|
| 158 |
+
for visual in visuals:
|
| 159 |
+
if isinstance(visual, str) and (".mp4" in visual or ".avi" in visual or ".mov" in visual or ".flv" in visual or ".wmv" in visual):
|
| 160 |
+
frames = self.encode_video(visual, self.max_frames_num)
|
| 161 |
+
imgs.extend(frames)
|
| 162 |
+
elif isinstance(visual, str) and (".jpg" in visual or ".jpeg" in visual or ".png" in visual or ".gif" in visual or ".bmp" in visual or ".tiff" in visual or ".webp" in visual):
|
| 163 |
+
img = self.encode_image(visual)
|
| 164 |
+
imgs.append(img)
|
| 165 |
+
elif isinstance(visual, Image.Image):
|
| 166 |
+
img = self.encode_image(visual)
|
| 167 |
+
imgs.append(img)
|
| 168 |
+
|
| 169 |
+
payload = {"messages": []}
|
| 170 |
+
payload["model"] = self.model_version
|
| 171 |
+
|
| 172 |
+
# When there is no image token in the context, append the image to the text
|
| 173 |
+
payload["messages"].append({"role": "user", "content": []})
|
| 174 |
+
payload["messages"][0]["content"].append({"type": "text", "text": contexts})
|
| 175 |
+
for img in imgs:
|
| 176 |
+
payload["messages"][0]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}})
|
| 177 |
+
|
| 178 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 179 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 180 |
+
if gen_kwargs["max_new_tokens"] > 4096:
|
| 181 |
+
gen_kwargs["max_new_tokens"] = 4096
|
| 182 |
+
if "temperature" not in gen_kwargs:
|
| 183 |
+
gen_kwargs["temperature"] = 0
|
| 184 |
+
if "top_p" not in gen_kwargs:
|
| 185 |
+
gen_kwargs["top_p"] = None
|
| 186 |
+
if "num_beams" not in gen_kwargs:
|
| 187 |
+
gen_kwargs["num_beams"] = 1
|
| 188 |
+
|
| 189 |
+
payload["max_tokens"] = gen_kwargs["max_new_tokens"]
|
| 190 |
+
payload["temperature"] = gen_kwargs["temperature"]
|
| 191 |
+
|
| 192 |
+
for attempt in range(self.max_retries):
|
| 193 |
+
try:
|
| 194 |
+
response = self.client.chat.completions.create(**payload)
|
| 195 |
+
response_text = response.choices[0].message.content
|
| 196 |
+
break # If successful, break out of the loop
|
| 197 |
+
|
| 198 |
+
except Exception as e:
|
| 199 |
+
error_msg = str(e)
|
| 200 |
+
eval_logger.info(f"Attempt {attempt + 1}/{self.max_retries} failed with error: {error_msg}")
|
| 201 |
+
|
| 202 |
+
# On last attempt, log error and set empty response
|
| 203 |
+
if attempt == self.max_retries - 1:
|
| 204 |
+
eval_logger.error(f"All {self.max_retries} attempts failed. Last error: {error_msg}")
|
| 205 |
+
response_text = ""
|
| 206 |
+
else:
|
| 207 |
+
time.sleep(self.timeout)
|
| 208 |
+
|
| 209 |
+
res.append(response_text)
|
| 210 |
+
pbar.update(1)
|
| 211 |
+
|
| 212 |
+
if self.continual_mode is True: # Cache the response
|
| 213 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 214 |
+
self.response_cache[doc_uuid] = response_text
|
| 215 |
+
with open(self.response_persistent_file, "w") as f:
|
| 216 |
+
json.dump(self.response_cache, f)
|
| 217 |
+
|
| 218 |
+
pbar.close()
|
| 219 |
+
return res
|
| 220 |
+
|
| 221 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 222 |
+
raise NotImplementedError("TODO: Implement multi-round generation for OpenAI compatible models")
|
| 223 |
+
|
| 224 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 225 |
+
raise NotImplementedError("TODO: Implement loglikelihood for OpenAI compatible models")
|
LaViDa/eval/lmms_eval/models/oryx.py
ADDED
|
@@ -0,0 +1,475 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import logging
|
| 3 |
+
import math
|
| 4 |
+
from datetime import timedelta
|
| 5 |
+
from typing import List, Optional, Sequence, Tuple, Union
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
|
| 10 |
+
from accelerate.state import AcceleratorState
|
| 11 |
+
from decord import VideoReader, cpu
|
| 12 |
+
from PIL import Image
|
| 13 |
+
from tqdm import tqdm
|
| 14 |
+
from transformers import AutoConfig
|
| 15 |
+
|
| 16 |
+
from lmms_eval.api.instance import Instance
|
| 17 |
+
from lmms_eval.api.model import lmms
|
| 18 |
+
from lmms_eval.api.registry import register_model
|
| 19 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav
|
| 20 |
+
|
| 21 |
+
eval_logger = logging.getLogger("lmms-eval")
|
| 22 |
+
import os
|
| 23 |
+
import sys
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
from oryx.constants import (
|
| 27 |
+
DEFAULT_IM_END_TOKEN,
|
| 28 |
+
DEFAULT_IM_START_TOKEN,
|
| 29 |
+
DEFAULT_IMAGE_TOKEN,
|
| 30 |
+
IMAGE_TOKEN_INDEX,
|
| 31 |
+
)
|
| 32 |
+
from oryx.conversation import SeparatorStyle, conv_templates
|
| 33 |
+
from oryx.mm_utils import (
|
| 34 |
+
KeywordsStoppingCriteria,
|
| 35 |
+
get_model_name_from_path,
|
| 36 |
+
process_anyres_highres_image_genli,
|
| 37 |
+
process_anyres_video_genli,
|
| 38 |
+
tokenizer_image_token,
|
| 39 |
+
)
|
| 40 |
+
from oryx.model.builder import load_pretrained_model
|
| 41 |
+
from oryx.model.language_model.oryx_llama import OryxConfig
|
| 42 |
+
except ImportError:
|
| 43 |
+
eval_logger.debug("Oryx is not installed. Please install Oryx to use this model.")
|
| 44 |
+
|
| 45 |
+
try:
|
| 46 |
+
from oryx.model.language_model.oryx_qwen import OryxQwenConfig
|
| 47 |
+
|
| 48 |
+
AutoConfig.register("oryx_qwen", OryxQwenConfig)
|
| 49 |
+
except:
|
| 50 |
+
eval_logger.debug("")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@register_model("oryx")
|
| 54 |
+
class Oryx(lmms):
|
| 55 |
+
def __init__(
|
| 56 |
+
self,
|
| 57 |
+
pretrained: str = "",
|
| 58 |
+
truncation: Optional[bool] = True,
|
| 59 |
+
device: Optional[str] = "cuda:0",
|
| 60 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 61 |
+
attn_implementation=(
|
| 62 |
+
"sdpa" if torch.__version__ >= "2.1.2" else "eager"
|
| 63 |
+
), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
|
| 64 |
+
device_map="",
|
| 65 |
+
conv_template="qwen_1_5",
|
| 66 |
+
use_cache=True,
|
| 67 |
+
truncate_context=False,
|
| 68 |
+
max_frames_num: int = 32,
|
| 69 |
+
mm_resampler_type: str = "spatial_pool",
|
| 70 |
+
overwrite: bool = True,
|
| 71 |
+
video_decode_backend: str = "decord",
|
| 72 |
+
**kwargs,
|
| 73 |
+
) -> None:
|
| 74 |
+
super().__init__()
|
| 75 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 76 |
+
|
| 77 |
+
accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
|
| 78 |
+
accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
|
| 79 |
+
if accelerator.num_processes > 1:
|
| 80 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 81 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 82 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 83 |
+
self._device = torch.device(device)
|
| 84 |
+
self.device_map = device_map
|
| 85 |
+
else:
|
| 86 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 87 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 88 |
+
|
| 89 |
+
self.pretrained = pretrained
|
| 90 |
+
self.model_name = get_model_name_from_path(pretrained)
|
| 91 |
+
self.video_decode_backend = video_decode_backend
|
| 92 |
+
# self._config = AutoConfig.from_pretrained(self.pretrained)
|
| 93 |
+
self.overwrite = overwrite
|
| 94 |
+
self.mm_resampler_type = mm_resampler_type
|
| 95 |
+
self.max_frames_num = int(max_frames_num)
|
| 96 |
+
if self.overwrite == True:
|
| 97 |
+
overwrite_config = {}
|
| 98 |
+
overwrite_config["mm_resampler_type"] = self.mm_resampler_type
|
| 99 |
+
overwrite_config["patchify_video_feature"] = False
|
| 100 |
+
overwrite_config["attn_implementation"] = attn_implementation
|
| 101 |
+
|
| 102 |
+
cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
|
| 103 |
+
|
| 104 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, self.model_name, device_map=self.device_map, overwrite_config=overwrite_config)
|
| 105 |
+
else:
|
| 106 |
+
self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(
|
| 107 |
+
pretrained,
|
| 108 |
+
None,
|
| 109 |
+
self.model_name,
|
| 110 |
+
device_map=self.device_map,
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
self._config = self._model.config
|
| 114 |
+
self.model.eval()
|
| 115 |
+
self.model.tie_weights()
|
| 116 |
+
self.truncation = truncation
|
| 117 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 118 |
+
self.conv_template = conv_template
|
| 119 |
+
self.use_cache = use_cache
|
| 120 |
+
self.truncate_context = truncate_context
|
| 121 |
+
if accelerator.num_processes > 1:
|
| 122 |
+
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 123 |
+
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
|
| 124 |
+
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
|
| 125 |
+
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
|
| 126 |
+
if accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 127 |
+
kwargs = {
|
| 128 |
+
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
|
| 129 |
+
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
|
| 130 |
+
}
|
| 131 |
+
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
|
| 132 |
+
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
|
| 133 |
+
if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
|
| 134 |
+
self._model = accelerator.prepare(self.model)
|
| 135 |
+
else:
|
| 136 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 137 |
+
self.accelerator = accelerator
|
| 138 |
+
if self.accelerator.is_local_main_process:
|
| 139 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 140 |
+
self._rank = self.accelerator.local_process_index
|
| 141 |
+
self._world_size = self.accelerator.num_processes
|
| 142 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 143 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
|
| 144 |
+
self._rank = 0
|
| 145 |
+
self._world_size = 1
|
| 146 |
+
else:
|
| 147 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 148 |
+
self.model.to(self._device)
|
| 149 |
+
self._rank = 0
|
| 150 |
+
self._world_size = 1
|
| 151 |
+
|
| 152 |
+
@property
|
| 153 |
+
def config(self):
|
| 154 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 155 |
+
return self._config
|
| 156 |
+
|
| 157 |
+
@property
|
| 158 |
+
def tokenizer(self):
|
| 159 |
+
return self._tokenizer
|
| 160 |
+
|
| 161 |
+
@property
|
| 162 |
+
def model(self):
|
| 163 |
+
# returns the model, unwrapping it if using Accelerate
|
| 164 |
+
if hasattr(self, "accelerator"):
|
| 165 |
+
return self.accelerator.unwrap_model(self._model)
|
| 166 |
+
else:
|
| 167 |
+
return self._model
|
| 168 |
+
|
| 169 |
+
@property
|
| 170 |
+
def eot_token_id(self):
|
| 171 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 172 |
+
return self.tokenizer.eos_token_id
|
| 173 |
+
|
| 174 |
+
@property
|
| 175 |
+
def max_length(self):
|
| 176 |
+
return self._max_length
|
| 177 |
+
|
| 178 |
+
def pad_sequence(self, input_ids, batch_first, padding_value):
|
| 179 |
+
if self.tokenizer.padding_side == "left":
|
| 180 |
+
input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
|
| 181 |
+
input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
|
| 182 |
+
if self.tokenizer.padding_side == "left":
|
| 183 |
+
input_ids = torch.flip(input_ids, [1])
|
| 184 |
+
return input_ids
|
| 185 |
+
|
| 186 |
+
@property
|
| 187 |
+
def batch_size(self):
|
| 188 |
+
return self.batch_size_per_gpu
|
| 189 |
+
|
| 190 |
+
@property
|
| 191 |
+
def device(self):
|
| 192 |
+
return self._device
|
| 193 |
+
|
| 194 |
+
@property
|
| 195 |
+
def rank(self):
|
| 196 |
+
return self._rank
|
| 197 |
+
|
| 198 |
+
@property
|
| 199 |
+
def world_size(self):
|
| 200 |
+
return self._world_size
|
| 201 |
+
|
| 202 |
+
def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
|
| 203 |
+
""" """
|
| 204 |
+
add_special_tokens = False if add_special_tokens is None else add_special_tokens
|
| 205 |
+
encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
|
| 206 |
+
# left-truncate the encoded context to be at most `left_truncate_len` tokens long
|
| 207 |
+
if left_truncate_len:
|
| 208 |
+
encoding = encoding[-left_truncate_len:]
|
| 209 |
+
return encoding
|
| 210 |
+
|
| 211 |
+
def load_video(self, video_path, max_frames_num):
|
| 212 |
+
vr = VideoReader(video_path, ctx=cpu(0))
|
| 213 |
+
total_frame_num = len(vr)
|
| 214 |
+
fps = round(vr.get_avg_fps())
|
| 215 |
+
|
| 216 |
+
uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
|
| 217 |
+
frame_idx = uniform_sampled_frames.tolist()
|
| 218 |
+
modality = "video"
|
| 219 |
+
|
| 220 |
+
spare_frames = vr.get_batch(frame_idx).asnumpy()
|
| 221 |
+
return spare_frames, modality # (frames, height, width, channels)
|
| 222 |
+
|
| 223 |
+
def tok_decode(self, tokens):
|
| 224 |
+
return self.tokenizer.decode(tokens)
|
| 225 |
+
|
| 226 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 227 |
+
res = []
|
| 228 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 229 |
+
|
| 230 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 231 |
+
# encode, pad, and truncate contexts for this batch
|
| 232 |
+
if type(doc_to_target) == str:
|
| 233 |
+
continuation = doc_to_target
|
| 234 |
+
else:
|
| 235 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 236 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 237 |
+
visuals = self.flatten(visuals)
|
| 238 |
+
videos = []
|
| 239 |
+
# video
|
| 240 |
+
if type(visuals[0][0]) == str:
|
| 241 |
+
for visual in visuals:
|
| 242 |
+
video = self.load_video(visual, self.max_frames_num)
|
| 243 |
+
video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].bfloat16().to(self.device)
|
| 244 |
+
videos.append(video)
|
| 245 |
+
task_type = "video"
|
| 246 |
+
# image
|
| 247 |
+
else:
|
| 248 |
+
for visual in visuals:
|
| 249 |
+
image_tensor_, image_highres_tensor_ = process_anyres_highres_image_genli(visual, self._image_processor)
|
| 250 |
+
image_tensor.append(image_tensor_)
|
| 251 |
+
image_highres_tensor.append(image_highres_tensor_)
|
| 252 |
+
if all(x.shape == image_tensor[0].shape for x in image_tensor):
|
| 253 |
+
image_tensor = torch.stack(image_tensor, dim=0)
|
| 254 |
+
if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor):
|
| 255 |
+
image_highres_tensor = torch.stack(image_highres_tensor, dim=0)
|
| 256 |
+
if type(image_tensor) is list:
|
| 257 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 258 |
+
else:
|
| 259 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 260 |
+
if type(image_highres_tensor) is list:
|
| 261 |
+
image_highres_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_highres_tensor]
|
| 262 |
+
else:
|
| 263 |
+
image_highres_tensor = image_highres_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 264 |
+
|
| 265 |
+
image_sizes = [visuals[idx].size for idx in range(len(visuals))]
|
| 266 |
+
task_type = "image"
|
| 267 |
+
|
| 268 |
+
qs = contexts
|
| 269 |
+
if self.model.config.mm_use_im_start_end:
|
| 270 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 271 |
+
else:
|
| 272 |
+
qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 273 |
+
|
| 274 |
+
conv = conv_templates[self.conv_template].copy()
|
| 275 |
+
conv.append_message(conv.roles[0], qs)
|
| 276 |
+
conv.append_message(conv.roles[1], None)
|
| 277 |
+
prompt = conv.get_prompt()
|
| 278 |
+
|
| 279 |
+
contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 280 |
+
|
| 281 |
+
conv = conv_templates[self.conv_template].copy()
|
| 282 |
+
conv.append_message(conv.roles[0], qs)
|
| 283 |
+
conv.append_message(conv.roles[1], continuation)
|
| 284 |
+
prompt = conv.get_prompt()
|
| 285 |
+
|
| 286 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 287 |
+
|
| 288 |
+
labels = input_ids.clone()
|
| 289 |
+
# Context part no need to calculate for loss
|
| 290 |
+
labels[0, : contxt_id.shape[1]] = -100
|
| 291 |
+
|
| 292 |
+
with torch.inference_mode():
|
| 293 |
+
if task_type == "video":
|
| 294 |
+
outputs = self.model(
|
| 295 |
+
input_ids=input_ids,
|
| 296 |
+
labels=labels,
|
| 297 |
+
modalities=["video"],
|
| 298 |
+
images=videos,
|
| 299 |
+
images_highres=videos,
|
| 300 |
+
)
|
| 301 |
+
else:
|
| 302 |
+
outputs = self.model(
|
| 303 |
+
input_ids=input_ids,
|
| 304 |
+
labels=labels,
|
| 305 |
+
modalities=["image"] * len(image_sizes),
|
| 306 |
+
images=image_tensor,
|
| 307 |
+
images_highres=image_highres_tensor,
|
| 308 |
+
image_sizes=image_sizes,
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
loss = outputs["loss"]
|
| 312 |
+
# loss = torch.exp(loss)
|
| 313 |
+
logits = outputs["logits"]
|
| 314 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 315 |
+
cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
|
| 316 |
+
greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
|
| 317 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 318 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 319 |
+
pbar.update(1)
|
| 320 |
+
pbar.close()
|
| 321 |
+
return res
|
| 322 |
+
|
| 323 |
+
def flatten(self, input):
|
| 324 |
+
new_list = []
|
| 325 |
+
for i in input:
|
| 326 |
+
for j in i:
|
| 327 |
+
new_list.append(j)
|
| 328 |
+
return new_list
|
| 329 |
+
|
| 330 |
+
def generate_until(self, requests) -> List[str]:
|
| 331 |
+
res = []
|
| 332 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 333 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 334 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 335 |
+
visuals = self.flatten(visuals)
|
| 336 |
+
videos = []
|
| 337 |
+
modalities = []
|
| 338 |
+
try:
|
| 339 |
+
if task == "mvbench_episodic_reasoning":
|
| 340 |
+
sampled_frm = min(len(visuals), self.max_frames_num)
|
| 341 |
+
indices = np.linspace(0, len(visuals) - 1, sampled_frm, dtype=int)
|
| 342 |
+
frames = [visuals[i] for i in indices]
|
| 343 |
+
video = np.stack([np.array(x) for x in frames])
|
| 344 |
+
modality = "video"
|
| 345 |
+
frames = []
|
| 346 |
+
for frame in video:
|
| 347 |
+
self._image_processor.do_resize = False
|
| 348 |
+
self._image_processor.do_center_crop = False
|
| 349 |
+
frames.append(process_anyres_video_genli(Image.fromarray(frame).convert("RGB"), self._image_processor))
|
| 350 |
+
video = torch.stack(frames, dim=0).bfloat16().to(self.device)
|
| 351 |
+
videos.append(video)
|
| 352 |
+
modalities.append(modality)
|
| 353 |
+
else:
|
| 354 |
+
if type(visuals[0][0]) == str:
|
| 355 |
+
for visual in visuals:
|
| 356 |
+
if self.video_decode_backend == "decord":
|
| 357 |
+
video, modality = self.load_video(visual, self.max_frames_num)
|
| 358 |
+
elif self.video_decode_backend == "pyav":
|
| 359 |
+
video, modality = read_video_pyav(visual, num_frm=self.max_frames_num)
|
| 360 |
+
# video = self.load_video(visual, self.max_frames_num)
|
| 361 |
+
frames = []
|
| 362 |
+
for frame in video:
|
| 363 |
+
self._image_processor.do_resize = False
|
| 364 |
+
self._image_processor.do_center_crop = False
|
| 365 |
+
frames.append(process_anyres_video_genli(Image.fromarray(frame).convert("RGB"), self._image_processor))
|
| 366 |
+
video = torch.stack(frames, dim=0).bfloat16().to(self.device)
|
| 367 |
+
videos.append(video)
|
| 368 |
+
modalities.append(modality)
|
| 369 |
+
task_type = "video"
|
| 370 |
+
else:
|
| 371 |
+
self._image_processor.do_resize = False
|
| 372 |
+
self._image_processor.do_center_crop = False
|
| 373 |
+
image_tensor, image_highres_tensor = [], []
|
| 374 |
+
for visual in visuals:
|
| 375 |
+
image_tensor_, image_highres_tensor_ = process_anyres_highres_image_genli(visual, self._image_processor)
|
| 376 |
+
image_tensor.append(image_tensor_)
|
| 377 |
+
image_highres_tensor.append(image_highres_tensor_)
|
| 378 |
+
if all(x.shape == image_tensor[0].shape for x in image_tensor):
|
| 379 |
+
image_tensor = torch.stack(image_tensor, dim=0)
|
| 380 |
+
if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor):
|
| 381 |
+
image_highres_tensor = torch.stack(image_highres_tensor, dim=0)
|
| 382 |
+
if type(image_tensor) is list:
|
| 383 |
+
image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
|
| 384 |
+
else:
|
| 385 |
+
image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 386 |
+
if type(image_highres_tensor) is list:
|
| 387 |
+
image_highres_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_highres_tensor]
|
| 388 |
+
else:
|
| 389 |
+
image_highres_tensor = image_highres_tensor.to(dtype=torch.bfloat16, device=self.device)
|
| 390 |
+
task_type = "image"
|
| 391 |
+
|
| 392 |
+
except Exception as e:
|
| 393 |
+
eval_logger.info(f"{e}")
|
| 394 |
+
eval_logger.info(f"Video {visuals} can not load, check the source")
|
| 395 |
+
video_path = "\n".join(visuals)
|
| 396 |
+
res.append(f"Video {video_path} can not load, check the source")
|
| 397 |
+
pbar.update(1)
|
| 398 |
+
continue
|
| 399 |
+
|
| 400 |
+
qs = contexts
|
| 401 |
+
if self.model.config.mm_use_im_start_end:
|
| 402 |
+
qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
|
| 403 |
+
else:
|
| 404 |
+
qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
|
| 405 |
+
|
| 406 |
+
conv = conv_templates[self.conv_template].copy()
|
| 407 |
+
|
| 408 |
+
conv.append_message(conv.roles[0], qs)
|
| 409 |
+
conv.append_message(conv.roles[1], None)
|
| 410 |
+
prompt = conv.get_prompt()
|
| 411 |
+
|
| 412 |
+
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
|
| 413 |
+
pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
|
| 414 |
+
attention_masks = input_ids.ne(pad_token_ids).long().to(self.device)
|
| 415 |
+
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
|
| 416 |
+
keywords = [stop_str]
|
| 417 |
+
stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
|
| 418 |
+
|
| 419 |
+
cur_prompt = contexts
|
| 420 |
+
if task_type == "image":
|
| 421 |
+
gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
|
| 422 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 423 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 424 |
+
if "temperature" not in gen_kwargs:
|
| 425 |
+
gen_kwargs["temperature"] = 0.2
|
| 426 |
+
if "top_p" not in gen_kwargs:
|
| 427 |
+
gen_kwargs["top_p"] = None
|
| 428 |
+
if "num_beams" not in gen_kwargs:
|
| 429 |
+
gen_kwargs["num_beams"] = 1
|
| 430 |
+
|
| 431 |
+
try:
|
| 432 |
+
with torch.inference_mode():
|
| 433 |
+
if task_type == "video":
|
| 434 |
+
output_ids = self.model.generate(
|
| 435 |
+
inputs=input_ids,
|
| 436 |
+
images=videos,
|
| 437 |
+
images_highres=videos,
|
| 438 |
+
attention_mask=attention_masks,
|
| 439 |
+
modalities=modalities,
|
| 440 |
+
use_cache=self.use_cache,
|
| 441 |
+
stopping_criteria=[stopping_criteria],
|
| 442 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 443 |
+
temperature=gen_kwargs["temperature"],
|
| 444 |
+
top_p=gen_kwargs["top_p"],
|
| 445 |
+
num_beams=gen_kwargs["num_beams"],
|
| 446 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 447 |
+
)
|
| 448 |
+
else:
|
| 449 |
+
output_ids = self.model.generate(
|
| 450 |
+
input_ids,
|
| 451 |
+
attention_mask=attention_masks,
|
| 452 |
+
pad_token_id=pad_token_ids,
|
| 453 |
+
modalities=["image"] * len(gen_kwargs["image_sizes"]),
|
| 454 |
+
images=image_tensor,
|
| 455 |
+
images_highres=image_highres_tensor,
|
| 456 |
+
image_sizes=gen_kwargs["image_sizes"],
|
| 457 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 458 |
+
temperature=gen_kwargs["temperature"],
|
| 459 |
+
top_p=gen_kwargs["top_p"],
|
| 460 |
+
num_beams=gen_kwargs["num_beams"],
|
| 461 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 462 |
+
use_cache=self.use_cache,
|
| 463 |
+
)
|
| 464 |
+
outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
|
| 465 |
+
# print(outputs)
|
| 466 |
+
res.append(outputs)
|
| 467 |
+
pbar.update(1)
|
| 468 |
+
except Exception as e:
|
| 469 |
+
eval_logger.info(f"{e}")
|
| 470 |
+
eval_logger.info(f"Video {visuals} generate failed, check the source")
|
| 471 |
+
video_path = "\n".join(visuals)
|
| 472 |
+
res.append(f"Video {video_path} generate failed, check the source")
|
| 473 |
+
pbar.update(1)
|
| 474 |
+
continue
|
| 475 |
+
return res
|
LaViDa/eval/lmms_eval/models/phi3v.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List, Optional, Tuple, Union
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from accelerate import Accelerator, DistributedType
|
| 5 |
+
from loguru import logger as eval_logger
|
| 6 |
+
from tqdm import tqdm
|
| 7 |
+
from transformers import AutoModelForCausalLM, AutoProcessor
|
| 8 |
+
|
| 9 |
+
from lmms_eval import utils
|
| 10 |
+
from lmms_eval.api.instance import Instance
|
| 11 |
+
from lmms_eval.api.model import lmms
|
| 12 |
+
from lmms_eval.api.registry import register_model
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@register_model("phi3v")
|
| 16 |
+
class Phi3v(lmms):
|
| 17 |
+
"""
|
| 18 |
+
This class implements inference for the microsoft/Phi-3-vision-128k-instruct model.
|
| 19 |
+
To learn more about this model please visit the following links:
|
| 20 |
+
1. https://huggingface.co/microsoft/Phi-3-vision-128k-instruct
|
| 21 |
+
2. https://azure.microsoft.com/en-us/blog/new-models-added-to-the-phi-3-family-available-on-microsoft-azure/
|
| 22 |
+
3. https://github.com/microsoft/Phi-3CookBook
|
| 23 |
+
|
| 24 |
+
NOTE: This class was adapted from quen_vl.py and llava_hf.py.
|
| 25 |
+
|
| 26 |
+
Example:
|
| 27 |
+
|
| 28 |
+
accelerate launch --num_processes=4 -m lmms_eval --model phi3v --tasks mmmu_val \
|
| 29 |
+
--batch_size 1 --log_samples --log_samples_suffix phi3v_mmmu --output_path ./logs/
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
model_id_name: str = "microsoft/Phi-3-vision-128k-instruct",
|
| 35 |
+
device: str = "cuda",
|
| 36 |
+
dtype: Optional[Union[str, torch.dtype]] = "auto",
|
| 37 |
+
batch_size: int = 1,
|
| 38 |
+
trust_remote_code: Optional[bool] = True,
|
| 39 |
+
use_cache: bool = True,
|
| 40 |
+
**kwargs,
|
| 41 |
+
) -> None:
|
| 42 |
+
super().__init__()
|
| 43 |
+
# Do not use kwargs for now
|
| 44 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 45 |
+
# Setup accelerator.
|
| 46 |
+
accelerator = Accelerator()
|
| 47 |
+
if accelerator.num_processes > 1:
|
| 48 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 49 |
+
else:
|
| 50 |
+
self._device = device
|
| 51 |
+
# Load model.
|
| 52 |
+
self._model = AutoModelForCausalLM.from_pretrained(model_id_name, device_map=device, trust_remote_code=trust_remote_code, torch_dtype=dtype)
|
| 53 |
+
self._processor = AutoProcessor.from_pretrained(model_id_name, trust_remote_code=trust_remote_code)
|
| 54 |
+
self._processor.tokenizer.padding_side = "left"
|
| 55 |
+
self._tokenizer = self._processor.tokenizer
|
| 56 |
+
self._config = self._model.config
|
| 57 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 58 |
+
assert self.batch_size_per_gpu == 1, "batch_size_per_gpu > 1 is not supported for now."
|
| 59 |
+
self.use_cache = use_cache
|
| 60 |
+
if accelerator.num_processes > 1:
|
| 61 |
+
distributed_type_list = [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
|
| 62 |
+
assert accelerator.distributed_type in distributed_type_list, "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 63 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 64 |
+
self._model = accelerator.prepare(self.model)
|
| 65 |
+
else:
|
| 66 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 67 |
+
self.accelerator = accelerator
|
| 68 |
+
if self.accelerator.is_local_main_process:
|
| 69 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 70 |
+
self._rank = self.accelerator.local_process_index
|
| 71 |
+
self._world_size = self.accelerator.num_processes
|
| 72 |
+
else:
|
| 73 |
+
eval_logger.info(f"Using single device: {self._device}")
|
| 74 |
+
self.model.to(self._device)
|
| 75 |
+
self._rank = 0
|
| 76 |
+
self._world_size = 1
|
| 77 |
+
|
| 78 |
+
@property
|
| 79 |
+
def config(self):
|
| 80 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 81 |
+
return self._config
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def tokenizer(self):
|
| 85 |
+
return self._tokenizer
|
| 86 |
+
|
| 87 |
+
@property
|
| 88 |
+
def model(self):
|
| 89 |
+
# returns the model, unwrapping it if using Accelerate
|
| 90 |
+
if hasattr(self, "accelerator"):
|
| 91 |
+
return self.accelerator.unwrap_model(self._model)
|
| 92 |
+
else:
|
| 93 |
+
return self._model
|
| 94 |
+
|
| 95 |
+
@property
|
| 96 |
+
def eot_token_id(self):
|
| 97 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 98 |
+
return self.tokenizer.eos_token_id
|
| 99 |
+
|
| 100 |
+
@property
|
| 101 |
+
def max_length(self):
|
| 102 |
+
return self._max_length
|
| 103 |
+
|
| 104 |
+
@property
|
| 105 |
+
def batch_size(self):
|
| 106 |
+
return self.batch_size_per_gpu
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def device(self):
|
| 110 |
+
return self._device
|
| 111 |
+
|
| 112 |
+
@property
|
| 113 |
+
def rank(self):
|
| 114 |
+
return self._rank
|
| 115 |
+
|
| 116 |
+
@property
|
| 117 |
+
def world_size(self):
|
| 118 |
+
return self._world_size
|
| 119 |
+
|
| 120 |
+
def flatten(self, input):
|
| 121 |
+
new_list = []
|
| 122 |
+
for i in input:
|
| 123 |
+
for j in i:
|
| 124 |
+
new_list.append(j)
|
| 125 |
+
return new_list
|
| 126 |
+
|
| 127 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 128 |
+
raise NotImplementedError("Not implemented for Phi3v.")
|
| 129 |
+
|
| 130 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 131 |
+
res = []
|
| 132 |
+
|
| 133 |
+
def _collate(x):
|
| 134 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 135 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 136 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 137 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 138 |
+
# automatic adaptive batches much much easier to implement
|
| 139 |
+
# - any OOMs will happen right away rather than near the end
|
| 140 |
+
toks = self.tokenizer.encode(x[0])
|
| 141 |
+
return -len(toks), x[0]
|
| 142 |
+
|
| 143 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 144 |
+
# we group requests by their generation_kwargs,
|
| 145 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 146 |
+
# in the same batch.
|
| 147 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 148 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 149 |
+
for chunk in chunks:
|
| 150 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 151 |
+
task = task[0]
|
| 152 |
+
split = split[0]
|
| 153 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 154 |
+
visuals = self.flatten(visuals)
|
| 155 |
+
# We assume all gen kwargs in the batch are the same
|
| 156 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 157 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 158 |
+
# Set default values for until and max_new_tokens
|
| 159 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 160 |
+
# Update values from gen_kwargs if present
|
| 161 |
+
if "until" in gen_kwargs:
|
| 162 |
+
until = gen_kwargs.pop("until")
|
| 163 |
+
if isinstance(until, str):
|
| 164 |
+
until = [until]
|
| 165 |
+
elif not isinstance(until, list):
|
| 166 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 167 |
+
if isinstance(contexts, tuple):
|
| 168 |
+
contexts = list(contexts)
|
| 169 |
+
for i in range(len(contexts)):
|
| 170 |
+
if "<image>" in contexts[i]:
|
| 171 |
+
query = "" + contexts[i]
|
| 172 |
+
img_placeholder_count = 1
|
| 173 |
+
while "<image>" in query:
|
| 174 |
+
query = query.replace("<image>", f"<|image_{img_placeholder_count}|>", 1)
|
| 175 |
+
img_placeholder_count += 1
|
| 176 |
+
else:
|
| 177 |
+
query = ""
|
| 178 |
+
for placeholder_id in range(len(visuals)):
|
| 179 |
+
query += f"<|image_{placeholder_id+1}|>\n"
|
| 180 |
+
query += contexts[i]
|
| 181 |
+
messages = [{"role": "user", "content": query}]
|
| 182 |
+
contexts[i] = self._tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 183 |
+
assert len(contexts) == 1
|
| 184 |
+
#
|
| 185 |
+
context = contexts[0]
|
| 186 |
+
input_ids = self._processor(text=context, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
|
| 187 |
+
# Setting default parameters.
|
| 188 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 189 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 190 |
+
if "temperature" not in gen_kwargs:
|
| 191 |
+
gen_kwargs["temperature"] = 0
|
| 192 |
+
if "top_p" not in gen_kwargs:
|
| 193 |
+
gen_kwargs["top_p"] = None
|
| 194 |
+
if "num_beams" not in gen_kwargs:
|
| 195 |
+
gen_kwargs["num_beams"] = 1
|
| 196 |
+
# Generate answer.
|
| 197 |
+
pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eod_id
|
| 198 |
+
generate_ids = self.model.generate(
|
| 199 |
+
**input_ids,
|
| 200 |
+
eos_token_id=self.tokenizer.eos_token_id,
|
| 201 |
+
pad_token_id=pad_token_id,
|
| 202 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 203 |
+
temperature=gen_kwargs["temperature"],
|
| 204 |
+
top_p=gen_kwargs["top_p"],
|
| 205 |
+
num_beams=gen_kwargs["num_beams"],
|
| 206 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 207 |
+
use_cache=self.use_cache,
|
| 208 |
+
)
|
| 209 |
+
generate_ids = generate_ids[:, input_ids["input_ids"].shape[1] :]
|
| 210 |
+
response = self._processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
|
| 211 |
+
res.append(response)
|
| 212 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response)
|
| 213 |
+
pbar.update(1)
|
| 214 |
+
# reorder this group of results back to original unsorted form
|
| 215 |
+
res = re_ords.get_original(res)
|
| 216 |
+
pbar.close()
|
| 217 |
+
return res
|
| 218 |
+
|
| 219 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 220 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/qwen2_5_vl.py
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
from io import BytesIO
|
| 3 |
+
from typing import List, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import decord
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
from accelerate import Accelerator, DistributedType
|
| 9 |
+
from loguru import logger as eval_logger
|
| 10 |
+
from PIL import Image
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from transformers import (
|
| 13 |
+
AutoProcessor,
|
| 14 |
+
AutoTokenizer,
|
| 15 |
+
Qwen2_5_VLForConditionalGeneration,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
from lmms_eval import utils
|
| 19 |
+
from lmms_eval.api.instance import Instance
|
| 20 |
+
from lmms_eval.api.model import lmms
|
| 21 |
+
from lmms_eval.api.registry import register_model
|
| 22 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav_base64
|
| 23 |
+
|
| 24 |
+
try:
|
| 25 |
+
from qwen_vl_utils import process_vision_info
|
| 26 |
+
except ImportError:
|
| 27 |
+
eval_logger.warning("Failed to import qwen_vl_utils; Please install it via `pip install qwen-vl-utils`")
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@register_model("qwen2_5_vl")
|
| 31 |
+
class Qwen2_5_VL(lmms):
|
| 32 |
+
"""
|
| 33 |
+
Qwen2.5_VL Model
|
| 34 |
+
"https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct"
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def __init__(
|
| 38 |
+
self,
|
| 39 |
+
pretrained: str = "Qwen/Qwen2.5-VL-3B-Instruct",
|
| 40 |
+
device: Optional[str] = "cuda",
|
| 41 |
+
device_map: Optional[str] = "auto",
|
| 42 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 43 |
+
use_cache=True,
|
| 44 |
+
use_flash_attention_2: Optional[bool] = False,
|
| 45 |
+
min_pixels: int = 256 * 28 * 28,
|
| 46 |
+
max_pixels: int = 1605632,
|
| 47 |
+
max_num_frames: int = 32,
|
| 48 |
+
use_custom_video_loader: Optional[bool] = False,
|
| 49 |
+
fps: Optional[float] = None, # Only applicable if use_custom_video_loader is True
|
| 50 |
+
max_image_size: Optional[int] = None, # Only applicable if use_custom_video_loader is True
|
| 51 |
+
**kwargs,
|
| 52 |
+
) -> None:
|
| 53 |
+
super().__init__()
|
| 54 |
+
# Do not use kwargs for now
|
| 55 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 56 |
+
|
| 57 |
+
self.use_custom_video_loader = use_custom_video_loader
|
| 58 |
+
self.fps = fps
|
| 59 |
+
# if self.fps and not self.use_custom_video_loader:
|
| 60 |
+
# raise ValueError("FPS is only applicable if use_custom_video_loader is True")
|
| 61 |
+
self.max_image_size = max_image_size
|
| 62 |
+
if self.max_image_size and not self.use_custom_video_loader:
|
| 63 |
+
raise ValueError("max_image_size is only applicable if use_custom_video_loader is True")
|
| 64 |
+
|
| 65 |
+
accelerator = Accelerator()
|
| 66 |
+
if accelerator.num_processes > 1:
|
| 67 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 68 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 69 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 70 |
+
self._device = torch.device(device)
|
| 71 |
+
self.device_map = device_map
|
| 72 |
+
else:
|
| 73 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 74 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 75 |
+
|
| 76 |
+
if use_flash_attention_2:
|
| 77 |
+
self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
| 78 |
+
pretrained,
|
| 79 |
+
torch_dtype=torch.bfloat16,
|
| 80 |
+
device_map=self.device_map,
|
| 81 |
+
attn_implementation="flash_attention_2",
|
| 82 |
+
).eval()
|
| 83 |
+
else:
|
| 84 |
+
self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
|
| 85 |
+
self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
|
| 86 |
+
self.max_pixels = max_pixels
|
| 87 |
+
self.min_pixels = min_pixels
|
| 88 |
+
self.max_num_frames = max_num_frames
|
| 89 |
+
self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
|
| 90 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
|
| 91 |
+
|
| 92 |
+
self._config = self.model.config
|
| 93 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 94 |
+
self.use_cache = use_cache
|
| 95 |
+
|
| 96 |
+
if accelerator.num_processes > 1:
|
| 97 |
+
assert accelerator.distributed_type in [
|
| 98 |
+
DistributedType.FSDP,
|
| 99 |
+
DistributedType.MULTI_GPU,
|
| 100 |
+
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 101 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 102 |
+
self._model = accelerator.prepare(self.model)
|
| 103 |
+
else:
|
| 104 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 105 |
+
self.accelerator = accelerator
|
| 106 |
+
if self.accelerator.is_local_main_process:
|
| 107 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 108 |
+
self._rank = self.accelerator.local_process_index
|
| 109 |
+
self._world_size = self.accelerator.num_processes
|
| 110 |
+
else:
|
| 111 |
+
self._rank = 0
|
| 112 |
+
self._world_size = 1
|
| 113 |
+
|
| 114 |
+
@property
|
| 115 |
+
def config(self):
|
| 116 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 117 |
+
return self._config
|
| 118 |
+
|
| 119 |
+
@property
|
| 120 |
+
def tokenizer(self):
|
| 121 |
+
return self._tokenizer
|
| 122 |
+
|
| 123 |
+
@property
|
| 124 |
+
def model(self):
|
| 125 |
+
# returns the model, unwrapping it if using Accelerate
|
| 126 |
+
if hasattr(self, "accelerator"):
|
| 127 |
+
return self.accelerator.unwrap_model(self._model)
|
| 128 |
+
else:
|
| 129 |
+
return self._model
|
| 130 |
+
|
| 131 |
+
@property
|
| 132 |
+
def eot_token_id(self):
|
| 133 |
+
return self.tokenizer.eos_token_id
|
| 134 |
+
|
| 135 |
+
@property
|
| 136 |
+
def max_length(self):
|
| 137 |
+
return self._max_length
|
| 138 |
+
|
| 139 |
+
@property
|
| 140 |
+
def batch_size(self):
|
| 141 |
+
return self.batch_size_per_gpu
|
| 142 |
+
|
| 143 |
+
@property
|
| 144 |
+
def device(self):
|
| 145 |
+
return self._device
|
| 146 |
+
|
| 147 |
+
@property
|
| 148 |
+
def rank(self):
|
| 149 |
+
return self._rank
|
| 150 |
+
|
| 151 |
+
@property
|
| 152 |
+
def world_size(self):
|
| 153 |
+
return self._world_size
|
| 154 |
+
|
| 155 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 156 |
+
raise NotImplementedError("Loglikelihood is not implemented for Qwen2.5_VL")
|
| 157 |
+
|
| 158 |
+
def flatten(self, input):
|
| 159 |
+
new_list = []
|
| 160 |
+
for i in input:
|
| 161 |
+
for j in i:
|
| 162 |
+
new_list.append(j)
|
| 163 |
+
return new_list
|
| 164 |
+
|
| 165 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 166 |
+
res = []
|
| 167 |
+
|
| 168 |
+
def _collate(x):
|
| 169 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 170 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 171 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 172 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 173 |
+
# automatic adaptive batches much much easier to implement
|
| 174 |
+
# - any OOMs will happen right away rather than near the end
|
| 175 |
+
toks = self.tokenizer.encode(x[0])
|
| 176 |
+
return -len(toks), x[0]
|
| 177 |
+
|
| 178 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 179 |
+
# we group requests by their generation_kwargs,
|
| 180 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 181 |
+
# in the same batch.
|
| 182 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 183 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 184 |
+
for chunk in chunks:
|
| 185 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 186 |
+
task = task[0]
|
| 187 |
+
split = split[0]
|
| 188 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 189 |
+
visuals = self.flatten(visuals)
|
| 190 |
+
|
| 191 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 192 |
+
|
| 193 |
+
# Set default values for until and max_new_tokens
|
| 194 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 195 |
+
|
| 196 |
+
# Update values from gen_kwargs if present
|
| 197 |
+
if "until" in gen_kwargs:
|
| 198 |
+
until = gen_kwargs.pop("until")
|
| 199 |
+
if isinstance(until, str):
|
| 200 |
+
until = [until]
|
| 201 |
+
elif not isinstance(until, list):
|
| 202 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 203 |
+
|
| 204 |
+
# if isinstance(contexts, tuple):
|
| 205 |
+
# contexts = list(contexts)
|
| 206 |
+
|
| 207 |
+
# for i in range(len(contexts)):
|
| 208 |
+
# for j in range(32):
|
| 209 |
+
# if f"<image {j}>" in contexts[i]:
|
| 210 |
+
# contexts[i] = contexts[i].replace(f"<image {j}>", "<image>")
|
| 211 |
+
# if f"\\<image {j}\\>" in contexts[i]:
|
| 212 |
+
# contexts[i] = contexts[i].replace(f"\\<image {j}\\>", "<image>")
|
| 213 |
+
# if "<image>" in contexts[i]:
|
| 214 |
+
# contexts[i] = contexts[i].replace("<image>", "")
|
| 215 |
+
# print(contexts[i])
|
| 216 |
+
|
| 217 |
+
# for i in range(len(contexts)):
|
| 218 |
+
# if "<image>" in contexts[i]:
|
| 219 |
+
# contexts[i] = contexts[i].replace("<image>", "")
|
| 220 |
+
|
| 221 |
+
messages = []
|
| 222 |
+
processed_visuals = []
|
| 223 |
+
for i, context in enumerate(contexts):
|
| 224 |
+
# context += "\nPlease think step by step."
|
| 225 |
+
# if "<image>" in context:
|
| 226 |
+
# context = context.replace("<image>", "")
|
| 227 |
+
|
| 228 |
+
message = [{"role": "system", "content": "You are a helpful assistant."}]
|
| 229 |
+
|
| 230 |
+
if len(visuals) > 0:
|
| 231 |
+
visual = visuals[i] if i < len(visuals) else None
|
| 232 |
+
if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")): # Video file
|
| 233 |
+
if self.use_custom_video_loader:
|
| 234 |
+
visual = read_video_pyav_base64(visual, num_frm=self.max_num_frames, fps=self.fps, img_format="JPEG", max_image_size=self.max_image_size)
|
| 235 |
+
image_contents = list(map(lambda x: f"data:image/jpeg;base64,{x}", visual))
|
| 236 |
+
message.append({"role": "user", "content": [{"type": "video", "video": image_contents}, {"type": "text", "text": context}]})
|
| 237 |
+
else:
|
| 238 |
+
vr = decord.VideoReader(visual)
|
| 239 |
+
first_frame = vr[0].asnumpy()
|
| 240 |
+
height, width = first_frame.shape[:2]
|
| 241 |
+
# max_pixels = height * width
|
| 242 |
+
message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": 360 * 420}, {"type": "text", "text": context}]})
|
| 243 |
+
elif isinstance(visual, Image.Image): # Single image
|
| 244 |
+
base64_image = visual.convert("RGB")
|
| 245 |
+
buffer = BytesIO()
|
| 246 |
+
base64_image.save(buffer, format="JPEG")
|
| 247 |
+
base64_bytes = base64.b64encode(buffer.getvalue())
|
| 248 |
+
base64_string = base64_bytes.decode("utf-8")
|
| 249 |
+
message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
|
| 250 |
+
elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual): # Multiple images
|
| 251 |
+
image_content = []
|
| 252 |
+
for v in visual:
|
| 253 |
+
base64_image = v.convert("RGB")
|
| 254 |
+
buffer = BytesIO()
|
| 255 |
+
base64_image.save(buffer, format="JPEG")
|
| 256 |
+
base64_bytes = base64.b64encode(buffer.getvalue())
|
| 257 |
+
base64_string = base64_bytes.decode("utf-8")
|
| 258 |
+
image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
|
| 259 |
+
message.append({"role": "user", "content": image_content + [{"type": "text", "text": context}]})
|
| 260 |
+
else:
|
| 261 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context}]})
|
| 262 |
+
else:
|
| 263 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context}]})
|
| 264 |
+
|
| 265 |
+
messages.append(message)
|
| 266 |
+
# print("message")
|
| 267 |
+
|
| 268 |
+
text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 269 |
+
image_inputs, video_inputs = process_vision_info(messages)
|
| 270 |
+
inputs = self.processor(
|
| 271 |
+
text=text,
|
| 272 |
+
images=image_inputs,
|
| 273 |
+
videos=video_inputs,
|
| 274 |
+
# fps=self.fps,
|
| 275 |
+
padding=True,
|
| 276 |
+
return_tensors="pt",
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
if self.device_map == "auto":
|
| 280 |
+
inputs = inputs.to("cuda")
|
| 281 |
+
else:
|
| 282 |
+
inputs = inputs.to(self.device)
|
| 283 |
+
|
| 284 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 285 |
+
gen_kwargs["max_new_tokens"] = 4096
|
| 286 |
+
if "temperature" not in gen_kwargs:
|
| 287 |
+
gen_kwargs["temperature"] = 0
|
| 288 |
+
if "top_p" not in gen_kwargs:
|
| 289 |
+
gen_kwargs["top_p"] = None
|
| 290 |
+
if "num_beams" not in gen_kwargs:
|
| 291 |
+
gen_kwargs["num_beams"] = 1
|
| 292 |
+
|
| 293 |
+
pad_token_id = self.tokenizer.pad_token_id
|
| 294 |
+
|
| 295 |
+
cont = self.model.generate(
|
| 296 |
+
**inputs,
|
| 297 |
+
eos_token_id=self.tokenizer.eos_token_id,
|
| 298 |
+
pad_token_id=pad_token_id,
|
| 299 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 300 |
+
temperature=gen_kwargs["temperature"],
|
| 301 |
+
top_p=gen_kwargs["top_p"],
|
| 302 |
+
num_beams=gen_kwargs["num_beams"],
|
| 303 |
+
max_new_tokens=4096,
|
| 304 |
+
use_cache=self.use_cache,
|
| 305 |
+
)
|
| 306 |
+
|
| 307 |
+
generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
|
| 308 |
+
answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
|
| 309 |
+
for i, ans in enumerate(answers):
|
| 310 |
+
answers[i] = ans
|
| 311 |
+
|
| 312 |
+
for ans, context in zip(answers, contexts):
|
| 313 |
+
res.append(ans)
|
| 314 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
|
| 315 |
+
pbar.update(1)
|
| 316 |
+
# reorder this group of results back to original unsorted form
|
| 317 |
+
res = re_ords.get_original(res)
|
| 318 |
+
|
| 319 |
+
pbar.close()
|
| 320 |
+
return res
|
| 321 |
+
|
| 322 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 323 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/qwen2_5_vl_interleave.py
ADDED
|
@@ -0,0 +1,408 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
from io import BytesIO
|
| 5 |
+
from re import I
|
| 6 |
+
from typing import List, Optional, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import decord
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
from accelerate import Accelerator, DistributedType
|
| 12 |
+
from loguru import logger as eval_logger
|
| 13 |
+
from PIL import Image
|
| 14 |
+
from tqdm import tqdm
|
| 15 |
+
from transformers import (
|
| 16 |
+
AutoProcessor,
|
| 17 |
+
AutoTokenizer,
|
| 18 |
+
Qwen2_5_VLForConditionalGeneration,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
from lmms_eval import utils
|
| 22 |
+
from lmms_eval.api.instance import Instance
|
| 23 |
+
from lmms_eval.api.model import lmms
|
| 24 |
+
from lmms_eval.api.registry import register_model
|
| 25 |
+
from lmms_eval.models.model_utils.load_video import read_video_pyav_base64
|
| 26 |
+
|
| 27 |
+
try:
|
| 28 |
+
from qwen_vl_utils import process_vision_info
|
| 29 |
+
except ImportError:
|
| 30 |
+
eval_logger.warning("Failed to import qwen_vl_utils; Please install it via `pip install qwen-vl-utils`")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@register_model("qwen2_5_vl_interleave")
|
| 34 |
+
class Qwen2_5_VL_Interleave(lmms):
|
| 35 |
+
"""
|
| 36 |
+
Qwen2.5_VL Model
|
| 37 |
+
"https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct"
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
def __init__(
|
| 41 |
+
self,
|
| 42 |
+
pretrained: str = "Qwen/Qwen2.5-VL-3B-Instruct",
|
| 43 |
+
device: Optional[str] = "cuda",
|
| 44 |
+
device_map: Optional[str] = "auto",
|
| 45 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 46 |
+
use_cache=True,
|
| 47 |
+
use_flash_attention_2: Optional[bool] = False,
|
| 48 |
+
min_pixels: int = 256 * 28 * 28,
|
| 49 |
+
max_pixels: int = 256 * 28 * 28,
|
| 50 |
+
max_num_frames: int = 32,
|
| 51 |
+
use_custom_video_loader: Optional[bool] = False,
|
| 52 |
+
fps: Optional[float] = None, # Only applicable if use_custom_video_loader is True
|
| 53 |
+
max_image_size: Optional[int] = None, # Only applicable if use_custom_video_loader is True
|
| 54 |
+
**kwargs,
|
| 55 |
+
) -> None:
|
| 56 |
+
super().__init__()
|
| 57 |
+
# Do not use kwargs for now
|
| 58 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 59 |
+
|
| 60 |
+
self.use_custom_video_loader = use_custom_video_loader
|
| 61 |
+
self.fps = fps
|
| 62 |
+
# if self.fps and not self.use_custom_video_loader:
|
| 63 |
+
# raise ValueError("FPS is only applicable if use_custom_video_loader is True")
|
| 64 |
+
self.max_image_size = max_image_size
|
| 65 |
+
# if self.max_image_size and not self.use_custom_video_loader:
|
| 66 |
+
# raise ValueError("max_image_size is only applicable if use_custom_video_loader is True")
|
| 67 |
+
|
| 68 |
+
accelerator = Accelerator()
|
| 69 |
+
if accelerator.num_processes > 1:
|
| 70 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 71 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 72 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 73 |
+
self._device = torch.device(device)
|
| 74 |
+
self.device_map = device_map
|
| 75 |
+
else:
|
| 76 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 77 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 78 |
+
|
| 79 |
+
if use_flash_attention_2:
|
| 80 |
+
self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
| 81 |
+
pretrained,
|
| 82 |
+
torch_dtype=torch.bfloat16,
|
| 83 |
+
device_map=self.device_map,
|
| 84 |
+
attn_implementation="flash_attention_2",
|
| 85 |
+
).eval()
|
| 86 |
+
else:
|
| 87 |
+
self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
|
| 88 |
+
self.max_pixels = max_pixels
|
| 89 |
+
self.min_pixels = min_pixels
|
| 90 |
+
self.max_num_frames = max_num_frames
|
| 91 |
+
self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
|
| 92 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
|
| 93 |
+
|
| 94 |
+
self._config = self.model.config
|
| 95 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 96 |
+
self.use_cache = use_cache
|
| 97 |
+
|
| 98 |
+
if accelerator.num_processes > 1:
|
| 99 |
+
assert accelerator.distributed_type in [
|
| 100 |
+
DistributedType.FSDP,
|
| 101 |
+
DistributedType.MULTI_GPU,
|
| 102 |
+
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 103 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 104 |
+
self._model = accelerator.prepare(self.model)
|
| 105 |
+
else:
|
| 106 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 107 |
+
self.accelerator = accelerator
|
| 108 |
+
if self.accelerator.is_local_main_process:
|
| 109 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 110 |
+
self._rank = self.accelerator.local_process_index
|
| 111 |
+
self._world_size = self.accelerator.num_processes
|
| 112 |
+
else:
|
| 113 |
+
self._rank = 0
|
| 114 |
+
self._world_size = 1
|
| 115 |
+
|
| 116 |
+
# self.continual_mode = continual_mode
|
| 117 |
+
# if self.continual_mode and response_persistent_folder is None:
|
| 118 |
+
# raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.")
|
| 119 |
+
# if self.continual_mode:
|
| 120 |
+
# self.response_persistent_folder = response_persistent_folder
|
| 121 |
+
# if not os.path.exists(self.response_persistent_folder):
|
| 122 |
+
# os.makedirs(self.response_persistent_folder)
|
| 123 |
+
# self.model_version = pretrained.split("/")[-1].replace("-", "_").lower()
|
| 124 |
+
# self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
|
| 125 |
+
|
| 126 |
+
# if os.path.exists(self.response_persistent_file):
|
| 127 |
+
# with open(self.response_persistent_file, "r") as f:
|
| 128 |
+
# self.response_cache = json.load(f)
|
| 129 |
+
# self.cache_mode = "resume"
|
| 130 |
+
# else:
|
| 131 |
+
# self.response_cache = {}
|
| 132 |
+
# self.cache_mode = "start"
|
| 133 |
+
|
| 134 |
+
@property
|
| 135 |
+
def config(self):
|
| 136 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 137 |
+
return self._config
|
| 138 |
+
|
| 139 |
+
@property
|
| 140 |
+
def tokenizer(self):
|
| 141 |
+
return self._tokenizer
|
| 142 |
+
|
| 143 |
+
@property
|
| 144 |
+
def model(self):
|
| 145 |
+
# returns the model, unwrapping it if using Accelerate
|
| 146 |
+
if hasattr(self, "accelerator"):
|
| 147 |
+
return self.accelerator.unwrap_model(self._model)
|
| 148 |
+
else:
|
| 149 |
+
return self._model
|
| 150 |
+
|
| 151 |
+
@property
|
| 152 |
+
def eot_token_id(self):
|
| 153 |
+
return self.tokenizer.eos_token_id
|
| 154 |
+
|
| 155 |
+
@property
|
| 156 |
+
def max_length(self):
|
| 157 |
+
return self._max_length
|
| 158 |
+
|
| 159 |
+
@property
|
| 160 |
+
def batch_size(self):
|
| 161 |
+
return self.batch_size_per_gpu
|
| 162 |
+
|
| 163 |
+
@property
|
| 164 |
+
def device(self):
|
| 165 |
+
return self._device
|
| 166 |
+
|
| 167 |
+
@property
|
| 168 |
+
def rank(self):
|
| 169 |
+
return self._rank
|
| 170 |
+
|
| 171 |
+
@property
|
| 172 |
+
def world_size(self):
|
| 173 |
+
return self._world_size
|
| 174 |
+
|
| 175 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 176 |
+
raise NotImplementedError("Loglikelihood is not implemented for Qwen2.5_VL")
|
| 177 |
+
|
| 178 |
+
def flatten(self, input):
|
| 179 |
+
new_list = []
|
| 180 |
+
for i in input:
|
| 181 |
+
for j in i:
|
| 182 |
+
new_list.append(j)
|
| 183 |
+
return new_list
|
| 184 |
+
|
| 185 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 186 |
+
res = []
|
| 187 |
+
|
| 188 |
+
def get_uuid(task, split, doc_id):
|
| 189 |
+
return f"{task}___{split}___{doc_id}"
|
| 190 |
+
|
| 191 |
+
def _collate(x):
|
| 192 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 193 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 194 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 195 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 196 |
+
# automatic adaptive batches much much easier to implement
|
| 197 |
+
# - any OOMs will happen right away rather than near the end
|
| 198 |
+
toks = self.tokenizer.encode(x[0])
|
| 199 |
+
return -len(toks), x[0]
|
| 200 |
+
|
| 201 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 202 |
+
# we group requests by their generation_kwargs,
|
| 203 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 204 |
+
# in the same batch.
|
| 205 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 206 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 207 |
+
for chunk in chunks:
|
| 208 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 209 |
+
|
| 210 |
+
task = task[0]
|
| 211 |
+
split = split[0]
|
| 212 |
+
# if self.continual_mode and self.cache_mode == "resume":
|
| 213 |
+
# doc_uuid = get_uuid(task, split, doc_id)
|
| 214 |
+
# # print(doc_uuid)
|
| 215 |
+
# if doc_uuid in self.response_cache:
|
| 216 |
+
# content = self.response_cache[doc_uuid]
|
| 217 |
+
# if content:
|
| 218 |
+
# res.extend(content)
|
| 219 |
+
# pbar.update(1)
|
| 220 |
+
# continue
|
| 221 |
+
visuals = [doc_to_visual[i](self.task_dict[task][split][ids]) for i, ids in enumerate(doc_id)]
|
| 222 |
+
|
| 223 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 224 |
+
|
| 225 |
+
# Set default values for until and max_new_tokens
|
| 226 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 227 |
+
|
| 228 |
+
# Update values from gen_kwargs if present
|
| 229 |
+
if "until" in gen_kwargs:
|
| 230 |
+
until = gen_kwargs.pop("until")
|
| 231 |
+
if isinstance(until, str):
|
| 232 |
+
until = [until]
|
| 233 |
+
elif not isinstance(until, list):
|
| 234 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 235 |
+
|
| 236 |
+
if isinstance(contexts, tuple):
|
| 237 |
+
contexts = list(contexts)
|
| 238 |
+
|
| 239 |
+
# for i in range(len(contexts)):
|
| 240 |
+
# for j in range(32):
|
| 241 |
+
# if f"<image {j}>" in contexts[i]:
|
| 242 |
+
# contexts[i] = contexts[i].replace(f"<image {j}>", "<image>")
|
| 243 |
+
# if f"\\<image {j}\\>" in contexts[i]:
|
| 244 |
+
# contexts[i] = contexts[i].replace(f"\\<image {j}\\>", "<image>")
|
| 245 |
+
# if "<image>" in contexts[i]:
|
| 246 |
+
# contexts[i] = contexts[i].replace("<image>", "")
|
| 247 |
+
# print(contexts[i])
|
| 248 |
+
|
| 249 |
+
messages = []
|
| 250 |
+
processed_visuals = []
|
| 251 |
+
for i, context in enumerate(contexts):
|
| 252 |
+
# context += "\nPlease think step by step."
|
| 253 |
+
|
| 254 |
+
# print("context", context)
|
| 255 |
+
|
| 256 |
+
if "<image 1>" in context:
|
| 257 |
+
context = context.split("<image 1>")
|
| 258 |
+
# print(json.dumps(context, indent=4))
|
| 259 |
+
else:
|
| 260 |
+
context = [context]
|
| 261 |
+
|
| 262 |
+
message = [{"role": "system", "content": "You are a helpful assistant."}]
|
| 263 |
+
|
| 264 |
+
if len(visuals) > 0:
|
| 265 |
+
visual = visuals[i] if i < len(visuals) else None
|
| 266 |
+
# print("visuals", visual)
|
| 267 |
+
if isinstance(visual, Image.Image):
|
| 268 |
+
visual = [visual]
|
| 269 |
+
if isinstance(visual, (list, tuple)) and isinstance(visual[0], str):
|
| 270 |
+
assert len(visual) == 1, f"Expected a single video file but got {len(visual)} files"
|
| 271 |
+
visual = visual[0]
|
| 272 |
+
if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")): # Video file
|
| 273 |
+
if self.use_custom_video_loader:
|
| 274 |
+
try:
|
| 275 |
+
visual = read_video_pyav_base64(
|
| 276 |
+
visual,
|
| 277 |
+
num_frm=self.max_num_frames,
|
| 278 |
+
fps=self.fps,
|
| 279 |
+
img_format="JPEG",
|
| 280 |
+
max_image_size=self.max_image_size,
|
| 281 |
+
)
|
| 282 |
+
except Exception as e:
|
| 283 |
+
eval_logger.error(f"Failed to load video: {visual}. Error: {e}")
|
| 284 |
+
continue
|
| 285 |
+
|
| 286 |
+
image_contents = list(map(lambda x: f"data:image/jpeg;base64,{x}", visual))
|
| 287 |
+
if len(context) == 2:
|
| 288 |
+
if len(image_contents) == 1:
|
| 289 |
+
message.append(
|
| 290 |
+
{
|
| 291 |
+
"role": "user",
|
| 292 |
+
"content": [{"type": "video", "video": image_contents[:-1]}, {"type": "text", "text": context[0]}, {"type": "image", "image": image_contents[-1]}, {"type": "text", "text": context[1]}],
|
| 293 |
+
}
|
| 294 |
+
)
|
| 295 |
+
else:
|
| 296 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context[0]}, {"type": "image", "image": image_contents[-1]}, {"type": "text", "text": context[1]}]})
|
| 297 |
+
else:
|
| 298 |
+
message.append({"role": "user", "content": [{"type": "video", "video": image_contents}, {"type": "text", "text": context}]})
|
| 299 |
+
# with open("alb.json", "w") as f:
|
| 300 |
+
# json.dump(message, f, indent=4)
|
| 301 |
+
else:
|
| 302 |
+
vr = decord.VideoReader(visual)
|
| 303 |
+
first_frame = vr[0].asnumpy()
|
| 304 |
+
height, width = first_frame.shape[:2]
|
| 305 |
+
# max_pixels = height * width
|
| 306 |
+
message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": self.max_pixels}, {"type": "text", "text": context}]})
|
| 307 |
+
elif isinstance(visual, Image.Image): # Single image
|
| 308 |
+
base64_image = visual.convert("RGB")
|
| 309 |
+
buffer = BytesIO()
|
| 310 |
+
base64_image.save(buffer, format="JPEG")
|
| 311 |
+
base64_bytes = base64.b64encode(buffer.getvalue())
|
| 312 |
+
base64_string = base64_bytes.decode("utf-8")
|
| 313 |
+
message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
|
| 314 |
+
elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual): # Multiple images
|
| 315 |
+
image_content = []
|
| 316 |
+
i = 0
|
| 317 |
+
for v in visual:
|
| 318 |
+
base64_image = v.convert("RGB")
|
| 319 |
+
buffer = BytesIO()
|
| 320 |
+
base64_image.save(buffer, format="JPEG")
|
| 321 |
+
base64_bytes = base64.b64encode(buffer.getvalue())
|
| 322 |
+
base64_string = base64_bytes.decode("utf-8")
|
| 323 |
+
image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
|
| 324 |
+
v.save(f"test_{i}.jpg")
|
| 325 |
+
i += 1
|
| 326 |
+
# message.append({"role": "user", "content": image_content + [{"type": "text", "text": context}]})
|
| 327 |
+
assert len(image_content) + 1 == len(context), f"Number of images and context do not match, {len(image_content)} images and {len(context)} context\n{json.dumps(context)}"
|
| 328 |
+
content = []
|
| 329 |
+
for i in range(len(image_content)):
|
| 330 |
+
content.append({"type": "text", "text": context[i]})
|
| 331 |
+
content.append(image_content[i])
|
| 332 |
+
content.append({"type": "text", "text": context[-1]})
|
| 333 |
+
# print("content", content)
|
| 334 |
+
else:
|
| 335 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context}]})
|
| 336 |
+
else:
|
| 337 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context}]})
|
| 338 |
+
|
| 339 |
+
messages.append(message)
|
| 340 |
+
# print(messages)
|
| 341 |
+
|
| 342 |
+
texts = [self.processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True) for msg in messages]
|
| 343 |
+
image_inputs, video_inputs = process_vision_info(messages)
|
| 344 |
+
|
| 345 |
+
inputs = self.processor(
|
| 346 |
+
text=texts,
|
| 347 |
+
images=image_inputs,
|
| 348 |
+
videos=video_inputs,
|
| 349 |
+
padding=True,
|
| 350 |
+
# fps=self.fps,
|
| 351 |
+
return_tensors="pt",
|
| 352 |
+
)
|
| 353 |
+
|
| 354 |
+
if self.device_map == "auto":
|
| 355 |
+
inputs = inputs.to("cuda")
|
| 356 |
+
else:
|
| 357 |
+
inputs = inputs.to(self.device)
|
| 358 |
+
|
| 359 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 360 |
+
gen_kwargs["max_new_tokens"] = 128
|
| 361 |
+
if "temperature" not in gen_kwargs:
|
| 362 |
+
gen_kwargs["temperature"] = 0
|
| 363 |
+
if "top_p" not in gen_kwargs:
|
| 364 |
+
gen_kwargs["top_p"] = None
|
| 365 |
+
if "num_beams" not in gen_kwargs:
|
| 366 |
+
gen_kwargs["num_beams"] = 1
|
| 367 |
+
|
| 368 |
+
pad_token_id = self.tokenizer.pad_token_id
|
| 369 |
+
|
| 370 |
+
cont = self.model.generate(
|
| 371 |
+
**inputs,
|
| 372 |
+
eos_token_id=self.tokenizer.eos_token_id,
|
| 373 |
+
pad_token_id=pad_token_id,
|
| 374 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 375 |
+
temperature=gen_kwargs["temperature"],
|
| 376 |
+
top_p=gen_kwargs["top_p"],
|
| 377 |
+
num_beams=gen_kwargs["num_beams"],
|
| 378 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 379 |
+
use_cache=self.use_cache,
|
| 380 |
+
)
|
| 381 |
+
# cont = self.model.generate(**inputs, max_new_tokens=1024)
|
| 382 |
+
|
| 383 |
+
generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
|
| 384 |
+
answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
|
| 385 |
+
for i, ans in enumerate(answers):
|
| 386 |
+
answers[i] = ans
|
| 387 |
+
|
| 388 |
+
content = []
|
| 389 |
+
for ans, context in zip(answers, contexts):
|
| 390 |
+
res.append(ans)
|
| 391 |
+
content.append(ans)
|
| 392 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
|
| 393 |
+
pbar.update(1)
|
| 394 |
+
# reorder this group of results back to original unsorted form
|
| 395 |
+
|
| 396 |
+
# if self.continual_mode is True: # Cache the response
|
| 397 |
+
# doc_uuid = get_uuid(task, split, doc_id)
|
| 398 |
+
# self.response_cache[doc_uuid] = content
|
| 399 |
+
# with open(self.response_persistent_file, "w") as f:
|
| 400 |
+
# json.dump(self.response_cache, f)
|
| 401 |
+
|
| 402 |
+
res = re_ords.get_original(res)
|
| 403 |
+
|
| 404 |
+
pbar.close()
|
| 405 |
+
return res
|
| 406 |
+
|
| 407 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 408 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/qwen2_audio.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
from io import BytesIO
|
| 3 |
+
from typing import List, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import decord
|
| 6 |
+
import torch
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from loguru import logger as eval_logger
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
|
| 11 |
+
|
| 12 |
+
from lmms_eval import utils
|
| 13 |
+
from lmms_eval.api.instance import Instance
|
| 14 |
+
from lmms_eval.api.model import lmms
|
| 15 |
+
from lmms_eval.api.registry import register_model
|
| 16 |
+
from lmms_eval.models.model_utils.audio_processing import downsample_audio
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@register_model("qwen2_audio")
|
| 20 |
+
class Qwen2_Audio(lmms):
|
| 21 |
+
"""
|
| 22 |
+
Qwen2_Audio Model
|
| 23 |
+
"https://github.com/QwenLM/Qwen2-Audio"
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
def __init__(
|
| 27 |
+
self,
|
| 28 |
+
pretrained: str = "Qwen/Qwen2-Audio-7B", # Qwen/Qwen2-Audio-7B-Instruct
|
| 29 |
+
device: Optional[str] = "cuda",
|
| 30 |
+
device_map: Optional[str] = "cuda",
|
| 31 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 32 |
+
use_cache=True,
|
| 33 |
+
add_generation_prompt: bool = True,
|
| 34 |
+
add_system_prompt: bool = True,
|
| 35 |
+
simple_prompt: bool = False,
|
| 36 |
+
**kwargs,
|
| 37 |
+
) -> None:
|
| 38 |
+
super().__init__()
|
| 39 |
+
# Do not use kwargs for now
|
| 40 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 41 |
+
|
| 42 |
+
accelerator = Accelerator()
|
| 43 |
+
self.add_generation_prompt = add_generation_prompt
|
| 44 |
+
self.add_system_prompt = add_system_prompt
|
| 45 |
+
# If using simple prompt, only add "<|audio_bos|><|AUDIO|><|audio_eos|>"
|
| 46 |
+
# and then prompt to align with original Qwen2 Audio
|
| 47 |
+
self.simple_prompt = simple_prompt
|
| 48 |
+
if accelerator.num_processes > 1:
|
| 49 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 50 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 51 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 52 |
+
self._device = torch.device(device)
|
| 53 |
+
self.device_map = device_map
|
| 54 |
+
else:
|
| 55 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 56 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 57 |
+
|
| 58 |
+
self._model = Qwen2AudioForConditionalGeneration.from_pretrained(
|
| 59 |
+
pretrained,
|
| 60 |
+
torch_dtype="auto",
|
| 61 |
+
device_map=self.device_map,
|
| 62 |
+
).eval()
|
| 63 |
+
|
| 64 |
+
self.processor = AutoProcessor.from_pretrained(pretrained)
|
| 65 |
+
self.processor.tokenizer.padding_side = "left"
|
| 66 |
+
self._tokenizer = self.processor.tokenizer
|
| 67 |
+
|
| 68 |
+
if not self.add_system_prompt:
|
| 69 |
+
# Overwrite chat template to exclude system prompt
|
| 70 |
+
self.processor.chat_template = (
|
| 71 |
+
"{% set audio_count = namespace(value=0) %}"
|
| 72 |
+
"{% for message in messages %}"
|
| 73 |
+
"<|im_start|>{{ message['role'] }}\n"
|
| 74 |
+
"{% if message['content'] is string %}"
|
| 75 |
+
"{{ message['content'] }}<|im_end|>\n"
|
| 76 |
+
"{% else %}"
|
| 77 |
+
"{% for content in message['content'] %}"
|
| 78 |
+
"{% if 'audio' in content or 'audio_url' in content %}"
|
| 79 |
+
"{% set audio_count.value = audio_count.value + 1 %}"
|
| 80 |
+
"Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
|
| 81 |
+
"{% elif 'text' in content %}"
|
| 82 |
+
"{{ content['text'] }}"
|
| 83 |
+
"{% endif %}"
|
| 84 |
+
"{% endfor %}"
|
| 85 |
+
"<|im_end|>\n"
|
| 86 |
+
"{% endif %}"
|
| 87 |
+
"{% endfor %}"
|
| 88 |
+
"{% if add_generation_prompt %}"
|
| 89 |
+
"<|im_start|>assistant\n"
|
| 90 |
+
"{% endif %}"
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
self._config = self.model.config
|
| 94 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 95 |
+
self.use_cache = use_cache
|
| 96 |
+
|
| 97 |
+
if accelerator.num_processes > 1:
|
| 98 |
+
assert accelerator.distributed_type in [
|
| 99 |
+
DistributedType.FSDP,
|
| 100 |
+
DistributedType.MULTI_GPU,
|
| 101 |
+
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 102 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 103 |
+
self._model = accelerator.prepare(self.model)
|
| 104 |
+
else:
|
| 105 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 106 |
+
self.accelerator = accelerator
|
| 107 |
+
if self.accelerator.is_local_main_process:
|
| 108 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 109 |
+
self._rank = self.accelerator.local_process_index
|
| 110 |
+
self._world_size = self.accelerator.num_processes
|
| 111 |
+
else:
|
| 112 |
+
self.model.to(self._device)
|
| 113 |
+
self._rank = 0
|
| 114 |
+
self._world_size = 1
|
| 115 |
+
|
| 116 |
+
@property
|
| 117 |
+
def config(self):
|
| 118 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 119 |
+
return self._config
|
| 120 |
+
|
| 121 |
+
@property
|
| 122 |
+
def tokenizer(self):
|
| 123 |
+
return self._tokenizer
|
| 124 |
+
|
| 125 |
+
@property
|
| 126 |
+
def model(self):
|
| 127 |
+
# returns the model, unwrapping it if using Accelerate
|
| 128 |
+
if hasattr(self, "accelerator"):
|
| 129 |
+
return self.accelerator.unwrap_model(self._model)
|
| 130 |
+
else:
|
| 131 |
+
return self._model
|
| 132 |
+
|
| 133 |
+
@property
|
| 134 |
+
def eot_token_id(self):
|
| 135 |
+
return self.tokenizer.eos_token_id
|
| 136 |
+
|
| 137 |
+
@property
|
| 138 |
+
def max_length(self):
|
| 139 |
+
return self._max_length
|
| 140 |
+
|
| 141 |
+
@property
|
| 142 |
+
def batch_size(self):
|
| 143 |
+
return self.batch_size_per_gpu
|
| 144 |
+
|
| 145 |
+
@property
|
| 146 |
+
def device(self):
|
| 147 |
+
return self._device
|
| 148 |
+
|
| 149 |
+
@property
|
| 150 |
+
def rank(self):
|
| 151 |
+
return self._rank
|
| 152 |
+
|
| 153 |
+
@property
|
| 154 |
+
def world_size(self):
|
| 155 |
+
return self._world_size
|
| 156 |
+
|
| 157 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 158 |
+
raise NotImplementedError("Loglikelihood is not implemented for Qwen2_Audio")
|
| 159 |
+
|
| 160 |
+
def flatten(self, input):
|
| 161 |
+
new_list = []
|
| 162 |
+
for i in input:
|
| 163 |
+
for j in i:
|
| 164 |
+
new_list.append(j)
|
| 165 |
+
return new_list
|
| 166 |
+
|
| 167 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 168 |
+
res = []
|
| 169 |
+
|
| 170 |
+
def _collate(x):
|
| 171 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 172 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 173 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 174 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 175 |
+
# automatic adaptive batches much much easier to implement
|
| 176 |
+
# - any OOMs will happen right away rather than near the end
|
| 177 |
+
toks = self.tokenizer.encode(x[0])
|
| 178 |
+
return -len(toks), x[0]
|
| 179 |
+
|
| 180 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 181 |
+
# we group requests by their generation_kwargs,
|
| 182 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 183 |
+
# in the same batch.
|
| 184 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 185 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 186 |
+
for chunk in chunks:
|
| 187 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 188 |
+
task = task[0]
|
| 189 |
+
split = split[0]
|
| 190 |
+
batched_audios = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 191 |
+
flattened_audios = self.flatten(batched_audios)
|
| 192 |
+
|
| 193 |
+
# we assume all gen kwargs in the batch are the same
|
| 194 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 195 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 196 |
+
|
| 197 |
+
# Set default values for until and max_new_tokens
|
| 198 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 199 |
+
|
| 200 |
+
# Update values from gen_kwargs if present
|
| 201 |
+
if "until" in gen_kwargs:
|
| 202 |
+
until = gen_kwargs.pop("until")
|
| 203 |
+
if isinstance(until, str):
|
| 204 |
+
until = [until]
|
| 205 |
+
elif not isinstance(until, list):
|
| 206 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 207 |
+
|
| 208 |
+
# contexts = "<|audio_bos|><|AUDIO|><|audio_eos|>" + contexts
|
| 209 |
+
|
| 210 |
+
if isinstance(contexts, tuple):
|
| 211 |
+
contexts = list(contexts)
|
| 212 |
+
|
| 213 |
+
if not self.simple_prompt:
|
| 214 |
+
conversations = []
|
| 215 |
+
for idx, context in enumerate(contexts):
|
| 216 |
+
conv = [{"role": "user", "content": []}]
|
| 217 |
+
for _ in batched_audios[idx]:
|
| 218 |
+
# This placeholder is just use to make chat template work
|
| 219 |
+
# We already have the sampled audio array
|
| 220 |
+
conv[0]["content"].append({"type": "audio", "audio_url": "placeholder.wav"})
|
| 221 |
+
conv[0]["content"].append({"type": "text", "text": context})
|
| 222 |
+
conversations.append(conv)
|
| 223 |
+
|
| 224 |
+
text = [self.processor.apply_chat_template(conversation, add_generation_prompt=self.add_generation_prompt, tokenize=False) for conversation in conversations]
|
| 225 |
+
else:
|
| 226 |
+
text = ["<|audio_bos|><|AUDIO|><|audio_eos|>" + context for context in contexts]
|
| 227 |
+
audios = [downsample_audio(audio["array"], audio["sampling_rate"], self.processor.feature_extractor.sampling_rate) for audio in flattened_audios]
|
| 228 |
+
|
| 229 |
+
inputs = self.processor(text=text, audios=audios, return_tensors="pt", padding=True, sampling_rate=self.processor.feature_extractor.sampling_rate)
|
| 230 |
+
|
| 231 |
+
if self.device_map == "auto":
|
| 232 |
+
inputs = inputs.to("cuda")
|
| 233 |
+
else:
|
| 234 |
+
inputs = inputs.to(self.device)
|
| 235 |
+
|
| 236 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 237 |
+
gen_kwargs["max_new_tokens"] = 256
|
| 238 |
+
if "temperature" not in gen_kwargs:
|
| 239 |
+
gen_kwargs["temperature"] = 0
|
| 240 |
+
if "top_p" not in gen_kwargs:
|
| 241 |
+
gen_kwargs["top_p"] = None
|
| 242 |
+
if "num_beams" not in gen_kwargs:
|
| 243 |
+
gen_kwargs["num_beams"] = 1
|
| 244 |
+
|
| 245 |
+
try:
|
| 246 |
+
cont = self.model.generate(
|
| 247 |
+
**inputs,
|
| 248 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 249 |
+
temperature=gen_kwargs["temperature"],
|
| 250 |
+
top_p=gen_kwargs["top_p"],
|
| 251 |
+
num_beams=gen_kwargs["num_beams"],
|
| 252 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 253 |
+
min_new_tokens=1,
|
| 254 |
+
use_cache=self.use_cache,
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
# cont = self.model.generate(**inputs, max_new_tokens=256, min_new_tokens=1, do_sample=False)
|
| 258 |
+
|
| 259 |
+
generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
|
| 260 |
+
# generated_ids_trimmed = cont[:, inputs.input_ids.size(1):]
|
| 261 |
+
answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
|
| 262 |
+
for i, ans in enumerate(answers):
|
| 263 |
+
for term in until:
|
| 264 |
+
if len(term) > 0:
|
| 265 |
+
ans = ans.split(term)[0]
|
| 266 |
+
answers[i] = ans
|
| 267 |
+
|
| 268 |
+
except Exception as e:
|
| 269 |
+
eval_logger.debug(f"Error while generating: {e}. It is possibly due to blank audio in {contexts}")
|
| 270 |
+
answers = [""] * len(contexts)
|
| 271 |
+
|
| 272 |
+
for ans, context in zip(answers, contexts):
|
| 273 |
+
res.append(ans)
|
| 274 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
|
| 275 |
+
pbar.update(1)
|
| 276 |
+
|
| 277 |
+
# reorder this group of results back to original unsorted form
|
| 278 |
+
res = re_ords.get_original(res)
|
| 279 |
+
|
| 280 |
+
pbar.close()
|
| 281 |
+
return res
|
| 282 |
+
|
| 283 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 284 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/qwen2_vl.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
from io import BytesIO
|
| 3 |
+
from typing import List, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import decord
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
from accelerate import Accelerator, DistributedType
|
| 9 |
+
from loguru import logger as eval_logger
|
| 10 |
+
from PIL import Image
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration
|
| 13 |
+
|
| 14 |
+
from lmms_eval import utils
|
| 15 |
+
from lmms_eval.api.instance import Instance
|
| 16 |
+
from lmms_eval.api.model import lmms
|
| 17 |
+
from lmms_eval.api.registry import register_model
|
| 18 |
+
from lmms_eval.models.model_utils.load_video import load_video_decord
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
from qwen_vl_utils import process_vision_info
|
| 22 |
+
except ImportError:
|
| 23 |
+
eval_logger.warning("Failed to import qwen_vl_utils; Please install it via `pip install qwen-vl-utils`")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@register_model("qwen2_vl")
|
| 27 |
+
class Qwen2_VL(lmms):
|
| 28 |
+
"""
|
| 29 |
+
Qwen2_VL Model
|
| 30 |
+
"https://github.com/QwenLM/Qwen2-VL"
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
def __init__(
|
| 34 |
+
self,
|
| 35 |
+
pretrained: str = "Qwen/Qwen2-VL-7B-Instruct",
|
| 36 |
+
device: Optional[str] = "cuda",
|
| 37 |
+
device_map: Optional[str] = "cuda",
|
| 38 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 39 |
+
use_cache=True,
|
| 40 |
+
use_flash_attention_2: Optional[bool] = False,
|
| 41 |
+
max_pixels: int = 12845056,
|
| 42 |
+
min_pixels: int = 3136,
|
| 43 |
+
max_num_frames: int = 32,
|
| 44 |
+
**kwargs,
|
| 45 |
+
) -> None:
|
| 46 |
+
super().__init__()
|
| 47 |
+
# Do not use kwargs for now
|
| 48 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 49 |
+
|
| 50 |
+
accelerator = Accelerator()
|
| 51 |
+
if accelerator.num_processes > 1:
|
| 52 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 53 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 54 |
+
elif accelerator.num_processes == 1 and device_map == "auto":
|
| 55 |
+
self._device = torch.device(device)
|
| 56 |
+
self.device_map = device_map
|
| 57 |
+
else:
|
| 58 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 59 |
+
self.device_map = f"cuda:{accelerator.local_process_index}"
|
| 60 |
+
|
| 61 |
+
if use_flash_attention_2:
|
| 62 |
+
self._model = Qwen2VLForConditionalGeneration.from_pretrained(
|
| 63 |
+
pretrained,
|
| 64 |
+
torch_dtype="auto",
|
| 65 |
+
device_map=self.device_map,
|
| 66 |
+
attn_implementation="flash_attention_2",
|
| 67 |
+
).eval()
|
| 68 |
+
else:
|
| 69 |
+
self._model = Qwen2VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
|
| 70 |
+
self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
|
| 71 |
+
self.max_pixels = max_pixels
|
| 72 |
+
self.min_pixels = min_pixels
|
| 73 |
+
self.max_num_frames = max_num_frames
|
| 74 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
|
| 75 |
+
|
| 76 |
+
self._config = self.model.config
|
| 77 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 78 |
+
self.use_cache = use_cache
|
| 79 |
+
|
| 80 |
+
if accelerator.num_processes > 1:
|
| 81 |
+
assert accelerator.distributed_type in [
|
| 82 |
+
DistributedType.FSDP,
|
| 83 |
+
DistributedType.MULTI_GPU,
|
| 84 |
+
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 85 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 86 |
+
self._model = accelerator.prepare(self.model)
|
| 87 |
+
else:
|
| 88 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 89 |
+
self.accelerator = accelerator
|
| 90 |
+
if self.accelerator.is_local_main_process:
|
| 91 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 92 |
+
self._rank = self.accelerator.local_process_index
|
| 93 |
+
self._world_size = self.accelerator.num_processes
|
| 94 |
+
else:
|
| 95 |
+
self._rank = 0
|
| 96 |
+
self._world_size = 1
|
| 97 |
+
|
| 98 |
+
@property
|
| 99 |
+
def config(self):
|
| 100 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 101 |
+
return self._config
|
| 102 |
+
|
| 103 |
+
@property
|
| 104 |
+
def tokenizer(self):
|
| 105 |
+
return self._tokenizer
|
| 106 |
+
|
| 107 |
+
@property
|
| 108 |
+
def model(self):
|
| 109 |
+
# returns the model, unwrapping it if using Accelerate
|
| 110 |
+
if hasattr(self, "accelerator"):
|
| 111 |
+
return self.accelerator.unwrap_model(self._model)
|
| 112 |
+
else:
|
| 113 |
+
return self._model
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def eot_token_id(self):
|
| 117 |
+
return self.tokenizer.eos_token_id
|
| 118 |
+
|
| 119 |
+
@property
|
| 120 |
+
def max_length(self):
|
| 121 |
+
return self._max_length
|
| 122 |
+
|
| 123 |
+
@property
|
| 124 |
+
def batch_size(self):
|
| 125 |
+
return self.batch_size_per_gpu
|
| 126 |
+
|
| 127 |
+
@property
|
| 128 |
+
def device(self):
|
| 129 |
+
return self._device
|
| 130 |
+
|
| 131 |
+
@property
|
| 132 |
+
def rank(self):
|
| 133 |
+
return self._rank
|
| 134 |
+
|
| 135 |
+
@property
|
| 136 |
+
def world_size(self):
|
| 137 |
+
return self._world_size
|
| 138 |
+
|
| 139 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 140 |
+
raise NotImplementedError("Loglikelihood is not implemented for Qwen2_VL")
|
| 141 |
+
|
| 142 |
+
def flatten(self, input):
|
| 143 |
+
new_list = []
|
| 144 |
+
for i in input:
|
| 145 |
+
for j in i:
|
| 146 |
+
new_list.append(j)
|
| 147 |
+
return new_list
|
| 148 |
+
|
| 149 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 150 |
+
res = []
|
| 151 |
+
|
| 152 |
+
def _collate(x):
|
| 153 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 154 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 155 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 156 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 157 |
+
# automatic adaptive batches much much easier to implement
|
| 158 |
+
# - any OOMs will happen right away rather than near the end
|
| 159 |
+
toks = self.tokenizer.encode(x[0])
|
| 160 |
+
return -len(toks), x[0]
|
| 161 |
+
|
| 162 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 163 |
+
# we group requests by their generation_kwargs,
|
| 164 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 165 |
+
# in the same batch.
|
| 166 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 167 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 168 |
+
for chunk in chunks:
|
| 169 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 170 |
+
task = task[0]
|
| 171 |
+
split = split[0]
|
| 172 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 173 |
+
if None in visuals:
|
| 174 |
+
visuals = []
|
| 175 |
+
else:
|
| 176 |
+
visuals = self.flatten(visuals)
|
| 177 |
+
|
| 178 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 179 |
+
|
| 180 |
+
# Set default values for until and max_new_tokens
|
| 181 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 182 |
+
|
| 183 |
+
# Update values from gen_kwargs if present
|
| 184 |
+
if "until" in gen_kwargs:
|
| 185 |
+
until = gen_kwargs.pop("until")
|
| 186 |
+
if isinstance(until, str):
|
| 187 |
+
until = [until]
|
| 188 |
+
elif not isinstance(until, list):
|
| 189 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 190 |
+
|
| 191 |
+
if isinstance(contexts, tuple):
|
| 192 |
+
contexts = list(contexts)
|
| 193 |
+
|
| 194 |
+
for i in range(len(contexts)):
|
| 195 |
+
if "<image>" in contexts[i]:
|
| 196 |
+
contexts[i] = contexts[i].replace("<image>", "")
|
| 197 |
+
|
| 198 |
+
messages = []
|
| 199 |
+
processed_visuals = []
|
| 200 |
+
for i, context in enumerate(contexts):
|
| 201 |
+
if "<image>" in context:
|
| 202 |
+
context = context.replace("<image>", "")
|
| 203 |
+
|
| 204 |
+
message = [{"role": "system", "content": "You are a helpful assistant."}]
|
| 205 |
+
|
| 206 |
+
if len(visuals) > 0:
|
| 207 |
+
visual = visuals[i] if i < len(visuals) else None
|
| 208 |
+
if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")): # Video file
|
| 209 |
+
vr = decord.VideoReader(visual)
|
| 210 |
+
first_frame = vr[0].asnumpy()
|
| 211 |
+
height, width = first_frame.shape[:2]
|
| 212 |
+
# max_pixels = height * width
|
| 213 |
+
message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": self.max_pixels}, {"type": "text", "text": context}]})
|
| 214 |
+
elif isinstance(visual, Image.Image): # Single image
|
| 215 |
+
base64_image = visual.convert("RGB")
|
| 216 |
+
buffer = BytesIO()
|
| 217 |
+
base64_image.save(buffer, format="JPEG")
|
| 218 |
+
base64_bytes = base64.b64encode(buffer.getvalue())
|
| 219 |
+
base64_string = base64_bytes.decode("utf-8")
|
| 220 |
+
message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
|
| 221 |
+
elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual): # Multiple images
|
| 222 |
+
image_content = []
|
| 223 |
+
for v in visual:
|
| 224 |
+
base64_image = v.convert("RGB")
|
| 225 |
+
buffer = BytesIO()
|
| 226 |
+
base64_image.save(buffer, format="JPEG")
|
| 227 |
+
base64_bytes = base64.b64encode(buffer.getvalue())
|
| 228 |
+
base64_string = base64_bytes.decode("utf-8")
|
| 229 |
+
image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
|
| 230 |
+
message.append({"role": "user", "content": image_content + [{"type": "text", "text": context}]})
|
| 231 |
+
else:
|
| 232 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context}]})
|
| 233 |
+
else:
|
| 234 |
+
message.append({"role": "user", "content": [{"type": "text", "text": context}]})
|
| 235 |
+
|
| 236 |
+
messages.append(message)
|
| 237 |
+
|
| 238 |
+
texts = [self.processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True) for msg in messages]
|
| 239 |
+
image_inputs, video_inputs = process_vision_info(messages)
|
| 240 |
+
if video_inputs is not None:
|
| 241 |
+
total_frames = video_inputs[0].shape[0]
|
| 242 |
+
indices = np.linspace(0, total_frames - 1, self.max_num_frames, dtype=int)
|
| 243 |
+
# Append the last frame index if not already included
|
| 244 |
+
if total_frames - 1 not in indices:
|
| 245 |
+
indices = np.append(indices, total_frames - 1)
|
| 246 |
+
video_inputs[0] = video_inputs[0][indices]
|
| 247 |
+
inputs = self.processor(text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt")
|
| 248 |
+
|
| 249 |
+
if self.device_map == "auto":
|
| 250 |
+
inputs = inputs.to("cuda")
|
| 251 |
+
else:
|
| 252 |
+
inputs = inputs.to(self.device)
|
| 253 |
+
|
| 254 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 255 |
+
gen_kwargs["max_new_tokens"] = 128
|
| 256 |
+
if "temperature" not in gen_kwargs:
|
| 257 |
+
gen_kwargs["temperature"] = 0
|
| 258 |
+
if "top_p" not in gen_kwargs:
|
| 259 |
+
gen_kwargs["top_p"] = None
|
| 260 |
+
if "num_beams" not in gen_kwargs:
|
| 261 |
+
gen_kwargs["num_beams"] = 1
|
| 262 |
+
|
| 263 |
+
pad_token_id = self.tokenizer.pad_token_id
|
| 264 |
+
|
| 265 |
+
cont = self.model.generate(
|
| 266 |
+
**inputs,
|
| 267 |
+
eos_token_id=self.tokenizer.eos_token_id,
|
| 268 |
+
pad_token_id=pad_token_id,
|
| 269 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 270 |
+
temperature=gen_kwargs["temperature"],
|
| 271 |
+
top_p=gen_kwargs["top_p"],
|
| 272 |
+
num_beams=gen_kwargs["num_beams"],
|
| 273 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 274 |
+
use_cache=self.use_cache,
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
|
| 278 |
+
answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
|
| 279 |
+
for i, ans in enumerate(answers):
|
| 280 |
+
for term in until:
|
| 281 |
+
if len(term) > 0:
|
| 282 |
+
ans = ans.split(term)[0]
|
| 283 |
+
answers[i] = ans
|
| 284 |
+
|
| 285 |
+
for ans, context in zip(answers, contexts):
|
| 286 |
+
res.append(ans)
|
| 287 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
|
| 288 |
+
pbar.update(1)
|
| 289 |
+
# reorder this group of results back to original unsorted form
|
| 290 |
+
res = re_ords.get_original(res)
|
| 291 |
+
|
| 292 |
+
pbar.close()
|
| 293 |
+
return res
|
| 294 |
+
|
| 295 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 296 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/qwen_vl.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import uuid
|
| 3 |
+
import warnings
|
| 4 |
+
from typing import List, Optional, Tuple, Union
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from accelerate import Accelerator, DistributedType
|
| 8 |
+
from tqdm import tqdm
|
| 9 |
+
|
| 10 |
+
from lmms_eval import utils
|
| 11 |
+
from lmms_eval.api.instance import Instance
|
| 12 |
+
from lmms_eval.api.model import lmms
|
| 13 |
+
from lmms_eval.api.registry import register_model
|
| 14 |
+
from lmms_eval.models.model_utils.qwen.qwen_generate_utils import make_context
|
| 15 |
+
|
| 16 |
+
warnings.simplefilter("ignore", category=DeprecationWarning)
|
| 17 |
+
warnings.filterwarnings("ignore")
|
| 18 |
+
|
| 19 |
+
from loguru import logger as eval_logger
|
| 20 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@register_model("qwen_vl")
|
| 24 |
+
class Qwen_VL(lmms):
|
| 25 |
+
"""
|
| 26 |
+
Qwen_VL Model
|
| 27 |
+
https://github.com/QwenLM/Qwen-VL/blob/master/eval_mm/evaluate_vqa.py
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(
|
| 31 |
+
self,
|
| 32 |
+
pretrained: str = "Qwen/Qwen-VL",
|
| 33 |
+
device: Optional[str] = "cuda",
|
| 34 |
+
batch_size: Optional[Union[int, str]] = 1,
|
| 35 |
+
trust_remote_code: Optional[bool] = True,
|
| 36 |
+
use_cache=True,
|
| 37 |
+
**kwargs,
|
| 38 |
+
) -> None:
|
| 39 |
+
super().__init__()
|
| 40 |
+
# Do not use kwargs for now
|
| 41 |
+
assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
|
| 42 |
+
|
| 43 |
+
accelerator = Accelerator()
|
| 44 |
+
if accelerator.num_processes > 1:
|
| 45 |
+
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
|
| 46 |
+
else:
|
| 47 |
+
self._device = device
|
| 48 |
+
self._model = AutoModelForCausalLM.from_pretrained(pretrained, device_map=self._device, trust_remote_code=trust_remote_code).eval()
|
| 49 |
+
self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code)
|
| 50 |
+
self.tokenizer.padding_side = "left"
|
| 51 |
+
self.tokenizer.pad_token_id = self.tokenizer.eod_id
|
| 52 |
+
self.prompt = "<img>{}</img>{}"
|
| 53 |
+
self._config = self._model.config
|
| 54 |
+
self.model.tie_weights()
|
| 55 |
+
self.batch_size_per_gpu = int(batch_size)
|
| 56 |
+
self.use_cache = use_cache
|
| 57 |
+
if accelerator.num_processes > 1:
|
| 58 |
+
assert accelerator.distributed_type in [
|
| 59 |
+
DistributedType.FSDP,
|
| 60 |
+
DistributedType.MULTI_GPU,
|
| 61 |
+
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
|
| 62 |
+
if accelerator.distributed_type == DistributedType.FSDP:
|
| 63 |
+
self._model = accelerator.prepare(self.model)
|
| 64 |
+
else:
|
| 65 |
+
self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
|
| 66 |
+
self.accelerator = accelerator
|
| 67 |
+
if self.accelerator.is_local_main_process:
|
| 68 |
+
eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
|
| 69 |
+
self._rank = self.accelerator.local_process_index
|
| 70 |
+
self._world_size = self.accelerator.num_processes
|
| 71 |
+
else:
|
| 72 |
+
self.model.to(self._device)
|
| 73 |
+
self._rank = 0
|
| 74 |
+
self._world_size = 1
|
| 75 |
+
|
| 76 |
+
@property
|
| 77 |
+
def config(self):
|
| 78 |
+
# return the associated transformers.AutoConfig for the given pretrained model.
|
| 79 |
+
return self._config
|
| 80 |
+
|
| 81 |
+
@property
|
| 82 |
+
def tokenizer(self):
|
| 83 |
+
return self._tokenizer
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def model(self):
|
| 87 |
+
# returns the model, unwrapping it if using Accelerate
|
| 88 |
+
if hasattr(self, "accelerator"):
|
| 89 |
+
return self.accelerator.unwrap_model(self._model)
|
| 90 |
+
else:
|
| 91 |
+
return self._model
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def eot_token_id(self):
|
| 95 |
+
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
|
| 96 |
+
return self.tokenizer.eod_id
|
| 97 |
+
|
| 98 |
+
@property
|
| 99 |
+
def max_length(self):
|
| 100 |
+
return self._max_length
|
| 101 |
+
|
| 102 |
+
# should be deleted since max_new_tokens is decided by gen_kwargs not a model property
|
| 103 |
+
# @property
|
| 104 |
+
# def max_new_tokens(self) -> int:
|
| 105 |
+
# return 256
|
| 106 |
+
|
| 107 |
+
@property
|
| 108 |
+
def batch_size(self):
|
| 109 |
+
return self.batch_size_per_gpu
|
| 110 |
+
|
| 111 |
+
@property
|
| 112 |
+
def device(self):
|
| 113 |
+
return self._device
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def rank(self):
|
| 117 |
+
return self._rank
|
| 118 |
+
|
| 119 |
+
@property
|
| 120 |
+
def world_size(self):
|
| 121 |
+
return self._world_size
|
| 122 |
+
|
| 123 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 124 |
+
res = []
|
| 125 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 126 |
+
|
| 127 |
+
for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 128 |
+
# encode, pad, and truncate contexts for this batch
|
| 129 |
+
if type(doc_to_target) == str:
|
| 130 |
+
continuation = doc_to_target
|
| 131 |
+
else:
|
| 132 |
+
continuation = doc_to_target(self.task_dict[task][split][doc_id])
|
| 133 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 134 |
+
visuals = self.flatten(visuals)
|
| 135 |
+
query = []
|
| 136 |
+
visual_paths = []
|
| 137 |
+
for visual in visuals:
|
| 138 |
+
name = uuid.uuid4().hex.upper()[0:6]
|
| 139 |
+
visual.save(f"/tmp/{name}.png")
|
| 140 |
+
visual_paths.append(f"/tmp/{name}.png")
|
| 141 |
+
query.append({"image": f"/tmp/{name}.png"})
|
| 142 |
+
|
| 143 |
+
# Make a copy for query to save context (text that needs to be masked)
|
| 144 |
+
context_query = [_ for _ in query]
|
| 145 |
+
context_query.append({"text": contexts})
|
| 146 |
+
query.append({"text": contexts + continuation})
|
| 147 |
+
|
| 148 |
+
context_query = self.tokenizer.from_list_format(context_query)
|
| 149 |
+
query = self.tokenizer.from_list_format(query)
|
| 150 |
+
|
| 151 |
+
raw_contxt_text, context_tokens = make_context(
|
| 152 |
+
self.tokenizer, context_query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format
|
| 153 |
+
)
|
| 154 |
+
context_tokens = torch.tensor([context_tokens])
|
| 155 |
+
|
| 156 |
+
raw_continuation_text, continuation_tokens = make_context(
|
| 157 |
+
self.tokenizer, query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format
|
| 158 |
+
)
|
| 159 |
+
continuation_tokens = torch.tensor([continuation_tokens]).to(self.model.device)
|
| 160 |
+
attn_mask = torch.ones_like(continuation_tokens).to(self.model.device)
|
| 161 |
+
labels = continuation_tokens.clone().to(self.model.device)
|
| 162 |
+
labels[:, : context_tokens.shape[1]] = -100
|
| 163 |
+
with torch.inference_mode():
|
| 164 |
+
outputs = self.model(input_ids=continuation_tokens, labels=labels, attention_mask=attn_mask)
|
| 165 |
+
loss = outputs.loss
|
| 166 |
+
logits = outputs["logits"]
|
| 167 |
+
greedy_tokens = logits.argmax(dim=-1)
|
| 168 |
+
cont_toks = continuation_tokens[:, context_tokens.shape[1] :]
|
| 169 |
+
greedy_tokens = greedy_tokens[:, context_tokens.shape[1] : continuation_tokens.shape[1]] # [1, seq]
|
| 170 |
+
max_equal = (greedy_tokens == cont_toks).all()
|
| 171 |
+
res.append((float(loss.item()), bool(max_equal)))
|
| 172 |
+
pbar.update(1)
|
| 173 |
+
|
| 174 |
+
pbar.close()
|
| 175 |
+
return res
|
| 176 |
+
|
| 177 |
+
def flatten(self, input):
|
| 178 |
+
new_list = []
|
| 179 |
+
for i in input:
|
| 180 |
+
for j in i:
|
| 181 |
+
new_list.append(j)
|
| 182 |
+
return new_list
|
| 183 |
+
|
| 184 |
+
def generate_until(self, requests: List[Instance]) -> List[str]:
|
| 185 |
+
res = []
|
| 186 |
+
|
| 187 |
+
def _collate(x):
|
| 188 |
+
# the negative sign on len(toks) sorts descending - this has a few advantages:
|
| 189 |
+
# - time estimates will always be over not underestimates, which is more useful for planning
|
| 190 |
+
# - to know the size of a batch when going through the list, you know the first one is always the batch
|
| 191 |
+
# padded context length. this is useful to simplify the batching logic and more importantly to make
|
| 192 |
+
# automatic adaptive batches much much easier to implement
|
| 193 |
+
# - any OOMs will happen right away rather than near the end
|
| 194 |
+
toks = self.tokenizer.encode(x[0])
|
| 195 |
+
return -len(toks), x[0]
|
| 196 |
+
|
| 197 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 198 |
+
# we group requests by their generation_kwargs,
|
| 199 |
+
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
|
| 200 |
+
# in the same batch.
|
| 201 |
+
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
|
| 202 |
+
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
|
| 203 |
+
for chunk in chunks:
|
| 204 |
+
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
|
| 205 |
+
task = task[0]
|
| 206 |
+
split = split[0]
|
| 207 |
+
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
|
| 208 |
+
visuals = self.flatten(visuals)
|
| 209 |
+
visual_paths = []
|
| 210 |
+
# save images to /tmp, name generated by hash function
|
| 211 |
+
# qwen accept image path. Have to do it here....
|
| 212 |
+
for visual in visuals:
|
| 213 |
+
name = uuid.uuid4().hex.upper()[0:6]
|
| 214 |
+
visual.save(f"/tmp/{name}.png")
|
| 215 |
+
visual_paths.append(f"/tmp/{name}.png")
|
| 216 |
+
|
| 217 |
+
# we assume all gen kwargs in the batch are the same
|
| 218 |
+
# this is safe to assume because the `grouper` object ensures it.
|
| 219 |
+
gen_kwargs = all_gen_kwargs[0]
|
| 220 |
+
|
| 221 |
+
# Set default values for until and max_new_tokens
|
| 222 |
+
until = [self.tokenizer.decode(self.eot_token_id)]
|
| 223 |
+
|
| 224 |
+
# Update values from gen_kwargs if present
|
| 225 |
+
if "until" in gen_kwargs:
|
| 226 |
+
until = gen_kwargs.pop("until")
|
| 227 |
+
if isinstance(until, str):
|
| 228 |
+
until = [until]
|
| 229 |
+
elif not isinstance(until, list):
|
| 230 |
+
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
|
| 231 |
+
|
| 232 |
+
if isinstance(contexts, tuple):
|
| 233 |
+
contexts = list(contexts)
|
| 234 |
+
|
| 235 |
+
for i in range(len(contexts)):
|
| 236 |
+
if "<image>" in contexts[i]:
|
| 237 |
+
contexts[i] = contexts[i].replace("<image>", "")
|
| 238 |
+
|
| 239 |
+
# Similar to llava, is visual paths has len 0
|
| 240 |
+
# Then nothing will be executed
|
| 241 |
+
query = []
|
| 242 |
+
if len(visual_paths) == 0:
|
| 243 |
+
for context in contexts:
|
| 244 |
+
query.append({"text": context})
|
| 245 |
+
else:
|
| 246 |
+
for visual_path, context in zip(visual_paths, contexts):
|
| 247 |
+
query.append({"image": visual_path})
|
| 248 |
+
query.append({"text": context})
|
| 249 |
+
|
| 250 |
+
questions = self.tokenizer.from_list_format(query)
|
| 251 |
+
input_ids = self.tokenizer(questions, return_tensors="pt", padding="longest")
|
| 252 |
+
|
| 253 |
+
# preconfigure gen_kwargs with defaults
|
| 254 |
+
if "image_sizes" not in gen_kwargs:
|
| 255 |
+
try:
|
| 256 |
+
gen_kwargs["image_sizes"] = [visuals[0].size]
|
| 257 |
+
except:
|
| 258 |
+
gen_kwargs["image_sizes"] = None
|
| 259 |
+
if "max_new_tokens" not in gen_kwargs:
|
| 260 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 261 |
+
if "temperature" not in gen_kwargs:
|
| 262 |
+
gen_kwargs["temperature"] = 0
|
| 263 |
+
if "top_p" not in gen_kwargs:
|
| 264 |
+
gen_kwargs["top_p"] = None
|
| 265 |
+
if "num_beams" not in gen_kwargs:
|
| 266 |
+
gen_kwargs["num_beams"] = 1
|
| 267 |
+
|
| 268 |
+
pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eod_id
|
| 269 |
+
|
| 270 |
+
cont = self.model.generate(
|
| 271 |
+
input_ids.input_ids.to(self.device),
|
| 272 |
+
attention_mask=input_ids.attention_mask.to(self.device),
|
| 273 |
+
eos_token_id=self.tokenizer.eod_id,
|
| 274 |
+
pad_token_id=pad_token_id,
|
| 275 |
+
do_sample=True if gen_kwargs["temperature"] > 0 else False,
|
| 276 |
+
temperature=gen_kwargs["temperature"],
|
| 277 |
+
top_p=gen_kwargs["top_p"],
|
| 278 |
+
num_beams=gen_kwargs["num_beams"],
|
| 279 |
+
max_new_tokens=gen_kwargs["max_new_tokens"],
|
| 280 |
+
use_cache=self.use_cache,
|
| 281 |
+
# kwargs=gen_kwargs
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
cont_toks_list = cont.tolist()
|
| 285 |
+
for cont_toks, context in zip(cont_toks_list, contexts):
|
| 286 |
+
# discard context + left-padding toks if using causal decoder-only LMM
|
| 287 |
+
cont_toks = cont_toks[input_ids.input_ids.shape[1] :]
|
| 288 |
+
text_outputs = self.tokenizer.decode(cont_toks, skip_special_tokens=True).strip()
|
| 289 |
+
for term in until:
|
| 290 |
+
if len(term) > 0:
|
| 291 |
+
# ignore '' separator,
|
| 292 |
+
# for seq2seq case where self.tok_decode(self.eot_token_id) = ''
|
| 293 |
+
text_outputs = text_outputs.split(term)[0]
|
| 294 |
+
|
| 295 |
+
res.append(text_outputs)
|
| 296 |
+
|
| 297 |
+
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
|
| 298 |
+
# remove visuals from tmp
|
| 299 |
+
for visual_path in visual_paths:
|
| 300 |
+
try:
|
| 301 |
+
os.remove(visual_path)
|
| 302 |
+
except:
|
| 303 |
+
pass
|
| 304 |
+
pbar.update(1)
|
| 305 |
+
# reorder this group of results back to original unsorted form
|
| 306 |
+
res = re_ords.get_original(res)
|
| 307 |
+
|
| 308 |
+
pbar.close()
|
| 309 |
+
return res
|
| 310 |
+
|
| 311 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 312 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|
LaViDa/eval/lmms_eval/models/qwen_vl_api.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import tempfile
|
| 5 |
+
import time
|
| 6 |
+
from copy import deepcopy
|
| 7 |
+
from io import BytesIO
|
| 8 |
+
from typing import List, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import requests as url_requests
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from tqdm import tqdm
|
| 13 |
+
|
| 14 |
+
from lmms_eval import utils
|
| 15 |
+
from lmms_eval.api.instance import Instance
|
| 16 |
+
from lmms_eval.api.model import lmms
|
| 17 |
+
from lmms_eval.api.registry import register_model
|
| 18 |
+
|
| 19 |
+
NUM_SECONDS_TO_SLEEP = 5
|
| 20 |
+
from loguru import logger as eval_logger
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
import dashscope
|
| 24 |
+
except:
|
| 25 |
+
eval_logger.debug("Can not import Dashscope")
|
| 26 |
+
|
| 27 |
+
API_KEY = os.getenv("DASHSCOPE_API_KEY", "YOUR_API_KEY")
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@register_model("qwen-vl-api")
|
| 31 |
+
class Qwen_VL_API(lmms):
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
model_version: str = "qwen-vl-max",
|
| 35 |
+
image_token: str = "<image>", # Use to separate interleaved image and text
|
| 36 |
+
system_prompt: str = "", # Whether you want some special system prompt here
|
| 37 |
+
tmp_folder: str = "./tmp", # Due to qwen's api restriction,
|
| 38 |
+
continual_mode: bool = False,
|
| 39 |
+
response_persistent_folder: str = None,
|
| 40 |
+
**kwargs,
|
| 41 |
+
) -> None:
|
| 42 |
+
super().__init__()
|
| 43 |
+
self.continual_mode = continual_mode
|
| 44 |
+
|
| 45 |
+
self.model_version = model_version
|
| 46 |
+
self.image_token = image_token
|
| 47 |
+
self.system_prompt = system_prompt
|
| 48 |
+
self.tmp_folder = tmp_folder
|
| 49 |
+
if self.continual_mode:
|
| 50 |
+
if response_persistent_folder is None:
|
| 51 |
+
raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
|
| 52 |
+
|
| 53 |
+
os.makedirs(response_persistent_folder, exist_ok=True)
|
| 54 |
+
self.response_persistent_folder = response_persistent_folder
|
| 55 |
+
self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
|
| 56 |
+
|
| 57 |
+
if os.path.exists(self.response_persistent_file):
|
| 58 |
+
with open(self.response_persistent_file, "r") as f:
|
| 59 |
+
self.response_cache = json.load(f)
|
| 60 |
+
self.cache_mode = "resume"
|
| 61 |
+
else:
|
| 62 |
+
self.response_cache = {}
|
| 63 |
+
self.cache_mode = "start"
|
| 64 |
+
|
| 65 |
+
@property
|
| 66 |
+
def rank(self):
|
| 67 |
+
return self._rank
|
| 68 |
+
|
| 69 |
+
@property
|
| 70 |
+
def world_size(self):
|
| 71 |
+
return self._world_size
|
| 72 |
+
|
| 73 |
+
def save_image_to_temp_file(self, image):
|
| 74 |
+
temp_file = tempfile.NamedTemporaryFile(suffix=".jpeg", delete=True)
|
| 75 |
+
image.save(temp_file.name)
|
| 76 |
+
return temp_file
|
| 77 |
+
|
| 78 |
+
def generate_until(self, requests) -> List[str]:
|
| 79 |
+
res = []
|
| 80 |
+
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
|
| 81 |
+
|
| 82 |
+
for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
|
| 83 |
+
if self.continual_mode is True and self.cache_mode == "resume":
|
| 84 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 85 |
+
if doc_uuid in self.response_cache:
|
| 86 |
+
response_text = self.response_cache[doc_uuid]
|
| 87 |
+
if response_text:
|
| 88 |
+
res.append(response_text)
|
| 89 |
+
pbar.update(1)
|
| 90 |
+
continue
|
| 91 |
+
# encode, pad, and truncate contexts for this batch
|
| 92 |
+
visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
|
| 93 |
+
visuals = self.flatten(visuals)
|
| 94 |
+
imgs = []
|
| 95 |
+
|
| 96 |
+
for idx, visual in enumerate(visuals):
|
| 97 |
+
temp_file = self.save_image_to_temp_file(visual)
|
| 98 |
+
imgs.append(temp_file.name)
|
| 99 |
+
|
| 100 |
+
messages = [{"role": "user", "content": []}]
|
| 101 |
+
|
| 102 |
+
if self.image_token not in contexts:
|
| 103 |
+
for img in imgs:
|
| 104 |
+
messages[0]["content"].append({"image": img})
|
| 105 |
+
messages[0]["content"].append({"text": contexts})
|
| 106 |
+
else:
|
| 107 |
+
contexts = contexts.split(self.image_token)
|
| 108 |
+
|
| 109 |
+
for idx, img in enumerate(imgs):
|
| 110 |
+
messages[0]["content"].append({"text": contexts[idx]})
|
| 111 |
+
messages[0]["content"].append({"image": img})
|
| 112 |
+
messages[0]["content"].append({"text": contexts[-1]})
|
| 113 |
+
|
| 114 |
+
if "max_new_tokens" not in gen_kwargs or gen_kwargs["max_new_tokens"] > 1500:
|
| 115 |
+
gen_kwargs["max_new_tokens"] = 1024
|
| 116 |
+
if "temperature" not in gen_kwargs:
|
| 117 |
+
gen_kwargs["temperature"] = 0
|
| 118 |
+
if "top_p" not in gen_kwargs:
|
| 119 |
+
gen_kwargs["top_p"] = None
|
| 120 |
+
if "num_beams" not in gen_kwargs:
|
| 121 |
+
gen_kwargs["num_beams"] = 1
|
| 122 |
+
|
| 123 |
+
for attempt in range(5):
|
| 124 |
+
try:
|
| 125 |
+
response_data = dashscope.MultiModalConversation.call(model=self.model_version, messages=messages, api_key=API_KEY, max_length=gen_kwargs["max_new_tokens"], temperature=gen_kwargs["temperature"])
|
| 126 |
+
break
|
| 127 |
+
except Exception as e:
|
| 128 |
+
eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
|
| 129 |
+
if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt
|
| 130 |
+
time.sleep(NUM_SECONDS_TO_SLEEP)
|
| 131 |
+
else: # If this was the last attempt, log and return empty
|
| 132 |
+
eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
|
| 133 |
+
res.append("")
|
| 134 |
+
pbar.update(1)
|
| 135 |
+
continue
|
| 136 |
+
try:
|
| 137 |
+
res.append(response_data["output"]["choices"][0]["message"]["content"][0]["text"].strip())
|
| 138 |
+
except Exception as e:
|
| 139 |
+
eval_logger.error(f"Error {e} happens when parsing input.")
|
| 140 |
+
eval_logger.error(f"{response_data}")
|
| 141 |
+
res.append("")
|
| 142 |
+
|
| 143 |
+
if self.continual_mode is True: # Cache the response
|
| 144 |
+
doc_uuid = f"{task}___{split}___{doc_id}"
|
| 145 |
+
self.response_cache[doc_uuid] = res[-1]
|
| 146 |
+
with open(self.response_persistent_file, "w") as f:
|
| 147 |
+
json.dump(self.response_cache, f)
|
| 148 |
+
pbar.update(1)
|
| 149 |
+
|
| 150 |
+
pbar.close()
|
| 151 |
+
|
| 152 |
+
return res
|
| 153 |
+
|
| 154 |
+
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
|
| 155 |
+
assert False, "Not supported for claude"
|
| 156 |
+
|
| 157 |
+
def flatten(self, input):
|
| 158 |
+
new_list = []
|
| 159 |
+
for i in input:
|
| 160 |
+
for j in i:
|
| 161 |
+
new_list.append(j)
|
| 162 |
+
return new_list
|
| 163 |
+
|
| 164 |
+
def generate_until_multi_round(self, requests) -> List[str]:
|
| 165 |
+
raise NotImplementedError("TODO: Implement multi-round generation")
|