GuoruiSong commited on
Commit
1f27be1
·
verified ·
1 Parent(s): 0a0ce86

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. LaViDa/eval/lmms_eval/models/__init__.py +97 -0
  2. LaViDa/eval/lmms_eval/models/aria.py +334 -0
  3. LaViDa/eval/lmms_eval/models/auroracap.py +541 -0
  4. LaViDa/eval/lmms_eval/models/batch_gpt4.py +207 -0
  5. LaViDa/eval/lmms_eval/models/cambrian.py +313 -0
  6. LaViDa/eval/lmms_eval/models/claude.py +272 -0
  7. LaViDa/eval/lmms_eval/models/cogvlm2.py +229 -0
  8. LaViDa/eval/lmms_eval/models/from_log.py +119 -0
  9. LaViDa/eval/lmms_eval/models/fuyu.py +269 -0
  10. LaViDa/eval/lmms_eval/models/gemini_api.py +306 -0
  11. LaViDa/eval/lmms_eval/models/gpt4v.py +244 -0
  12. LaViDa/eval/lmms_eval/models/idefics2.py +249 -0
  13. LaViDa/eval/lmms_eval/models/instructblip.py +230 -0
  14. LaViDa/eval/lmms_eval/models/internvideo2.py +366 -0
  15. LaViDa/eval/lmms_eval/models/internvideo2_5.py +295 -0
  16. LaViDa/eval/lmms_eval/models/internvl.py +496 -0
  17. LaViDa/eval/lmms_eval/models/internvl2.py +334 -0
  18. LaViDa/eval/lmms_eval/models/llama_vid.py +282 -0
  19. LaViDa/eval/lmms_eval/models/llama_vision.py +218 -0
  20. LaViDa/eval/lmms_eval/models/llava.py +442 -0
  21. LaViDa/eval/lmms_eval/models/llava_dream.py +855 -0
  22. LaViDa/eval/lmms_eval/models/llava_hf.py +390 -0
  23. LaViDa/eval/lmms_eval/models/llava_llada.py +894 -0
  24. LaViDa/eval/lmms_eval/models/llava_onevision.py +785 -0
  25. LaViDa/eval/lmms_eval/models/llava_onevision_moviechat.py +526 -0
  26. LaViDa/eval/lmms_eval/models/llava_sglang.py +165 -0
  27. LaViDa/eval/lmms_eval/models/llava_vid.py +533 -0
  28. LaViDa/eval/lmms_eval/models/longva.py +476 -0
  29. LaViDa/eval/lmms_eval/models/mantis.py +312 -0
  30. LaViDa/eval/lmms_eval/models/minimonkey.py +334 -0
  31. LaViDa/eval/lmms_eval/models/model_utils/__init__.py +0 -0
  32. LaViDa/eval/lmms_eval/models/model_utils/audio_processing.py +7 -0
  33. LaViDa/eval/lmms_eval/models/model_utils/load_video.py +151 -0
  34. LaViDa/eval/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py +369 -0
  35. LaViDa/eval/lmms_eval/models/mplug_owl_video.py +199 -0
  36. LaViDa/eval/lmms_eval/models/mplug_owl_video/__init__.py +84 -0
  37. LaViDa/eval/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py +286 -0
  38. LaViDa/eval/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py +1850 -0
  39. LaViDa/eval/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py +263 -0
  40. LaViDa/eval/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py +59 -0
  41. LaViDa/eval/lmms_eval/models/ola.py +709 -0
  42. LaViDa/eval/lmms_eval/models/openai_compatible.py +225 -0
  43. LaViDa/eval/lmms_eval/models/oryx.py +475 -0
  44. LaViDa/eval/lmms_eval/models/phi3v.py +220 -0
  45. LaViDa/eval/lmms_eval/models/qwen2_5_vl.py +323 -0
  46. LaViDa/eval/lmms_eval/models/qwen2_5_vl_interleave.py +408 -0
  47. LaViDa/eval/lmms_eval/models/qwen2_audio.py +284 -0
  48. LaViDa/eval/lmms_eval/models/qwen2_vl.py +296 -0
  49. LaViDa/eval/lmms_eval/models/qwen_vl.py +312 -0
  50. LaViDa/eval/lmms_eval/models/qwen_vl_api.py +165 -0
LaViDa/eval/lmms_eval/models/__init__.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib
2
+ import os
3
+ import sys
4
+
5
+ import hf_transfer
6
+ from loguru import logger
7
+
8
+ os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
9
+
10
+ logger.remove()
11
+ logger.add(sys.stdout, level="WARNING")
12
+
13
+ AVAILABLE_MODELS = {
14
+ "aria": "Aria",
15
+ "auroracap": "AuroraCap",
16
+ "batch_gpt4": "BatchGPT4",
17
+ "claude": "Claude",
18
+ "cogvlm2": "CogVLM2",
19
+ "from_log": "FromLog",
20
+ "fuyu": "Fuyu",
21
+ "gemini_api": "GeminiAPI",
22
+ "gpt4v": "GPT4V",
23
+ "idefics2": "Idefics2",
24
+ "instructblip": "InstructBLIP",
25
+ "internvideo2": "InternVideo2",
26
+ "internvl": "InternVLChat",
27
+ "internvl2": "InternVL2",
28
+ "llama_vid": "LLaMAVid",
29
+ "llama_vision": "LlamaVision",
30
+ "llava": "Llava",
31
+ "llava_hf": "LlavaHf",
32
+ "llava_onevision": "Llava_OneVision",
33
+ "llava_onevision_moviechat": "Llava_OneVision_MovieChat",
34
+ "llava_sglang": "LlavaSglang",
35
+ "llava_vid": "LlavaVid",
36
+ "longva": "LongVA",
37
+ "mantis": "Mantis",
38
+ "minicpm_v": "MiniCPM_V",
39
+ "minimonkey": "MiniMonkey",
40
+ "moviechat": "MovieChat",
41
+ "mplug_owl_video": "mplug_Owl",
42
+ "ola": "Ola",
43
+ "openai_compatible": "OpenAICompatible",
44
+ "oryx": "Oryx",
45
+ "phi3v": "Phi3v",
46
+ "qwen2_5_vl": "Qwen2_5_VL",
47
+ "qwen2_5_vl_interleave": "Qwen2_5_VL_Interleave",
48
+ "qwen2_audio": "Qwen2_Audio",
49
+ "qwen2_vl": "Qwen2_VL",
50
+ "qwen_vl": "Qwen_VL",
51
+ "qwen_vl_api": "Qwen_VL_API",
52
+ "reka": "Reka",
53
+ "ross": "Ross",
54
+ "slime": "Slime",
55
+ "srt_api": "SRT_API",
56
+ "tinyllava": "TinyLlava",
57
+ "videoChatGPT": "VideoChatGPT",
58
+ "videochat2": "VideoChat2",
59
+ "video_llava": "VideoLLaVA",
60
+ "vila": "VILA",
61
+ "vita": "VITA",
62
+ "vllm": "VLLM",
63
+ "xcomposer2_4KHD": "XComposer2_4KHD",
64
+ "xcomposer2d5": "XComposer2D5",
65
+ "egogpt": "EgoGPT",
66
+ "internvideo2_5": "InternVideo2_5",
67
+ "videochat_flash": "VideoChat_Flash",
68
+ "whisper": "Whisper",
69
+ "whisper_vllm": "WhisperVllm",
70
+ "llava_llada":"Llava_Llada",
71
+ "llava_dream":"Llava_Dream"
72
+ }
73
+
74
+
75
+ def get_model(model_name):
76
+ if model_name not in AVAILABLE_MODELS:
77
+ raise ValueError(f"Model {model_name} not found in available models.")
78
+
79
+ model_class = AVAILABLE_MODELS[model_name]
80
+ if "." not in model_class:
81
+ model_class = f"lmms_eval.models.{model_name}.{model_class}"
82
+
83
+ try:
84
+ model_module, model_class = model_class.rsplit(".", 1)
85
+ module = __import__(model_module, fromlist=[model_class])
86
+ return getattr(module, model_class)
87
+ except Exception as e:
88
+ logger.error(f"Failed to import {model_class} from {model_name}: {e}")
89
+ raise
90
+
91
+
92
+ if os.environ.get("LMMS_EVAL_PLUGINS", None):
93
+ # Allow specifying other packages to import models from
94
+ for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","):
95
+ m = importlib.import_module(f"{plugin}.models")
96
+ for model_name, model_class in getattr(m, "AVAILABLE_MODELS").items():
97
+ AVAILABLE_MODELS[model_name] = f"{plugin}.models.{model_name}.{model_class}"
LaViDa/eval/lmms_eval/models/aria.py ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import numpy as np
5
+ import PIL
6
+ import requests
7
+ import torch
8
+ from accelerate import Accelerator, DistributedType
9
+ from accelerate.state import AcceleratorState
10
+ from decord import VideoReader, cpu
11
+ from PIL import Image
12
+ from tqdm import tqdm
13
+ from transformers import AutoModelForCausalLM, AutoProcessor
14
+
15
+ from lmms_eval import utils
16
+ from lmms_eval.api.instance import Instance
17
+ from lmms_eval.api.model import lmms
18
+ from lmms_eval.api.registry import register_model
19
+
20
+ warnings.filterwarnings("ignore")
21
+
22
+ from loguru import logger as eval_logger
23
+
24
+ DEFAULT_IMAGE_TOKEN = "<image>"
25
+
26
+
27
+ @register_model("aria")
28
+ class Aria(lmms):
29
+ """
30
+ Llava Model for Hugging Face Transformers: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava
31
+
32
+ Adapted from the LLaVA-HF model in lmms_eval/models/llava_hf.py
33
+
34
+ Example usage:
35
+
36
+ accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
37
+ --model aria \
38
+ --model_args pretrained=rhymes-ai/Aria \
39
+ --tasks seedbench \
40
+ --batch_size 1 \
41
+ --output_path ./logs/ \
42
+ --log_samples
43
+ """
44
+
45
+ def __init__(
46
+ self,
47
+ pretrained: str = "rhymes-ai/Aria",
48
+ revision: str = "main",
49
+ device: str = "cuda",
50
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
51
+ batch_size: int = 1,
52
+ attn_implementation: Optional[str] = None,
53
+ device_map: str = "",
54
+ chat_template: Optional[str] = None,
55
+ use_cache: bool = True,
56
+ specified_eot_token_id: Optional[int] = None,
57
+ max_frames_num: Optional[int] = 64,
58
+ **kwargs,
59
+ ) -> None:
60
+ super().__init__()
61
+ # Do not use kwargs for now
62
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
63
+
64
+ accelerator = Accelerator()
65
+ if accelerator.num_processes > 1 and device_map == "":
66
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
67
+ self.device_map = f"cuda:{accelerator.local_process_index}"
68
+ else:
69
+ self._device = torch.device(device)
70
+ self.device_map = device_map
71
+ if isinstance(dtype, str) and dtype != "auto":
72
+ dtype = getattr(torch, dtype)
73
+
74
+ self.max_frames_num = max_frames_num
75
+ self._model = AutoModelForCausalLM.from_pretrained(pretrained, revision=revision, device_map=self.device_map, torch_dtype=torch.bfloat16, trust_remote_code=True, attn_implementation=attn_implementation)
76
+
77
+ self.pretrained = pretrained
78
+ self._image_processor = AutoProcessor.from_pretrained(pretrained, revision=revision, trust_remote_code=True)
79
+ self._tokenizer = self._image_processor.tokenizer
80
+
81
+ self._config = self._model.config
82
+ self.batch_size_per_gpu = int(batch_size)
83
+ self.chat_template = chat_template
84
+ self.specified_eot_token_id = specified_eot_token_id
85
+ if accelerator.num_processes > 1 and device_map == "":
86
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
87
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
88
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
89
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
90
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
91
+ kwargs = {
92
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
93
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
94
+ }
95
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
96
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
97
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
98
+ self._model = accelerator.prepare(self.model)
99
+ else:
100
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
101
+ self.accelerator = accelerator
102
+ if self.accelerator.is_local_main_process:
103
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
104
+ self._rank = self.accelerator.local_process_index
105
+ self._world_size = self.accelerator.num_processes
106
+ elif accelerator.num_processes == 1 and device_map == "auto":
107
+ eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
108
+ self._rank = 0
109
+ self._world_size = 1
110
+ else:
111
+ eval_logger.info(f"Using single device: {self._device}")
112
+ self.model.to(self._device)
113
+ self._rank = 0
114
+ self._world_size = 1
115
+ self.accelerator = accelerator
116
+
117
+ @property
118
+ def config(self):
119
+ # return the associated transformers.AutoConfig for the given pretrained model.
120
+ return self._config
121
+
122
+ @property
123
+ def tokenizer(self):
124
+ return self._tokenizer
125
+
126
+ @property
127
+ def model(self):
128
+ # returns the model, unwrapping it if using Accelerate
129
+ if hasattr(self, "accelerator"):
130
+ return self.accelerator.unwrap_model(self._model)
131
+ else:
132
+ return self._model
133
+
134
+ @property
135
+ def eot_token_id(self):
136
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
137
+ return self.tokenizer.eos_token_id
138
+
139
+ @property
140
+ def max_length(self):
141
+ return self._max_length
142
+
143
+ @property
144
+ def batch_size(self):
145
+ return self.batch_size_per_gpu
146
+
147
+ @property
148
+ def device(self):
149
+ return self._device
150
+
151
+ @property
152
+ def rank(self):
153
+ return self._rank
154
+
155
+ @property
156
+ def world_size(self):
157
+ return self._world_size
158
+
159
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
160
+ """ """
161
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
162
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
163
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
164
+ if left_truncate_len:
165
+ encoding = encoding[-left_truncate_len:]
166
+ return encoding
167
+
168
+ def tok_decode(self, tokens):
169
+ return self.tokenizer.decode(tokens)
170
+
171
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
172
+ raise NotImplementedError("Not implemented for Aria.")
173
+
174
+ def flatten(self, input):
175
+ new_list = []
176
+ for i in input:
177
+ for j in i:
178
+ new_list.append(j)
179
+ return new_list
180
+
181
+ def load_video(self, video_path, max_frames_num):
182
+ if type(video_path) == str:
183
+ vr = VideoReader(video_path, ctx=cpu(0))
184
+ else:
185
+ vr = VideoReader(video_path[0], ctx=cpu(0))
186
+ total_frame_num = len(vr)
187
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
188
+ frame_idx = uniform_sampled_frames.tolist()
189
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
190
+ spare_frames = [Image.fromarray(x) for x in spare_frames]
191
+ return spare_frames # (frames, height, width, channels)
192
+
193
+ def generate_until(self, requests: List[Instance]) -> List[str]:
194
+ res = []
195
+
196
+ def _collate(x):
197
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
198
+ # - time estimates will always be over not underestimates, which is more useful for planning
199
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
200
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
201
+ # automatic adaptive batches much much easier to implement
202
+ # - any OOMs will happen right away rather than near the end
203
+ toks = self.tok_encode(x[0])
204
+ return -len(toks), x[0]
205
+
206
+ # we group requests by their generation_kwargs,
207
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
208
+ # in the same batch.
209
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
210
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
211
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
212
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
213
+ for chunk in chunks:
214
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
215
+ task = task[0]
216
+ split = split[0]
217
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
218
+ visuals = self.flatten(visuals)
219
+ if len(visuals) == 0:
220
+ task_type = "text"
221
+ elif isinstance(visuals[0], PIL.Image.Image):
222
+ task_type = "image"
223
+ elif isinstance(visuals[0], str):
224
+ task_type = "video"
225
+ # we assume all gen kwargs in the batch are the same
226
+ # this is safe to assume because the `grouper` object ensures it.
227
+ gen_kwargs = all_gen_kwargs[0]
228
+
229
+ assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
230
+ text_context = contexts[0]
231
+ text_context = text_context.replace("\n\n", "\n")
232
+
233
+ context = []
234
+
235
+ if task_type == "video":
236
+ try:
237
+ visuals = self.load_video(visuals, self.max_frames_num)
238
+ except Exception as e:
239
+ res.append("")
240
+ eval_logger.info(f"Error {e} when loading video : {visuals}")
241
+ pbar.update(1)
242
+
243
+ if DEFAULT_IMAGE_TOKEN not in context:
244
+ context += [{"text": None, "type": "image"}] * len(visuals)
245
+ context += [{"text": "\n" + text_context, "type": "text"}]
246
+ else:
247
+ assert text_context.count(DEFAULT_IMAGE_TOKEN) == len(visuals)
248
+ for i, text_chunk in enumerate(text_context.split(DEFAULT_IMAGE_TOKEN)):
249
+ context += [{"text": text_chunk, "type": "text"}]
250
+ if i < len(visuals):
251
+ context += [{"text": None, "type": "image"}] * len(visuals)
252
+ context += [{"text": "\n", "type": "text"}]
253
+
254
+ # Apply chat template
255
+ messages = [{"role": "user", "content": context}]
256
+
257
+ text = self._image_processor.apply_chat_template(messages, add_generation_prompt=True)
258
+
259
+ # removing redundant placeholders
260
+ text = re.sub(r"<image \d+>", "", text)
261
+ text = re.sub(r"<image>", "", text)
262
+
263
+ eval_logger.debug("DEBUGGING FOR ARIA:" + text)
264
+
265
+ if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
266
+ eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
267
+
268
+ if task_type == "video":
269
+ inputs = self._image_processor(images=visuals, text=text, return_tensors="pt", max_image_size=490)
270
+ else:
271
+ inputs = self._image_processor(images=visuals, text=text, return_tensors="pt", max_image_size=980)
272
+
273
+ inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype)
274
+ inputs = {k: v.to(self._device) for k, v in inputs.items()}
275
+
276
+ if "temperature" not in gen_kwargs:
277
+ gen_kwargs["temperature"] = 0
278
+ if "top_p" not in gen_kwargs:
279
+ gen_kwargs["top_p"] = None
280
+ if "num_beams" not in gen_kwargs:
281
+ gen_kwargs["num_beams"] = 1
282
+
283
+ gen_kwargs["do_sample"] = False
284
+ gen_kwargs["max_new_tokens"] = 1024
285
+
286
+ if "until" in gen_kwargs:
287
+ gen_kwargs.pop("until")
288
+
289
+ eval_logger.debug(f"generate kwargs: {gen_kwargs}")
290
+
291
+ try:
292
+ with torch.inference_mode(), torch.cuda.amp.autocast(dtype=torch.bfloat16):
293
+ output = self.model.generate(
294
+ **inputs,
295
+ stop_strings=["<|im_end|>"],
296
+ tokenizer=self._image_processor.tokenizer,
297
+ **gen_kwargs,
298
+ )
299
+ output_ids = output[0][inputs["input_ids"].shape[1] :]
300
+ text_outputs = self._image_processor.decode(output_ids, skip_special_tokens=True).replace("<|im_end|>", "")
301
+
302
+ ### Basic Model-wise Parsing for CoT-alike Outputs
303
+ """
304
+ keywords = [
305
+ "Answer:",
306
+ "answer is:", "choice is:", "option is:",
307
+ "Answer is:", "Choice is:", "Option is:",
308
+ "answer is", "choice is", "option is",
309
+ "Answer is", "Choice is", "Option is"
310
+ ]
311
+
312
+ for keyword in keywords:
313
+ if keyword in text_outputs:
314
+ eval_logger.debug(f"ARIA Original generated output simplified by keyword [{keyword}]: {text_outputs}")
315
+ text_outputs = text_outputs.split(keyword, 1)[-1]
316
+ break
317
+ """
318
+ eval_logger.debug(f"Generated output: {text_outputs}")
319
+ except Exception as ex:
320
+ eval_logger.debug(f"Generation failed: {ex}")
321
+ if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
322
+ eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n")
323
+
324
+ res.append(text_outputs)
325
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
326
+ pbar.update(1)
327
+ # reorder this group of results back to original unsorted form
328
+ res = re_ords.get_original(res)
329
+
330
+ pbar.close()
331
+ return res
332
+
333
+ def generate_until_multi_round(self, requests) -> List[str]:
334
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
LaViDa/eval/lmms_eval/models/auroracap.py ADDED
@@ -0,0 +1,541 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import json
3
+ import logging
4
+ import os
5
+ import os.path as osp
6
+ from typing import List, Optional, Tuple, Union
7
+
8
+ import av
9
+ import numpy as np
10
+ import torch
11
+ from accelerate import Accelerator, DistributedType
12
+ from accelerate.state import AcceleratorState
13
+ from huggingface_hub import snapshot_download
14
+ from peft import PeftModel
15
+ from PIL import Image
16
+ from tqdm import tqdm
17
+ from transformers import (
18
+ AutoModel,
19
+ AutoModelForCausalLM,
20
+ AutoTokenizer,
21
+ BitsAndBytesConfig,
22
+ CLIPImageProcessor,
23
+ )
24
+
25
+ from lmms_eval import utils
26
+ from lmms_eval.api.instance import Instance
27
+ from lmms_eval.api.model import lmms
28
+ from lmms_eval.api.registry import register_model
29
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
30
+ from lmms_eval.utils import stop_sequences_criteria
31
+
32
+ try:
33
+ from lmms_eval.models.aurora_xtuner.model.aurora import (
34
+ AuroraEncoder,
35
+ AuroraModel,
36
+ AuroraSigEncoder,
37
+ )
38
+ from lmms_eval.models.aurora_xtuner.utils import PROMPT_TEMPLATE
39
+ except ImportError:
40
+ eval_logger.error("AuroraCap is not installed. Please install AuroraCap to use this model by `git clone https://github.com/rese1f/aurora.git` and link `src/xtuner/xtuner` to `lmms_eval/models/aurora_xtuner`")
41
+ import warnings
42
+
43
+ warnings.filterwarnings("ignore")
44
+
45
+ eval_logger = logging.getLogger("lmms-eval")
46
+
47
+ try:
48
+ from llava.constants import (
49
+ DEFAULT_IM_END_TOKEN,
50
+ DEFAULT_IM_START_TOKEN,
51
+ DEFAULT_IMAGE_TOKEN,
52
+ IGNORE_INDEX,
53
+ IMAGE_TOKEN_INDEX,
54
+ )
55
+ from llava.conversation import SeparatorStyle, conv_templates
56
+ from llava.mm_utils import get_model_name_from_path, tokenizer_image_token
57
+ except ImportError:
58
+ eval_logger.error("LLaVA is not installed. Please install LLaVA to use this model.")
59
+
60
+
61
+ @register_model("auroracap")
62
+ class AuroraCap(lmms):
63
+ """
64
+ auroracap Model
65
+ """
66
+
67
+ def __init__(
68
+ self,
69
+ pretrained_llm: str = "meta-llama/Meta-Llama-3-8B-Instruct",
70
+ pretrained_vit: str = "google/siglip-so400m-patch14-384",
71
+ pretrained: str = "model/PATH",
72
+ resolution: int = 378,
73
+ token_merge_ratio: float = 0.4,
74
+ device: Optional[str] = "cuda",
75
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
76
+ batch_size: Optional[Union[int, str]] = 1,
77
+ conv_template="vicuna_v1", # vicuna_v1",
78
+ video_decode_backend: str = "pyav",
79
+ max_frames_num: int = 16,
80
+ slowfast: bool = False,
81
+ **kwargs,
82
+ ) -> None:
83
+ super().__init__()
84
+ # Do not use kwargs for now
85
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
86
+
87
+ accelerator = Accelerator()
88
+ if accelerator.num_processes > 1:
89
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
90
+ else:
91
+ self._device = device
92
+
93
+ pretrained_pth = snapshot_download(repo_id=pretrained) if not osp.isdir(pretrained) else pretrained
94
+ pretrained_llm = pretrained_pth
95
+ pretrained_vit = osp.join(pretrained_pth, "visual_encoder")
96
+
97
+ self._model = AuroraModel(
98
+ slowfast=slowfast,
99
+ llm=AutoModelForCausalLM.from_pretrained(
100
+ pretrained_model_name_or_path=pretrained_llm,
101
+ trust_remote_code=True,
102
+ torch_dtype=torch.float16,
103
+ ),
104
+ visual_encoder=AuroraEncoder.from_pretrained(
105
+ pretrained_model_name_or_path=pretrained_vit,
106
+ torch_dtype=torch.float16,
107
+ ),
108
+ )
109
+
110
+ projector_path = osp.join(pretrained_pth, "projector")
111
+ self.model.projector = AutoModel.from_pretrained(projector_path, torch_dtype=torch.float16, trust_remote_code=True)
112
+
113
+ self._image_processor = CLIPImageProcessor.from_pretrained(
114
+ pretrained_model_name_or_path="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", # use standard CLIP processor
115
+ trust_remote_code=True,
116
+ size=resolution,
117
+ crop_size=resolution,
118
+ )
119
+ self._tokenizer = AutoTokenizer.from_pretrained(
120
+ pretrained_model_name_or_path=pretrained_llm,
121
+ trust_remote_code=True,
122
+ padding_side="right",
123
+ )
124
+ # compute token merge ratio settings
125
+ self.patch_size = self._model.visual_encoder.config.patch_size
126
+ self.num_layers = self._model.visual_encoder.config.num_hidden_layers
127
+ self.token_merge_ratio = token_merge_ratio
128
+
129
+ self._config = self._model.config
130
+ self.model.eval()
131
+ self.model.tie_weights()
132
+ self.batch_size_per_gpu = int(batch_size)
133
+ self.conv_template = conv_template
134
+ if accelerator.num_processes > 1:
135
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
136
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
137
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
138
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
139
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
140
+ kwargs = {
141
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
142
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
143
+ }
144
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
145
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
146
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
147
+ self._model = accelerator.prepare(self.model)
148
+ self._model.visual_encoder = accelerator.prepare(self.model.visual_encoder)
149
+ self._model.projector = accelerator.prepare(self.model.projector)
150
+ else: # DistributedType.MULTI_GPU
151
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
152
+ self._model.visual_encoder = accelerator.prepare_model(self.model.visual_encoder, evaluation_mode=True)
153
+ self._model.projector = accelerator.prepare_model(self.model.projector, evaluation_mode=True)
154
+ self.accelerator = accelerator
155
+ if self.accelerator.is_local_main_process:
156
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
157
+ self._rank = self.accelerator.local_process_index
158
+ self._world_size = self.accelerator.num_processes
159
+ else:
160
+ self.model.to(self._device)
161
+ self._rank = 0
162
+ self._world_size = 1
163
+
164
+ # For Video Caption
165
+ self.video_decode_backend = video_decode_backend
166
+ self.max_frames_num = int(max_frames_num)
167
+
168
+ @property
169
+ def config(self):
170
+ # return the associated transformers.AutoConfig for the given pretrained model.
171
+ return self._config
172
+
173
+ @property
174
+ def tokenizer(self):
175
+ return self._tokenizer
176
+
177
+ @property
178
+ def model(self):
179
+ # returns the model, unwrapping it if using Accelerate
180
+ if hasattr(self, "accelerator"):
181
+ return self.accelerator.unwrap_model(self._model)
182
+ else:
183
+ return self._model
184
+
185
+ @property
186
+ def eot_token_id(self):
187
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
188
+ return self.tokenizer.eos_token_id
189
+
190
+ @property
191
+ def max_length(self):
192
+ return self._max_length
193
+
194
+ def pad_sequence(self, input_ids, batch_first, padding_value):
195
+ if self.tokenizer.padding_side == "left":
196
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
197
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
198
+ if self.tokenizer.padding_side == "left":
199
+ input_ids = torch.flip(input_ids, [1])
200
+ return input_ids
201
+
202
+ @property
203
+ def batch_size(self):
204
+ return self.batch_size_per_gpu
205
+
206
+ @property
207
+ def device(self):
208
+ return self._device
209
+
210
+ @property
211
+ def rank(self):
212
+ return self._rank
213
+
214
+ @property
215
+ def world_size(self):
216
+ return self._world_size
217
+
218
+ def process_images(self, images, image_processor, model_cfg):
219
+ image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None)
220
+ new_images = []
221
+ if image_aspect_ratio == "pad":
222
+ for image in images:
223
+ image = expand2square(image, tuple(int(x * 255) for x in image_processor.image_mean))
224
+ image = image_processor.preprocess(image, return_tensors="pt")["pixel_values"][0]
225
+ new_images.append(image)
226
+ elif image_aspect_ratio == "anyres":
227
+ for image in images:
228
+ image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints)
229
+ new_images.append(image)
230
+ else:
231
+ return image_processor(images, return_tensors="pt")["pixel_values"]
232
+ if all(x.shape == new_images[0].shape for x in new_images):
233
+ new_images = torch.stack(new_images, dim=0)
234
+ return new_images
235
+
236
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
237
+ """ """
238
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
239
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
240
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
241
+ if left_truncate_len:
242
+ encoding = encoding[-left_truncate_len:]
243
+ return encoding
244
+
245
+ def tok_decode(self, tokens):
246
+ return self.tokenizer.decode(tokens)
247
+
248
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
249
+ res = []
250
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
251
+
252
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
253
+ # encode, pad, and truncate contexts for this batch
254
+ if type(doc_to_target) == str:
255
+ continuation = doc_to_target
256
+ else:
257
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
258
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
259
+ visuals = self.flatten(visuals)
260
+ if visuals:
261
+ image = self.process_images(visuals, self._image_processor, self._config)
262
+ if type(image) is list:
263
+ image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
264
+ else:
265
+ image = image.to(dtype=torch.float16, device=self.device)
266
+ else:
267
+ image = None
268
+
269
+ prompts_input = contexts[0]
270
+
271
+ if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
272
+ """
273
+ Three senarios:
274
+ 1. No image, and there for, no image token should be added.
275
+ 2. image token is already specified in the context, so we don't need to add it.
276
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
277
+ """
278
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
279
+ image_tokens = " ".join(image_tokens)
280
+ prompts_input = image_tokens + "\n" + contexts[0]
281
+ conv = conv_templates[self.conv_template].copy()
282
+ conv.append_message(conv.roles[0], prompts_input)
283
+ conv.append_message(conv.roles[1], None)
284
+ prompt = conv.get_prompt()
285
+ pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
286
+ contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
287
+ # Add the answer of the second role
288
+ conv.messages[1][1] = continuation
289
+
290
+ prompt = conv.get_prompt()
291
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
292
+ labels = input_ids.clone()
293
+ # Context part no need to calculate for loss
294
+ labels[0, : contxt_id.shape[1]] = -100
295
+ with torch.inference_mode():
296
+ data = dict()
297
+ data["pixel_values"] = image_tensor
298
+ data["input_ids"] = input_ids
299
+ data["attention_mask"] = attention_masks
300
+ self.model.visual_encoder.reset_tome_r(self.token_merge_ratio)
301
+ output = self.model(data, mode="tensor")
302
+
303
+ loss = outputs["loss"]
304
+ # loss = torch.exp(loss)
305
+ logits = outputs["logits"]
306
+ greedy_tokens = logits.argmax(dim=-1)
307
+ cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
308
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
309
+ max_equal = (greedy_tokens == cont_toks).all()
310
+ res.append((float(loss.item()), bool(max_equal)))
311
+ pbar.update(1)
312
+ pbar.close()
313
+ return res
314
+
315
+ def flatten(self, input):
316
+ new_list = []
317
+ for i in input:
318
+ for j in i:
319
+ new_list.append(j)
320
+ return new_list
321
+
322
+ def load_video(self, video_path, max_frames_num):
323
+ vr = VideoReader(video_path, ctx=cpu(0))
324
+ total_frame_num = len(vr)
325
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
326
+ frame_idx = uniform_sampled_frames.tolist()
327
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
328
+ return spare_frames # (frames, height, width, channels)
329
+
330
+ def extract_keyframes(self, video_path, keyframes):
331
+ container = av.open(video_path)
332
+ video_stream = container.streams.video[0]
333
+ fps = video_stream.average_rate
334
+ time_base = video_stream.time_base
335
+ frames = []
336
+
337
+ for keyframe in keyframes:
338
+ keyframe_time = float(keyframe)
339
+ frame_number = int(keyframe_time * fps)
340
+ container.seek(int(keyframe_time / time_base))
341
+ found = False
342
+ for packet in container.demux(video=0):
343
+ for frame in packet.decode():
344
+ if frame.index >= frame_number:
345
+ frames.append(frame)
346
+ found = True
347
+ break
348
+ if found:
349
+ break
350
+
351
+ if not found:
352
+ container.seek(-1, any_frame=False)
353
+ for packet in container.demux(video=0):
354
+ for frame in packet.decode():
355
+ pass
356
+ frames.append(frame)
357
+
358
+ return np.stack([x.to_ndarray(format="rgb24") for x in frames])
359
+
360
+ def generate_until(self, requests: List[Instance]) -> List[str]:
361
+ res = []
362
+
363
+ def _collate(x):
364
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
365
+ # - time estimates will always be over not underestimates, which is more useful for planning
366
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
367
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
368
+ # automatic adaptive batches much much easier to implement
369
+ # - any OOMs will happen right away rather than near the end
370
+ toks = self.tok_encode(x[0])
371
+ return -len(toks), x[0]
372
+
373
+ # we group requests by their generation_kwargs,
374
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
375
+ # in the same batch.
376
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
377
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
378
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
379
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
380
+
381
+ for chunk in chunks:
382
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
383
+ task = task[0]
384
+ split = split[0]
385
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # the length of visuals is 1, equal to batchsize
386
+ visuals = self.flatten(visuals)
387
+ # we assume all gen kwargs in the batch are the same
388
+ # this is safe to assume because the `grouper` object ensures it.
389
+ gen_kwargs = all_gen_kwargs[0]
390
+
391
+ # Set default values for until and max_new_tokens
392
+ until = [self.tok_decode(self.eot_token_id)]
393
+
394
+ # Update values from gen_kwargs if present
395
+ if "until" in gen_kwargs:
396
+ until = gen_kwargs.pop("until")
397
+ if isinstance(until, str):
398
+ until = [until]
399
+ elif not isinstance(until, list):
400
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
401
+
402
+ if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
403
+ # here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
404
+ self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
405
+ eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
406
+ # encode, pad, and truncate contexts for this batch
407
+ if visuals:
408
+ if isinstance(visuals[0], dict):
409
+ video_path = visuals[0]["video_path"]
410
+ keyframe = visuals[0]["keyframe"]
411
+ video = self.extract_keyframes(video_path, keyframe)
412
+ image_tensor = self.process_images(video, self._image_processor, self._config).cuda()
413
+ elif isinstance(visuals, list):
414
+ print(visuals[0])
415
+ if isinstance(visuals[0], Image.Image):
416
+ image_tensor = self.process_images(visuals, self._image_processor, self._config)
417
+ else:
418
+ if visuals[0].endswith("mp4"):
419
+ if self.video_decode_backend == "decord":
420
+ video = self.load_video(visuals[0], self.max_frames_num)
421
+ elif self.video_decode_backend == "pyav":
422
+ video = read_video_pyav(visuals[0], num_frm=self.max_frames_num)
423
+ image_tensor = self.process_images(video, self._image_processor, self._config).cuda()
424
+ elif visuals[0].endswith("mkv"):
425
+ assert self.video_decode_backend == "pyav", "we only tested this case, decord may not work"
426
+ video = read_video_pyav(visuals[0], num_frm=self.max_frames_num)
427
+ image_tensor = self.process_images(video, self._image_processor, self._config).cuda()
428
+
429
+ if type(image_tensor) is list:
430
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
431
+ else:
432
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
433
+
434
+ else:
435
+ image_tensor = None
436
+
437
+ question_input = []
438
+
439
+ for visual, context in zip(visuals, contexts):
440
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
441
+ """
442
+ Three senarios:
443
+ 1. No image, and there for, no image token should be added.
444
+ 2. image token is already specified in the context, so we don't need to add it.
445
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
446
+ """
447
+ if isinstance(visuals[0], dict):
448
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(video)
449
+ elif isinstance(visuals, list):
450
+ if isinstance(visuals[0], Image.Image):
451
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
452
+ else:
453
+ if visual.endswith("mp4") or visual.endswith("mkv"):
454
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(video)
455
+
456
+ image_tokens = " ".join(image_tokens)
457
+ question = image_tokens + "\n" + context
458
+
459
+ else:
460
+ question = context
461
+ conv = conv_templates[self.conv_template].copy()
462
+ conv.append_message(conv.roles[0], question)
463
+ conv.append_message(conv.roles[1], None)
464
+ prompt_question = conv.get_prompt()
465
+ question_input.append(prompt_question)
466
+
467
+ # The above for loop has bugs. When there is no visuals, e.g. pure text,
468
+ # there will be no for loop execute resulting in an empty question_input (because no visuals)
469
+ # Scenario 1 won't even be execute
470
+ if len(visuals) == 0:
471
+ for context in contexts:
472
+ question = context
473
+ conv = conv_templates[self.conv_template].copy()
474
+ conv.append_message(conv.roles[0], question)
475
+ conv.append_message(conv.roles[1], None)
476
+ prompt_question = conv.get_prompt()
477
+ question_input.append(prompt_question)
478
+
479
+ # preconfigure gen_kwargs with defaults
480
+ if isinstance(visuals[0], dict):
481
+ gen_kwargs["image_sizes"] = [video[idx].size for idx in range(len(video))]
482
+ elif isinstance(visuals, list):
483
+ if isinstance(visuals[0], Image.Image):
484
+ gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
485
+ else:
486
+ if visuals[0].endswith("mp4"):
487
+ gen_kwargs["image_sizes"] = [video[idx].size for idx in range(len(video))]
488
+
489
+ if "max_new_tokens" not in gen_kwargs:
490
+ gen_kwargs["max_new_tokens"] = 1024
491
+ if "temperature" not in gen_kwargs:
492
+ gen_kwargs["temperature"] = 0
493
+ if "top_p" not in gen_kwargs:
494
+ gen_kwargs["top_p"] = None
495
+ if "num_beams" not in gen_kwargs:
496
+ gen_kwargs["num_beams"] = 1
497
+
498
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
499
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
500
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
501
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
502
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
503
+ try:
504
+ data = dict()
505
+ if isinstance(visuals[0], dict):
506
+ data["pixel_values"] = image_tensor.unsqueeze(0)
507
+ elif isinstance(visuals, list):
508
+ if isinstance(visuals[0], Image.Image):
509
+ data["pixel_values"] = image_tensor
510
+ else:
511
+ if visuals[0].endswith("mp4") or visuals[0].endswith("mkv"):
512
+ data["pixel_values"] = image_tensor.unsqueeze(0)
513
+
514
+ data["input_ids"] = input_ids
515
+ data["attention_mask"] = attention_masks
516
+ self.model.visual_encoder.reset_tome_r(self.token_merge_ratio)
517
+ output = self.model(data, mode="inference")
518
+ cont = self.model.llm.generate(
519
+ **output,
520
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
521
+ temperature=gen_kwargs["temperature"],
522
+ top_p=gen_kwargs["top_p"],
523
+ num_beams=gen_kwargs["num_beams"],
524
+ max_new_tokens=gen_kwargs["max_new_tokens"],
525
+ )
526
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
527
+ except Exception as e:
528
+ eval_logger.error(f"Error {e} in generating")
529
+ cont = ""
530
+ text_outputs = [""]
531
+
532
+ print(text_outputs)
533
+
534
+ res.extend(text_outputs)
535
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
536
+ pbar.update(1)
537
+ # reorder this group of results back to original unsorted form
538
+ res = re_ords.get_original(res)
539
+
540
+ pbar.close()
541
+ return res
LaViDa/eval/lmms_eval/models/batch_gpt4.py ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Standard library imports
2
+ import base64
3
+ import json
4
+ import os
5
+ import time
6
+ from copy import deepcopy
7
+ from io import BytesIO
8
+
9
+ import numpy as np
10
+ import requests as url_requests
11
+
12
+ # Related third-party imports
13
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
14
+ from accelerate.state import AcceleratorState
15
+ from loguru import logger as eval_logger
16
+ from openai import OpenAI
17
+ from PIL import Image
18
+ from tqdm import tqdm
19
+
20
+ # Local application/library specific imports
21
+ from lmms_eval.api.instance import Instance
22
+ from lmms_eval.api.model import lmms
23
+ from lmms_eval.api.registry import register_model
24
+
25
+ # Conditional imports
26
+ try:
27
+ from decord import VideoReader, cpu
28
+ except ImportError:
29
+ eval_logger.warning("Decord is not installed. Video input will not be supported.")
30
+
31
+ # Constants and global configurations
32
+ API_TYPE = os.getenv("API_TYPE", "openai")
33
+ NUM_SECONDS_TO_SLEEP = 5
34
+
35
+ if API_TYPE == "openai":
36
+ API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
37
+ API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
38
+ headers = {
39
+ "Authorization": f"Bearer {API_KEY}",
40
+ "Content-Type": "application/json",
41
+ }
42
+ elif API_TYPE == "azure":
43
+ API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
44
+ API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
45
+ headers = {
46
+ "api-key": API_KEY,
47
+ "Content-Type": "application/json",
48
+ }
49
+ else:
50
+ API_URL = "YOUR_API_URL"
51
+ API_KEY = "YOUR_API_KEY"
52
+
53
+
54
+ @register_model("batch_gpt4")
55
+ class BatchGPT4(lmms):
56
+ def __init__(
57
+ self,
58
+ model_version: str = "gpt-4o",
59
+ api_key: str = API_KEY,
60
+ api_url: str = API_URL,
61
+ modality: str = "image",
62
+ max_frames_num: int = 10,
63
+ timeout: int = 120,
64
+ **kwargs,
65
+ ) -> None:
66
+ super().__init__()
67
+ # Manually set a image token for GPT4V so that we can search for it
68
+ # and split the text and image
69
+ # Here we just use the same token as llava for convenient
70
+ self.model_version = model_version
71
+ self.modality = modality
72
+ self.max_frames_num = max_frames_num
73
+ self.image_token = "<image>"
74
+ self.timeout = timeout
75
+
76
+ self.api_key = api_key
77
+ self.api_url = api_url
78
+ self.client = OpenAI(api_key=api_key)
79
+
80
+ accelerator = Accelerator()
81
+ assert accelerator.state.local_process_index == 0, "BatchGPT4 does not support distributed inference."
82
+ assert accelerator.state.num_processes == 1, "BatchGPT4 does not support distributed inference."
83
+
84
+ # Function to encode the image
85
+ def encode_image(self, image: Image):
86
+ output_buffer = BytesIO()
87
+ image.save(output_buffer, format="PNG")
88
+ byte_data = output_buffer.getvalue()
89
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
90
+ return base64_str
91
+
92
+ # Function to encode the video
93
+ def encode_video(self, video_path, for_get_frames_num):
94
+ vr = VideoReader(video_path, ctx=cpu(0))
95
+ total_frame_num = len(vr)
96
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int)
97
+ frame_idx = uniform_sampled_frames.tolist()
98
+ frames = vr.get_batch(frame_idx).asnumpy()
99
+
100
+ base64_frames = []
101
+ for frame in frames:
102
+ img = Image.fromarray(frame)
103
+ output_buffer = BytesIO()
104
+ img.save(output_buffer, format="PNG")
105
+ byte_data = output_buffer.getvalue()
106
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
107
+ base64_frames.append(base64_str)
108
+
109
+ return base64_frames
110
+
111
+ def flatten(self, input):
112
+ new_list = []
113
+ for i in input:
114
+ for j in i:
115
+ new_list.append(j)
116
+ return new_list
117
+
118
+ def generate_until(self, requests):
119
+ # Prepare the batch requests data
120
+ requests_data = {}
121
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Batch Preparing")
122
+ for idx, (contexts, gen_kwargs, doc_to_visual, doc_id, task, split) in enumerate([reg.args for reg in requests]):
123
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
124
+ visuals = self.flatten(visuals)
125
+ imgs = []
126
+ for visual in visuals:
127
+ if self.modality == "image":
128
+ img = self.encode_image(visual)
129
+ imgs.append(img)
130
+ elif self.modality == "video":
131
+ frames = self.encode_video(visual, self.max_frames_num)
132
+ imgs.extend(frames)
133
+
134
+ messages = []
135
+ if self.image_token not in contexts:
136
+ messages.append({"role": "user", "content": contexts})
137
+ for img in imgs:
138
+ messages.append({"role": "user", "content": f"data:image/jpeg;base64,{img}"})
139
+ else:
140
+ contexts_split = contexts.split(self.image_token)
141
+ for idx, context in enumerate(contexts_split):
142
+ if idx < len(imgs):
143
+ messages.append({"role": "user", "content": context})
144
+ messages.append({"role": "user", "content": f"data:image/jpeg;base64,{imgs[idx]}"})
145
+ if len(contexts_split) > len(imgs):
146
+ messages.append({"role": "user", "content": contexts_split[-1]})
147
+
148
+ requests_data[f"request-{idx}"] = {"model": self.model_version, "messages": messages, "max_tokens": gen_kwargs.get("max_new_tokens", 1024)}
149
+ pbar.update(1)
150
+
151
+ file_path = os.getenv("HF_HOME", "~/.cache/huggingface") + f"/batchinput_{len(requests_data)}.jsonl"
152
+ file_path = self.create_batch_input_file(requests_data, file_path)
153
+ file_id = self.upload_input_file(file_path)
154
+
155
+ batch_response = self.create_batch(file_id, metadata={"description": "Batch Processing for GPT-4"})
156
+ batch_status = self.check_batch_status(batch_response.id)
157
+ while True:
158
+ batch_status = self.check_batch_status(batch_response.id)
159
+ if batch_status.status == "completed":
160
+ eval_logger.info("Batch processing completed.")
161
+ batch_results = self.retrieve_batch_results(batch_status.output_file_id)
162
+ res = [result["response"]["choices"][0]["message"]["content"] for result in json.loads(batch_results)]
163
+ return res
164
+ elif batch_status.status == "failed":
165
+ eval_logger.info("Batch processing failed.")
166
+ res = ["Batch failed"] * len(requests)
167
+ return res
168
+ else:
169
+ eval_logger.info(f"Batch status: {batch_status.status}. Retrying in {NUM_SECONDS_TO_SLEEP} seconds.")
170
+ time.sleep(NUM_SECONDS_TO_SLEEP)
171
+
172
+ def loglikelihood(self, requests):
173
+ # TODO
174
+ assert False, "GPT4V not support"
175
+
176
+ def create_batch_input_file(self, requests_data, file_path="batchinput.jsonl"):
177
+ with open(file_path, "w") as file:
178
+ for request_id, data in requests_data.items():
179
+ json_record = json.dumps({"custom_id": request_id, "method": "POST", "url": "/v1/chat/completions", "body": data})
180
+ file.write(json_record + "\n")
181
+ return file_path
182
+
183
+ def upload_input_file(self, file_path):
184
+ with open(file_path, "rb") as file:
185
+ response = self.client.files.create(file=file, purpose="batch")
186
+ return response.id
187
+
188
+ def create_batch(self, file_id, metadata=None):
189
+ if metadata is None:
190
+ metadata = {}
191
+ response = self.client.batches.create(input_file_id=file_id, endpoint="/v1/chat/completions", completion_window="24h", metadata=metadata)
192
+ return response
193
+
194
+ def check_batch_status(self, batch_id):
195
+ return self.client.batches.retrieve(batch_id)
196
+
197
+ def retrieve_batch_results(self, file_id):
198
+ return self.client.files.content(file_id)
199
+
200
+ def cancel_batch(self, batch_id):
201
+ return self.client.batches.cancel(batch_id)
202
+
203
+ def list_batches(self, limit=10):
204
+ return self.client.batches.list(limit=limit)
205
+
206
+ def generate_until_multi_round(self, requests) -> List[str]:
207
+ raise NotImplementedError("TODO: Implement multi-round generation for BatchGPT4")
LaViDa/eval/lmms_eval/models/cambrian.py ADDED
@@ -0,0 +1,313 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # code is migrated from https://github.com/kangreen0210/LIME-M/blob/main/lmms_eval/models/cambrian.py
2
+ import os
3
+ import uuid
4
+ import warnings
5
+ from typing import List, Optional, Tuple, Union
6
+
7
+ import torch
8
+ from accelerate import Accelerator, DistributedType
9
+ from PIL import Image
10
+ from tqdm import tqdm
11
+ from transformers import PreTrainedTokenizer
12
+
13
+ from lmms_eval import utils
14
+ from lmms_eval.api.instance import Instance
15
+ from lmms_eval.api.model import lmms
16
+ from lmms_eval.api.registry import register_model
17
+
18
+ warnings.simplefilter("ignore", category=DeprecationWarning)
19
+ warnings.filterwarnings("ignore")
20
+
21
+ from loguru import logger as eval_logger
22
+
23
+ try:
24
+ from cambrian.conversation import conv_templates
25
+ from cambrian.mm_utils import (
26
+ get_model_name_from_path,
27
+ process_images,
28
+ tokenizer_image_token,
29
+ )
30
+ from cambrian.model.builder import load_pretrained_model
31
+ except ImportError:
32
+ eval_logger.error("Cambrian is not installed. Please install it by running `pip install cambrian`.")
33
+
34
+ # Model Constants
35
+ IMAGE_TOKEN_INDEX = -200
36
+ DEFAULT_IMAGE_TOKEN = "<image>"
37
+ DEFAULT_IM_START_TOKEN = "<im_start>"
38
+ DEFAULT_IM_END_TOKEN = "<im_end>"
39
+
40
+
41
+ def process(image, question, tokenizer, image_processor, model_config, conv_mode):
42
+ qs = question
43
+
44
+ if model_config.mm_use_im_start_end:
45
+ qs = f"{DEFAULT_IM_START_TOKEN}{DEFAULT_IMAGE_TOKEN}{DEFAULT_IM_END_TOKEN}\n{qs}"
46
+ else:
47
+ qs = f"{DEFAULT_IMAGE_TOKEN}\n{qs}"
48
+
49
+ conv = conv_templates[conv_mode].copy()
50
+ conv.append_message(conv.roles[0], qs)
51
+ conv.append_message(conv.roles[1], None)
52
+ prompt = conv.get_prompt()
53
+
54
+ image_size = [image.size]
55
+ image_tensor = process_images([image], image_processor, model_config)
56
+
57
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
58
+
59
+ return input_ids, image_tensor, image_size, prompt
60
+
61
+
62
+ def make_context(
63
+ tokenizer: PreTrainedTokenizer,
64
+ query: str,
65
+ history: List[Tuple[str, str]] = None,
66
+ system: str = "",
67
+ max_window_size: int = 6144,
68
+ chat_format: str = "chatml",
69
+ ):
70
+ if history is None:
71
+ history = []
72
+
73
+ if chat_format == "chatml":
74
+ im_start, im_end = "<|im_start|>", "<|im_end|>"
75
+ im_start_tokens = [tokenizer.im_start_id]
76
+ im_end_tokens = [tokenizer.im_end_id]
77
+ nl_tokens = tokenizer.encode("\n")
78
+
79
+ def _tokenize_str(role, content):
80
+ return f"{role}\n{content}", tokenizer.encode(role, allowed_special=set(tokenizer.IMAGE_ST)) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST))
81
+
82
+ system_text, system_tokens_part = _tokenize_str("system", system)
83
+ system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
84
+
85
+ raw_text = ""
86
+ context_tokens = []
87
+
88
+ for turn_query, turn_response in reversed(history):
89
+ query_text, query_tokens_part = _tokenize_str("user", turn_query)
90
+ query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
91
+ if turn_response is not None:
92
+ response_text, response_tokens_part = _tokenize_str("assistant", turn_response)
93
+ response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
94
+
95
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
96
+ prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
97
+ else:
98
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens
99
+ prev_chat = f"\n{im_start}{query_text}{im_end}\n"
100
+
101
+ current_context_size = len(system_tokens) + len(next_context_tokens) + len(context_tokens)
102
+ if current_context_size < max_window_size:
103
+ context_tokens = next_context_tokens + context_tokens
104
+ raw_text = prev_chat + raw_text
105
+ else:
106
+ break
107
+
108
+ context_tokens = system_tokens + context_tokens
109
+ raw_text = f"{im_start}{system_text}{im_end}" + raw_text
110
+ context_tokens += nl_tokens + im_start_tokens + _tokenize_str("user", query)[1] + im_end_tokens + nl_tokens + im_start_tokens + tokenizer.encode("assistant") + nl_tokens
111
+ raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
112
+
113
+ elif chat_format == "raw":
114
+ raw_text = query
115
+ context_tokens = tokenizer.encode(raw_text)
116
+ else:
117
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
118
+
119
+ return raw_text, context_tokens
120
+
121
+
122
+ @register_model("cambrian")
123
+ class Cambrian(lmms):
124
+ def __init__(
125
+ self,
126
+ pretrained: str = "nyu-visionx/cambrian-8b",
127
+ device: Optional[str] = "cuda",
128
+ device_map="auto",
129
+ batch_size: Optional[Union[int, str]] = 1,
130
+ trust_remote_code: Optional[bool] = True,
131
+ use_cache=True,
132
+ **kwargs,
133
+ ) -> None:
134
+ super().__init__()
135
+ assert not kwargs, f"Unexpected kwargs: {kwargs}"
136
+
137
+ accelerator = Accelerator()
138
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}") if accelerator.num_processes > 1 else device
139
+
140
+ self.model_name = get_model_name_from_path(pretrained)
141
+ tokenizer, model, self.image_processor, context_len = load_pretrained_model(pretrained, None, self.model_name, device_map=self._device)
142
+
143
+ self.conv_mode = {"cambrian-8b": "llama_3", "cambrian-13b": "vicuna_v1", "cambrian-34b": "chatml_direct"}.get(self.model_name)
144
+
145
+ if not self.conv_mode:
146
+ raise ValueError(f"Unsupported model: {self.model_name}")
147
+
148
+ self._model = model
149
+ self._tokenizer = tokenizer
150
+ self._model.eval()
151
+ self.batch_size_per_gpu = int(batch_size)
152
+ self.use_cache = use_cache
153
+ self._rank = 0
154
+ self._world_size = 1
155
+
156
+ if accelerator.num_processes > 1:
157
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU], "Unsupported distributed type. Only DDP and FSDP are supported."
158
+ self._model = accelerator.prepare(self.model) if accelerator.distributed_type == DistributedType.FSDP else accelerator.prepare_model(self.model, evaluation_mode=True)
159
+ self.accelerator = accelerator
160
+ if self.accelerator.is_local_main_process:
161
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
162
+ self._rank = self.accelerator.local_process_index
163
+ self._world_size = self.accelerator.num_processes
164
+ else:
165
+ self.model.to(self._device)
166
+
167
+ self.accelerator = accelerator
168
+
169
+ @property
170
+ def model(self):
171
+ return self.accelerator.unwrap_model(self._model) if hasattr(self, "accelerator") else self._model
172
+
173
+ @property
174
+ def eot_token_id(self):
175
+ return self.tokenizer.eos_token_id
176
+
177
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
178
+ res = []
179
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
180
+
181
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
182
+ continuation = doc_to_target if isinstance(doc_to_target, str) else doc_to_target(self.task_dict[task][split][doc_id])
183
+ visuals = self.flatten([doc_to_visual(self.task_dict[task][split][doc_id])])
184
+
185
+ query = []
186
+ visual_paths = []
187
+ for visual in visuals:
188
+ name = uuid.uuid4().hex.upper()[0:6]
189
+ visual_path = f"/tmp/{name}.png"
190
+ visual.save(visual_path)
191
+ visual_paths.append(visual_path)
192
+ query.append({"image": visual_path})
193
+
194
+ context_query = query.copy()
195
+ context_query.append({"text": contexts})
196
+ query.append({"text": contexts + continuation})
197
+
198
+ context_query = self.tokenizer.from_list_format(context_query)
199
+ query = self.tokenizer.from_list_format(query)
200
+
201
+ _, context_tokens = make_context(
202
+ self.tokenizer, context_query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format
203
+ )
204
+ context_tokens = torch.tensor([context_tokens])
205
+
206
+ _, continuation_tokens = make_context(self.tokenizer, query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format)
207
+ continuation_tokens = torch.tensor([continuation_tokens]).to(self.model.device)
208
+ attn_mask = torch.ones_like(continuation_tokens).to(self.model.device)
209
+ labels = continuation_tokens.clone().to(self.model.device)
210
+ labels[:, : context_tokens.shape[1]] = -100
211
+
212
+ with torch.inference_mode():
213
+ outputs = self.model(input_ids=continuation_tokens, labels=labels, attention_mask=attn_mask)
214
+
215
+ loss = outputs.loss
216
+ logits = outputs["logits"]
217
+ greedy_tokens = logits.argmax(dim=-1)
218
+ cont_toks = continuation_tokens[:, context_tokens.shape[1] :]
219
+ greedy_tokens = greedy_tokens[:, context_tokens.shape[1] : continuation_tokens.shape[1]]
220
+ max_equal = (greedy_tokens == cont_toks).all()
221
+ res.append((float(loss.item()), bool(max_equal)))
222
+ pbar.update(1)
223
+
224
+ pbar.close()
225
+ return res
226
+
227
+ @staticmethod
228
+ def flatten(input_list):
229
+ return [item for sublist in input_list for item in sublist]
230
+
231
+ def generate_until(self, requests: List[Instance]) -> List[str]:
232
+ res = []
233
+
234
+ def _collate(x):
235
+ toks = self.tokenizer.encode(x[0])
236
+ return -len(toks), x[0]
237
+
238
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
239
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
240
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
241
+
242
+ for chunk in chunks:
243
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
244
+ task = task[0]
245
+ split = split[0]
246
+ visuals = self.flatten([doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id])
247
+
248
+ visual_paths = []
249
+ for visual in visuals:
250
+ name = uuid.uuid4().hex.upper()[0:6]
251
+ visual_path = f"/xpfs/public/gezhang/zk/lmms-eval/lmms_eval/tmp/{name}.png"
252
+ visual.save(visual_path)
253
+ visual_paths.append(visual_path)
254
+
255
+ gen_kwargs = all_gen_kwargs[0]
256
+ until = [self.tokenizer.decode(self.eot_token_id)]
257
+
258
+ if "until" in gen_kwargs:
259
+ until = gen_kwargs.pop("until")
260
+ if isinstance(until, str):
261
+ until = [until]
262
+ elif not isinstance(until, list):
263
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
264
+
265
+ gen_kwargs.setdefault("image_sizes", [visuals[0].size] if visuals else None)
266
+ gen_kwargs.setdefault("max_new_tokens", 1024)
267
+ gen_kwargs.setdefault("temperature", 0)
268
+ gen_kwargs.setdefault("top_p", None)
269
+ gen_kwargs.setdefault("num_beams", 1)
270
+
271
+ until.append("<|eot_id|>")
272
+
273
+ image = Image.open(visual_paths[0]).convert("RGB")
274
+ question = contexts[0]
275
+
276
+ input_ids, image_tensor, image_sizes, prompt = process(image, question, self.tokenizer, self.image_processor, self.model.config, self.conv_mode)
277
+ input_ids = input_ids.to(device=self.model.device, non_blocking=True)
278
+
279
+ with torch.inference_mode():
280
+ output_ids = self.model.generate(
281
+ input_ids,
282
+ images=image_tensor,
283
+ image_sizes=image_sizes,
284
+ do_sample=gen_kwargs["temperature"] > 0,
285
+ temperature=gen_kwargs["temperature"],
286
+ num_beams=gen_kwargs["num_beams"],
287
+ max_new_tokens=gen_kwargs["max_new_tokens"],
288
+ use_cache=True,
289
+ )
290
+
291
+ text_outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
292
+
293
+ for term in until:
294
+ if term:
295
+ text_outputs = text_outputs.split(term)[0]
296
+
297
+ print(text_outputs)
298
+ res.append(text_outputs)
299
+
300
+ for visual_path in visual_paths:
301
+ try:
302
+ os.remove(visual_path)
303
+ except OSError:
304
+ pass
305
+
306
+ pbar.update(1)
307
+
308
+ res = re_ords.get_original(res)
309
+ pbar.close()
310
+ return res
311
+
312
+ def generate_until_multi_round(self, requests) -> List[str]:
313
+ raise NotImplementedError("TODO: Implement multi-round generation for Cambrian")
LaViDa/eval/lmms_eval/models/claude.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import json
3
+ import os
4
+ import time
5
+ from copy import deepcopy
6
+ from io import BytesIO
7
+ from typing import List, Tuple, Union
8
+
9
+ from accelerate import Accelerator, DistributedType
10
+ from PIL import Image
11
+ from tqdm import tqdm
12
+
13
+ from lmms_eval.api.instance import Instance
14
+ from lmms_eval.api.model import lmms
15
+ from lmms_eval.api.registry import register_model
16
+
17
+ NUM_SECONDS_TO_SLEEP = 5
18
+
19
+ from loguru import logger
20
+
21
+ eval_logger = logger
22
+
23
+ try:
24
+ import anthropic
25
+ import numpy as np
26
+ from decord import VideoReader, cpu
27
+ except Exception as e:
28
+ eval_logger.warning(f"Error importing claude: {e}")
29
+
30
+ API_URL = os.getenv("ANTHROPIC_API_URL", "https://api.anthropic.com/v1/complete")
31
+ API_KEY = os.getenv("ANTHROPIC_API_KEY", "YOUR_API_KEY")
32
+
33
+
34
+ @register_model("claude")
35
+ class Claude(lmms):
36
+ def __init__(
37
+ self,
38
+ model_version: str = "claude-3-opus-20240229",
39
+ image_token: str = "<image>", # Use to separate interleaved image and text
40
+ system_prompt: str = "", # Whether you want some special system prompt here
41
+ modality: str = "image",
42
+ max_frames_num: int = 10,
43
+ continual_mode: bool = False,
44
+ response_persistent_folder: str = None,
45
+ **kwargs,
46
+ ) -> None:
47
+ super().__init__()
48
+ self.model_version = model_version
49
+ self.image_token = image_token
50
+ self.system_prompt = system_prompt
51
+ self.modality = modality
52
+ self.max_frames_num = max_frames_num
53
+
54
+ self.continual_mode = continual_mode
55
+ if self.continual_mode:
56
+ if response_persistent_folder is None:
57
+ raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
58
+
59
+ os.makedirs(response_persistent_folder, exist_ok=True)
60
+ self.response_persistent_folder = response_persistent_folder
61
+ self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
62
+
63
+ if os.path.exists(self.response_persistent_file):
64
+ with open(self.response_persistent_file, "r") as f:
65
+ self.response_cache = json.load(f)
66
+ self.cache_mode = "resume"
67
+ else:
68
+ self.response_cache = {}
69
+ self.cache_mode = "start"
70
+
71
+ accelerator = Accelerator()
72
+ if accelerator.num_processes > 1:
73
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
74
+ self.accelerator = accelerator
75
+ if self.accelerator.is_local_main_process:
76
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
77
+ self._rank = self.accelerator.local_process_index
78
+ self._world_size = self.accelerator.num_processes
79
+ else:
80
+ self.accelerator = accelerator
81
+ self._rank = self.accelerator.local_process_index
82
+ self._world_size = self.accelerator.num_processes
83
+
84
+ self.device = self.accelerator.device
85
+
86
+ def encode_image(self, image):
87
+ output_buffer = BytesIO()
88
+ image.save(output_buffer, format="JPEG")
89
+ byte_data = output_buffer.getvalue()
90
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
91
+ return base64_str
92
+
93
+ def flatten(self, input):
94
+ new_list = []
95
+ for i in input:
96
+ for j in i:
97
+ new_list.append(j)
98
+ return new_list
99
+
100
+ def get_image_size(self, image):
101
+ # Create a BytesIO object to store the image bytes
102
+ img_byte_array = BytesIO()
103
+
104
+ # Save the image to the BytesIO object
105
+ image.save(img_byte_array, format="PNG")
106
+
107
+ # Get the size of the BytesIO object
108
+ img_size = img_byte_array.tell()
109
+
110
+ return img_size
111
+
112
+ # The max file size is 5MB for claude
113
+ def shrink_image_to_file_size(self, img: Image, max_file_size=4838990) -> Image:
114
+ # Get the current size of the image
115
+ original_size = self.get_image_size(img)
116
+
117
+ # If the image size is already smaller than the desired size, return
118
+ if original_size <= max_file_size:
119
+ return img
120
+
121
+ # Calculate the ratio to shrink the image
122
+ # Somehow I found out sqrt ratio is not enough to shrink the image
123
+ # below threshold, so I guess we do more
124
+ shrink_ratio = min(0.9, max_file_size / original_size)
125
+
126
+ # Resize the image with the calculated ratio
127
+ new_width = int(img.width * shrink_ratio)
128
+ new_height = int(img.height * shrink_ratio)
129
+ img = img.resize((new_width, new_height), Image.LANCZOS)
130
+
131
+ return self.shrink_image_to_file_size(img, max_file_size)
132
+
133
+ def encode_video(self, video_path):
134
+ vr = VideoReader(video_path, ctx=cpu(0))
135
+ total_frame_num = len(vr)
136
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, self.max_frames_num, dtype=int)
137
+ frame_idx = uniform_sampled_frames.tolist()
138
+ frames = vr.get_batch(frame_idx).asnumpy()
139
+
140
+ base64_frames = []
141
+ for frame in frames:
142
+ img = Image.fromarray(frame)
143
+ output_buffer = BytesIO()
144
+ img.save(output_buffer, format="JPEG")
145
+ byte_data = output_buffer.getvalue()
146
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
147
+ base64_frames.append(f"{base64_str}")
148
+
149
+ return base64_frames
150
+
151
+ def generate_until(self, requests) -> List[str]:
152
+ client = anthropic.Anthropic()
153
+
154
+ res = []
155
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
156
+
157
+ empty_image_block = {
158
+ "type": "image",
159
+ "source": {
160
+ "type": "base64",
161
+ "media_type": "image/jpeg",
162
+ },
163
+ }
164
+ empty_text_block = {"type": "text"}
165
+ empty_messages = [
166
+ {
167
+ "role": "user",
168
+ "content": [],
169
+ }
170
+ ]
171
+
172
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
173
+ ###################### CONTINUAL MODE ######################
174
+ if self.continual_mode is True and self.cache_mode == "resume":
175
+ doc_uuid = f"{task}___{split}___{doc_id}"
176
+ if doc_uuid in self.response_cache:
177
+ response_text = self.response_cache[doc_uuid]
178
+ if response_text:
179
+ res.append(response_text)
180
+ pbar.update(1)
181
+ continue
182
+
183
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
184
+ visuals = self.flatten(visuals)
185
+ imgs = []
186
+ for visual in visuals:
187
+ if isinstance(visual, str) and os.path.exists(visual): # Assuming visual is a path to a video
188
+ visual = self.encode_video(visual)
189
+ for img in visual:
190
+ imgs.append(img)
191
+ else:
192
+ visual = self.shrink_image_to_file_size(visual)
193
+ img = self.encode_image(visual)
194
+ imgs.append(img)
195
+
196
+ messages = deepcopy(empty_messages)
197
+
198
+ if self.image_token not in contexts:
199
+ for img in imgs:
200
+ image_block = deepcopy(empty_image_block)
201
+ image_block["source"]["data"] = img
202
+ messages[0]["content"].append(image_block)
203
+ text_block = deepcopy(empty_text_block)
204
+ text_block["text"] = contexts
205
+ messages[0]["content"].append(text_block)
206
+ else:
207
+ contexts = contexts.split(self.image_token)
208
+ for idx, img in enumerate(imgs):
209
+ text_block = deepcopy(empty_text_block)
210
+ image_block = deepcopy(empty_image_block)
211
+ text_block["text"] = contexts
212
+ messages[0]["content"].append(text_block)
213
+ image_block["source"]["data"] = img
214
+ messages[0]["content"].append(image_block)
215
+
216
+ # If n image tokens are in the contexts
217
+ # contexts will be splitted into n+1 chunks
218
+ # Manually add it into the messages
219
+ text_block = deepcopy(empty_text_block)
220
+ text_block["text"] = contexts
221
+ messages["content"].append(text_block)
222
+
223
+ if "max_new_tokens" not in gen_kwargs:
224
+ gen_kwargs["max_new_tokens"] = 1024
225
+ if gen_kwargs["max_new_tokens"] > 4096:
226
+ gen_kwargs["max_new_tokens"] = 4096
227
+ if "temperature" not in gen_kwargs:
228
+ gen_kwargs["temperature"] = 0
229
+ if "top_p" not in gen_kwargs or gen_kwargs["top_p"] is None:
230
+ gen_kwargs["top_p"] = 1
231
+ if "num_beams" not in gen_kwargs:
232
+ gen_kwargs["num_beams"] = 1
233
+
234
+ for attempt in range(5):
235
+ retry_flag = True
236
+ try:
237
+ message = client.messages.create(model=self.model_version, max_tokens=gen_kwargs["max_new_tokens"], system=self.system_prompt, temperature=gen_kwargs["temperature"], top_p=gen_kwargs["top_p"], messages=messages)
238
+ retry_flag = False
239
+ except Exception as e:
240
+ eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
241
+ if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt
242
+ time.sleep(NUM_SECONDS_TO_SLEEP)
243
+ else: # If this was the last attempt, log and return empty
244
+ eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
245
+ res.append("")
246
+ pbar.update(1)
247
+ continue
248
+ if not retry_flag:
249
+ break
250
+ eval_logger.info("Retrying...")
251
+
252
+ response_text = message.content[0].text
253
+ res.append(message.content[0].text)
254
+ pbar.update(1)
255
+
256
+ ###################### CONTINUAL MODE ######################
257
+ if self.continual_mode is True: # Cache the response
258
+ response_text = message.content[0].text
259
+ doc_uuid = f"{task}___{split}___{doc_id}"
260
+ self.response_cache[doc_uuid] = response_text
261
+ with open(self.response_persistent_file, "w") as f:
262
+ json.dump(self.response_cache, f, indent=4)
263
+
264
+ pbar.close()
265
+
266
+ return res
267
+
268
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
269
+ assert False, "Not supported for claude"
270
+
271
+ def generate_until_multi_round(self, requests) -> List[str]:
272
+ raise NotImplementedError("TODO: Implement multi-round generation for Claude")
LaViDa/eval/lmms_eval/models/cogvlm2.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ from accelerate import Accelerator, DistributedType
6
+ from accelerate.state import AcceleratorState
7
+ from tqdm import tqdm
8
+ from transformers import AutoModelForCausalLM, AutoTokenizer
9
+
10
+ from lmms_eval import utils
11
+ from lmms_eval.api.instance import Instance
12
+ from lmms_eval.api.model import lmms
13
+ from lmms_eval.api.registry import register_model
14
+
15
+ warnings.filterwarnings("ignore")
16
+
17
+ from loguru import logger as eval_logger
18
+
19
+
20
+ @register_model("cogvlm2")
21
+ class CogVLM2(lmms):
22
+ """
23
+ CogVLM2 Model
24
+ """
25
+
26
+ def __init__(
27
+ self,
28
+ pretrained: str = "THUDM/cogvlm2-llama3-chinese-chat-19B",
29
+ device: Optional[str] = "cuda",
30
+ dtype: Optional[Union[str, torch.dtype]] = torch.bfloat16,
31
+ batch_size: Optional[Union[int, str]] = 1,
32
+ trust_remote_code: Optional[bool] = True,
33
+ **kwargs,
34
+ ) -> None:
35
+ super().__init__()
36
+ # Do not use kwargs for now
37
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
38
+
39
+ accelerator = Accelerator()
40
+ if accelerator.num_processes > 1:
41
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
42
+ else:
43
+ self._device = device
44
+ self.dtype = dtype
45
+ self._model = AutoModelForCausalLM.from_pretrained(pretrained, trust_remote_code=trust_remote_code, torch_dtype=dtype, device_map=self._device)
46
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code)
47
+ self._config = self._model.config
48
+ self.model.eval()
49
+ self.model.tie_weights()
50
+ self.batch_size_per_gpu = int(batch_size)
51
+ if accelerator.num_processes > 1:
52
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
53
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
54
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
55
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
56
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
57
+ kwargs = {
58
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
59
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
60
+ }
61
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
62
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
63
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
64
+ self._model = accelerator.prepare(self.model)
65
+ else:
66
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
67
+ self.accelerator = accelerator
68
+ if self.accelerator.is_local_main_process:
69
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
70
+ self._rank = self.accelerator.local_process_index
71
+ self._world_size = self.accelerator.num_processes
72
+ else:
73
+ self._rank = 0
74
+ self._world_size = 1
75
+
76
+ @property
77
+ def config(self):
78
+ # return the associated transformers.AutoConfig for the given pretrained model.
79
+ return self._config
80
+
81
+ @property
82
+ def tokenizer(self):
83
+ return self._tokenizer
84
+
85
+ @property
86
+ def model(self):
87
+ # returns the model, unwrapping it if using Accelerate
88
+ if hasattr(self, "accelerator"):
89
+ return self.accelerator.unwrap_model(self._model)
90
+ else:
91
+ return self._model
92
+
93
+ @property
94
+ def eot_token_id(self):
95
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
96
+ return self.tokenizer.eos_token_id
97
+
98
+ @property
99
+ def max_length(self):
100
+ return self._max_length
101
+
102
+ @property
103
+ def batch_size(self):
104
+ return self.batch_size_per_gpu
105
+
106
+ @property
107
+ def device(self):
108
+ return self._device
109
+
110
+ @property
111
+ def rank(self):
112
+ return self._rank
113
+
114
+ @property
115
+ def world_size(self):
116
+ return self._world_size
117
+
118
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
119
+ """ """
120
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
121
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
122
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
123
+ if left_truncate_len:
124
+ encoding = encoding[-left_truncate_len:]
125
+ return encoding
126
+
127
+ def tok_decode(self, tokens):
128
+ return self.tokenizer.decode(tokens)
129
+
130
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
131
+ # TODO
132
+ assert False, "We have not implemented this function for CogVLM2 yet"
133
+
134
+ def flatten(self, input):
135
+ new_list = []
136
+ for i in input:
137
+ for j in i:
138
+ new_list.append(j)
139
+ return new_list
140
+
141
+ def generate_until(self, requests: List[Instance]) -> List[str]:
142
+ res = []
143
+
144
+ def _collate(x):
145
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
146
+ # - time estimates will always be over not underestimates, which is more useful for planning
147
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
148
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
149
+ # automatic adaptive batches much much easier to implement
150
+ # - any OOMs will happen right away rather than near the end
151
+ toks = self.tok_encode(x[0])
152
+ return -len(toks), x[0]
153
+
154
+ # we group requests by their generation_kwargs,
155
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
156
+ # in the same batch.
157
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
158
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
159
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
160
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
161
+ for chunk in chunks:
162
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
163
+ task = task[0]
164
+ split = split[0]
165
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
166
+ visuals = self.flatten(visuals)
167
+ # we assume all gen kwargs in the batch are the same
168
+ # this is safe to assume because the `grouper` object ensures it.
169
+ gen_kwargs = all_gen_kwargs[0]
170
+
171
+ # Set default values for until and max_new_tokens
172
+ until = [self.tok_decode(self.eot_token_id)]
173
+
174
+ # Update values from gen_kwargs if present
175
+ if "until" in gen_kwargs:
176
+ until = gen_kwargs.pop("until")
177
+ if isinstance(until, str):
178
+ until = [until]
179
+ elif not isinstance(until, list):
180
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
181
+ assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
182
+ assert len(visuals) == 1, "CogVLM2 interface does not support bn_image > 1 for now"
183
+ context = contexts[0]
184
+ if "<image>" in context:
185
+ context = context.replace("<image>", "")
186
+
187
+ if "max_new_tokens" not in gen_kwargs:
188
+ gen_kwargs["max_new_tokens"] = 1024
189
+ if "temperature" not in gen_kwargs:
190
+ gen_kwargs["temperature"] = 0
191
+ if "top_p" not in gen_kwargs:
192
+ gen_kwargs["top_p"] = None
193
+ if "num_beams" not in gen_kwargs:
194
+ gen_kwargs["num_beams"] = 1
195
+
196
+ image = visuals[0]
197
+ input_by_model = self.model.build_conversation_input_ids(self.tokenizer, query=context, history=[], images=[image])
198
+
199
+ inputs = {
200
+ "input_ids": input_by_model["input_ids"].unsqueeze(0).to(self.device),
201
+ "token_type_ids": input_by_model["token_type_ids"].unsqueeze(0).to(self.device),
202
+ "attention_mask": input_by_model["attention_mask"].unsqueeze(0).to(self.device),
203
+ "images": [[input_by_model["images"][0].to(self.device).to(self.dtype)]],
204
+ }
205
+ if "cross_images" in input_by_model and input_by_model["cross_images"]:
206
+ inputs["cross_images"] = [[input_by_model["cross_images"][0].to(self.device).to(self.dtype)]]
207
+
208
+ try:
209
+ outputs = self.model.generate(**inputs, **gen_kwargs)
210
+ outputs = outputs[:, inputs["input_ids"].shape[1] :]
211
+ response = self.tokenizer.decode(outputs[0])
212
+ response = response.split("</s>")[0]
213
+ response = response.split("<|end_of_text|>")[0]
214
+
215
+ context = [{"role": "user", "content": context}, {"role": "assistant", "content": response}]
216
+ except Exception as e:
217
+ eval_logger.error(f"Error {e} in generating")
218
+ cont = ""
219
+ res.append(response)
220
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response)
221
+ pbar.update(1)
222
+ # reorder this group of results back to original unsorted form
223
+ res = re_ords.get_original(res)
224
+
225
+ pbar.close()
226
+ return res
227
+
228
+ def generate_until_multi_round(self, requests) -> List[str]:
229
+ raise NotImplementedError("TODO: Implement multi-round generation for CogVLM2")
LaViDa/eval/lmms_eval/models/from_log.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import re
4
+ from datetime import datetime
5
+ from typing import List, Tuple
6
+
7
+ from accelerate import Accelerator, DistributedType
8
+ from loguru import logger as eval_logger
9
+ from tqdm import tqdm
10
+
11
+ from lmms_eval.api.instance import Instance
12
+ from lmms_eval.api.model import lmms
13
+ from lmms_eval.api.registry import register_model
14
+
15
+
16
+ @register_model("from_log")
17
+ class FromLog(lmms):
18
+ def __init__(
19
+ self,
20
+ logs: str = "logs",
21
+ model_name: str = None,
22
+ model_args: str = None,
23
+ have_limits: bool = False,
24
+ **kwargs,
25
+ ) -> None:
26
+ super().__init__()
27
+
28
+ self.logs = {}
29
+
30
+ log_folders = logs.split(",")
31
+
32
+ def matched_model(_model_args):
33
+ if model_name and model_name != _model_args["model"]:
34
+ return False
35
+
36
+ if model_args:
37
+ _model_args_list = model_args.split(",")
38
+
39
+ for _model_arg in _model_args_list:
40
+ if _model_arg not in _model_args["model_args"]:
41
+ return False
42
+
43
+ if not have_limits and _model_args["limit"] is not None:
44
+ return False
45
+
46
+ return True
47
+
48
+ for log_folder in log_folders:
49
+ for root, dirs, files in os.walk(log_folder):
50
+ for file in files:
51
+ if file.endswith(".json"):
52
+ try:
53
+ log_file = os.path.join(root, file)
54
+
55
+ with open(log_file, "r") as f:
56
+ log_data = json.load(f)
57
+
58
+ # check if model is matched
59
+ _model_args = log_data["args"]
60
+ if not matched_model(_model_args):
61
+ raise Exception("Model not matched")
62
+
63
+ # load logs
64
+ logs = {}
65
+ for data in log_data["logs"]:
66
+ id = data["doc_id"]
67
+ response = data["resps"][0]
68
+ logs[id] = response
69
+
70
+ task = log_data["model_configs"]["task"]
71
+
72
+ pattern = re.compile(r"\d{4}_\d{4}")
73
+
74
+ if "time" in log_data:
75
+ log_time = log_data["time"]
76
+ elif pattern.search(os.path.abspath(log_file)):
77
+ log_time = pattern.findall(os.path.abspath(log_file))[-1]
78
+ else:
79
+ log_time = "unknown"
80
+
81
+ if task not in self.logs or (self.logs[task]["time"] == "unknown" or datetime.strptime(log_time, "%m%d_%H%M") > datetime.strptime(self.logs[task]["time"], "%m%d_%H%M")):
82
+ self.logs[task] = {"time": log_time, "logs": logs}
83
+
84
+ except Exception as e:
85
+ pass
86
+
87
+ accelerator = Accelerator()
88
+ if accelerator.num_processes > 1:
89
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
90
+ self.accelerator = accelerator
91
+ if self.accelerator.is_local_main_process:
92
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
93
+ self._rank = self.accelerator.local_process_index
94
+ self._world_size = self.accelerator.num_processes
95
+ else:
96
+ self.accelerator = accelerator
97
+ self._rank = self.accelerator.local_process_index
98
+ self._world_size = self.accelerator.num_processes
99
+
100
+ self.device = self.accelerator.device
101
+
102
+ def generate_until(self, requests) -> List[str]:
103
+ res = []
104
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
105
+
106
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
107
+ response = self.logs[task]["logs"][doc_id]
108
+ res.append(response[0])
109
+ pbar.update(1)
110
+
111
+ pbar.close()
112
+ return res
113
+
114
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
115
+ # TODO
116
+ assert False, "not support"
117
+
118
+ def generate_until_multi_round(self, requests) -> List[str]:
119
+ return generate_until(self, requests)
LaViDa/eval/lmms_eval/models/fuyu.py ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+
3
+ warnings.simplefilter("ignore", category=DeprecationWarning)
4
+ warnings.filterwarnings("ignore")
5
+
6
+ from typing import List, Optional, Tuple, Union
7
+
8
+ import torch
9
+ from accelerate import Accelerator, DistributedType
10
+ from accelerate.state import AcceleratorState
11
+ from loguru import logger as eval_logger
12
+ from PIL import Image
13
+ from tqdm import tqdm
14
+ from transformers import (
15
+ AutoTokenizer,
16
+ FuyuForCausalLM,
17
+ FuyuImageProcessor,
18
+ FuyuProcessor,
19
+ )
20
+
21
+ from lmms_eval import utils
22
+ from lmms_eval.api.instance import Instance
23
+ from lmms_eval.api.model import lmms
24
+ from lmms_eval.api.registry import register_model
25
+
26
+
27
+ @register_model("fuyu")
28
+ class Fuyu(lmms):
29
+ """
30
+ Fuyu Model
31
+ """
32
+
33
+ def __init__(
34
+ self,
35
+ pretrained: str = "adept/fuyu-8b",
36
+ device: Optional[str] = "cuda",
37
+ max_new_tokens: int = 256,
38
+ batch_size: Optional[Union[int, str]] = 1,
39
+ **kwargs,
40
+ ) -> None:
41
+ super().__init__()
42
+ # Do not use kwargs for now
43
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
44
+
45
+ accelerator = Accelerator()
46
+ if accelerator.num_processes > 1:
47
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
48
+ else:
49
+ self._device = device
50
+
51
+ self._model = FuyuForCausalLM.from_pretrained(pretrained, torch_dtype=torch.bfloat16, device_map=self.device)
52
+ self.model.eval()
53
+ self.model.tie_weights()
54
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
55
+ self._config = self.model.config
56
+
57
+ self.image_processor = FuyuImageProcessor()
58
+ self.processor = FuyuProcessor(image_processor=self.image_processor, tokenizer=self.tokenizer)
59
+ self.max_new_tokens = max_new_tokens
60
+ self.batch_size_per_gpu = int(batch_size)
61
+ accelerator = Accelerator()
62
+ if accelerator.num_processes > 1:
63
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
64
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
65
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
66
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
67
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
68
+ kwargs = {
69
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
70
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
71
+ }
72
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
73
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
74
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
75
+ self._model = accelerator.prepare(self.model)
76
+ else:
77
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
78
+ self.accelerator = accelerator
79
+ if self.accelerator.is_local_main_process:
80
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
81
+ self._rank = self.accelerator.local_process_index
82
+ self._world_size = self.accelerator.num_processes
83
+ else:
84
+ self.model.to(self._device)
85
+ self._rank = 0
86
+ self._world_size = 1
87
+
88
+ """if accelerator.num_processes > 1:
89
+ assert accelerator.distributed_type in [
90
+ DistributedType.FSDP,
91
+ DistributedType.MULTI_GPU,
92
+ ], "Unsupported distributed type provided. Only DDP and FSDP are supported."
93
+ if accelerator.distributed_type == DistributedType.FSDP:
94
+ self._model = accelerator.prepare(self.model)
95
+ else:
96
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
97
+ self.accelerator = accelerator
98
+ if self.accelerator.is_local_main_process:
99
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
100
+ self._rank = self.accelerator.local_process_index
101
+ self._world_size = self.accelerator.num_processes"""
102
+
103
+ @property
104
+ def config(self):
105
+ # return the associated transformers.AutoConfig for the given pretrained model.
106
+ return self._config
107
+
108
+ @property
109
+ def tokenizer(self):
110
+ return self._tokenizer
111
+
112
+ @property
113
+ def model(self):
114
+ # returns the model, unwrapping it if using Accelerate
115
+ if hasattr(self, "accelerator"):
116
+ return self.accelerator.unwrap_model(self._model)
117
+ else:
118
+ return self._model
119
+
120
+ @property
121
+ def eot_token_id(self):
122
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
123
+ return self.tokenizer.eos_token_id
124
+
125
+ @property
126
+ def max_length(self):
127
+ # Assuming max_length is the sum of max context tokens and max new tokens
128
+ return self.tokenizer.model_max_length
129
+
130
+ @property
131
+ def batch_size(self):
132
+ return self.batch_size_per_gpu
133
+
134
+ @property
135
+ def device(self):
136
+ return self._device
137
+
138
+ @property
139
+ def rank(self):
140
+ return self._rank
141
+
142
+ @property
143
+ def world_size(self):
144
+ return self._world_size
145
+
146
+ def flatten(self, input, only_get_first=False):
147
+ new_list = []
148
+ for i in input:
149
+ for j in i:
150
+ new_list.append(j)
151
+ if only_get_first:
152
+ break
153
+ return new_list
154
+
155
+ def generate_until(self, requests: List[Instance]) -> List[str]:
156
+ res = []
157
+
158
+ def _collate(x):
159
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
160
+ # - time estimates will always be over not underestimates, which is more useful for planning
161
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
162
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
163
+ # automatic adaptive batches much much easier to implement
164
+ # - any OOMs will happen right away rather than near the end
165
+ toks = self.tok_encode(x[0])
166
+ return -len(toks), x[0]
167
+
168
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
169
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
170
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
171
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
172
+
173
+ for chunk in chunks:
174
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
175
+ task = task[0]
176
+ split = split[0]
177
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
178
+ visuals = self.flatten(visuals, only_get_first=True)
179
+ gen_kwargs = all_gen_kwargs[0]
180
+
181
+ # if isinstance(visuals[0], list):
182
+ # visuals = [visuals[idx][0] for idx in range(len(visuals))] # get the first image in multi-image scenarios.
183
+
184
+ # assert len(contexts) == self.batch_size_per_gpu, f"Expected contexts batch size {self.batch_size_per_gpu}, got {len(contexts)}"
185
+ # assert len(visuals) == self.batch_size_per_gpu, f"Expected visuals batch size {self.batch_size_per_gpu}, got {len(visuals)}"
186
+ formatted_contexts = [f"{context}\n" for context in contexts]
187
+ model_inputs = self.processor(text=formatted_contexts, images=visuals, device=self.device)
188
+ for k, v in model_inputs.items():
189
+ model_inputs[k] = v.to(self.device, non_blocking=True) if isinstance(v, torch.Tensor) else [vv.to(self.device, non_blocking=True) for vv in v]
190
+
191
+ for index in range(len(model_inputs["image_patches"])):
192
+ model_inputs["image_patches"][index] = model_inputs["image_patches"][index].to(dtype=next(self.model.parameters()).dtype)
193
+
194
+ # preconfigure gen_kwargs with defaults
195
+ gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
196
+ if "max_new_tokens" not in gen_kwargs:
197
+ gen_kwargs["max_new_tokens"] = 256
198
+ if "temperature" not in gen_kwargs:
199
+ gen_kwargs["temperature"] = 0
200
+ if "top_p" not in gen_kwargs:
201
+ gen_kwargs["top_p"] = None
202
+ if "num_beams" not in gen_kwargs:
203
+ gen_kwargs["num_beams"] = 1
204
+ # generation_output = self.model.generate(
205
+ # **model_inputs, temperature=gen_kwargs["temperature"], max_new_tokens=gen_kwargs["max_new_tokens"], top_p=gen_kwargs["top_p"], num_beams=gen_kwargs["num_beams"], pad_token_id=self.tokenizer.eos_token_id
206
+ # )
207
+ generation_output = self.model.generate(**model_inputs, max_new_tokens=gen_kwargs["max_new_tokens"], pad_token_id=self.tokenizer.eos_token_id)
208
+ generation_texts = self.processor.batch_decode(generation_output, skip_special_tokens=True)
209
+ response = [gen_text.split("\x04")[1].strip(" ").strip("\n") for gen_text in generation_texts]
210
+ res.extend(response)
211
+ pbar.update(1)
212
+
213
+ pbar.close()
214
+ return res
215
+
216
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
217
+ # TODO
218
+ res = []
219
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
220
+
221
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
222
+ # encode, pad, and truncate contexts for this batch
223
+ if type(doc_to_target) == str:
224
+ continuation = doc_to_target
225
+ else:
226
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
227
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
228
+ visuals = self.flatten(visuals)
229
+ formatted_contexts = [f"{contexts}\n"]
230
+ formatted_continuation = [f"{contexts}\n{continuation}"]
231
+ model_inputs = self.processor(text=formatted_continuation, images=visuals, device=self.device)
232
+ for k, v in model_inputs.items():
233
+ model_inputs[k] = v.to(self.device, non_blocking=True) if isinstance(v, torch.Tensor) else [vv.to(self.device, non_blocking=True) for vv in v]
234
+
235
+ for index in range(len(model_inputs["image_patches"])):
236
+ model_inputs["image_patches"][index] = model_inputs["image_patches"][index].to(dtype=next(self.model.parameters()).dtype)
237
+
238
+ labels = model_inputs["input_ids"].clone()
239
+ contxt_id = self.processor(text=formatted_contexts, return_tensors="pt")["input_ids"]
240
+ labels[: len(contxt_id)] = -100
241
+ with torch.inference_mode():
242
+ outputs = self.model(**model_inputs, labels=labels)
243
+ loss = outputs["loss"]
244
+ # loss = torch.exp(loss)
245
+ logits = outputs["logits"]
246
+ greedy_tokens = logits.argmax(dim=-1)
247
+ cont_toks = model_inputs["input_ids"][:, contxt_id.shape[1] :] # [1, seq]
248
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : model_inputs["input_ids"].shape[1]] # [1, seq]
249
+ max_equal = (greedy_tokens == cont_toks).all()
250
+ res.append((float(loss.item()), bool(max_equal)))
251
+ pbar.update(1)
252
+
253
+ pbar.close()
254
+ return res
255
+
256
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
257
+ """ """
258
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
259
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
260
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
261
+ if left_truncate_len:
262
+ encoding = encoding[-left_truncate_len:]
263
+ return encoding
264
+
265
+ def tok_decode(self, tokens):
266
+ return self.tokenizer.decode(tokens)
267
+
268
+ def generate_until_multi_round(self, requests) -> List[str]:
269
+ raise NotImplementedError("TODO: Implement multi-round generation for Fuyu")
LaViDa/eval/lmms_eval/models/gemini_api.py ADDED
@@ -0,0 +1,306 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import io
2
+ import json
3
+ import os
4
+ import pathlib
5
+ import re
6
+ import time
7
+ from typing import List, Tuple
8
+
9
+ import datasets
10
+ from accelerate import Accelerator, DistributedType
11
+ from loguru import logger as eval_logger
12
+ from PIL import Image
13
+ from tqdm import tqdm
14
+
15
+ from lmms_eval.api.instance import Instance
16
+ from lmms_eval.api.model import lmms
17
+ from lmms_eval.api.registry import register_model
18
+
19
+ try:
20
+ import google.generativeai as genai
21
+ from google.generativeai.types import HarmBlockThreshold, HarmCategory
22
+
23
+ NUM_SECONDS_TO_SLEEP = 30
24
+ GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
25
+ genai.configure(api_key=GOOGLE_API_KEY)
26
+
27
+ except Exception as e:
28
+ eval_logger.error(f"Error importing generativeai: {str(e)}")
29
+ genai = None
30
+
31
+ try:
32
+ import soundfile as sf
33
+ except Exception as e:
34
+ eval_logger.warning(f"Error importing soundfile, audio generation will not work: {str(e)}")
35
+
36
+
37
+ @register_model("gemini_api")
38
+ class GeminiAPI(lmms):
39
+ def __init__(
40
+ self,
41
+ model_version: str = "gemini-1.5-pro",
42
+ # modality: str = "image",
43
+ timeout: int = 120,
44
+ continual_mode: bool = True,
45
+ response_persistent_folder: str = "./logs/gemini_persistent_folder",
46
+ interleave: bool = False,
47
+ # We will cache the Gemini API response in this path and use it for future requests
48
+ **kwargs,
49
+ ) -> None:
50
+ super().__init__()
51
+ self.model_version = model_version
52
+ self.timeout = timeout
53
+ self.model = genai.GenerativeModel(model_version)
54
+ self.continual_mode = continual_mode
55
+ self.response_persistent_file = ""
56
+ self.interleave = interleave
57
+ # if self.continual_mode and response_persistent_folder is None:
58
+ # raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.")
59
+ if self.continual_mode:
60
+ self.response_persistent_folder = response_persistent_folder
61
+ if not os.path.exists(self.response_persistent_folder):
62
+ os.makedirs(self.response_persistent_folder)
63
+ self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
64
+
65
+ if os.path.exists(self.response_persistent_file):
66
+ with open(self.response_persistent_file, "r") as f:
67
+ self.response_cache = json.load(f)
68
+ self.cache_mode = "resume"
69
+ else:
70
+ self.response_cache = {}
71
+ self.cache_mode = "start"
72
+
73
+ accelerator = Accelerator()
74
+ if accelerator.num_processes > 1:
75
+ assert self.continual_mode is False, "Continual mode is not supported with distributed inference."
76
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
77
+ self.accelerator = accelerator
78
+ if self.accelerator.is_local_main_process:
79
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
80
+ self._rank = self.accelerator.local_process_index
81
+ self._world_size = self.accelerator.num_processes
82
+ else:
83
+ self.accelerator = accelerator
84
+ self._rank = self.accelerator.local_process_index
85
+ self._world_size = self.accelerator.num_processes
86
+
87
+ self.device = self.accelerator.device
88
+
89
+ # self.modality = modality
90
+
91
+ self.video_pool = []
92
+
93
+ def free_video(self):
94
+ for video in self.video_pool:
95
+ video.delete()
96
+ self.video_pool = []
97
+
98
+ def flatten(self, input):
99
+ new_list = []
100
+ for i in input:
101
+ for j in i:
102
+ new_list.append(j)
103
+ return new_list
104
+
105
+ def get_image_size(self, image):
106
+ # Create a BytesIO object to store the image bytes
107
+ img_byte_array = io.BytesIO()
108
+
109
+ # Save the image to the BytesIO object
110
+ image.save(img_byte_array, format="PNG")
111
+
112
+ # Get the size of the BytesIO object
113
+ img_size = img_byte_array.tell()
114
+
115
+ return img_size
116
+
117
+ def encode_video(self, video_path):
118
+ uploaded_obj = genai.upload_file(path=video_path)
119
+ time.sleep(5)
120
+ self.video_pool.append(uploaded_obj)
121
+ return uploaded_obj
122
+
123
+ def encode_audio(self, audio):
124
+ audio_io = io.BytesIO()
125
+ sf.write(audio_io, audio["array"], audio["sampling_rate"], format="WAV")
126
+ return genai.upload_file(audio_io, mime_type="audio/wav")
127
+
128
+ def convert_modality(self, images):
129
+ for idx, img in enumerate(images):
130
+ if isinstance(img, dict) and "sampling_rate" in img: # audio
131
+ audio = self.encode_audio(img)
132
+ images[idx] = audio
133
+ elif isinstance(img, str): # video
134
+ try:
135
+ images[idx] = self.encode_video(img)
136
+ except Exception as e:
137
+ eval_logger.error(f"Error converting video: {str(e)}")
138
+ return images
139
+
140
+ def construct_interleaved_input(self, content, media):
141
+ pattern = r"<media_(\d+)>"
142
+ parts = re.split(pattern, content)
143
+ result = []
144
+ for i, part in enumerate(parts):
145
+ if i % 2 == 0:
146
+ if part == "":
147
+ continue
148
+ result.append(part)
149
+ else:
150
+ result.append(media[int(part)])
151
+
152
+ return result
153
+
154
+ def generate_until(self, requests) -> List[str]:
155
+ res = []
156
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
157
+
158
+ def get_uuid(task, split, doc_id):
159
+ return f"{task}___{split}___{doc_id}"
160
+
161
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
162
+ if self.continual_mode and self.cache_mode == "resume":
163
+ doc_uuid = get_uuid(task, split, doc_id)
164
+ if doc_uuid in self.response_cache:
165
+ content = self.response_cache[doc_uuid]
166
+ if content:
167
+ res.append(content)
168
+ pbar.update(1)
169
+ continue
170
+
171
+ if "max_new_tokens" not in gen_kwargs:
172
+ gen_kwargs["max_new_tokens"] = 1024
173
+ if "temperature" not in gen_kwargs:
174
+ gen_kwargs["temperature"] = 0
175
+
176
+ config = genai.GenerationConfig(
177
+ max_output_tokens=gen_kwargs["max_new_tokens"],
178
+ temperature=gen_kwargs["temperature"],
179
+ )
180
+
181
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
182
+ visuals = self.flatten(visuals)
183
+ visuals = self.convert_modality(visuals)
184
+
185
+ if self.interleave:
186
+ message = self.construct_interleaved_input(contexts, visuals)
187
+ else:
188
+ message = [contexts] + visuals
189
+
190
+ for attempt in range(5):
191
+ try:
192
+ content = self.model.generate_content(
193
+ message,
194
+ generation_config=config,
195
+ safety_settings={
196
+ HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
197
+ HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
198
+ HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
199
+ HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
200
+ },
201
+ )
202
+ content = content.text
203
+ break
204
+ except Exception as e:
205
+ eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
206
+ if isinstance(e, ValueError):
207
+ try:
208
+ eval_logger.info(f"Prompt feed_back: {content.prompt_feedback}")
209
+ content = ""
210
+ break
211
+ except Exception:
212
+ pass
213
+ if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt
214
+ time.sleep(NUM_SECONDS_TO_SLEEP)
215
+ else: # If this was the last attempt, log and return empty
216
+ eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
217
+ content = ""
218
+ res.append(content)
219
+ pbar.update(1)
220
+
221
+ self.free_video()
222
+
223
+ if self.continual_mode is True: # Cache the response
224
+ doc_uuid = get_uuid(task, split, doc_id)
225
+ self.response_cache[doc_uuid] = content
226
+ with open(self.response_persistent_file, "w") as f:
227
+ json.dump(self.response_cache, f)
228
+
229
+ pbar.close()
230
+ return res
231
+
232
+ def generate_until_multi_round(self, requests) -> List[str]:
233
+ raise NotImplementedError("TODO: Implement multi-round generation for Gemini API")
234
+
235
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
236
+ # TODO
237
+ assert False, "Gemini API not support"
238
+
239
+ def get_image_audio_text_interleaved_messsage(self, image_path, audio_path, question):
240
+ # image_path for list of image path
241
+ # audio_path for list of audio path
242
+ # question for question
243
+
244
+ # fixed image token and no audio in text
245
+ for index in range(1, 1 + len(image_path)):
246
+ question = question.replace(f"[img{index}]", "<image>")
247
+ for index in range(1, 1 + len(audio_path)):
248
+ question = question.replace(f"[audio{index}]", "<audio>")
249
+
250
+ text = question
251
+
252
+ info_list = []
253
+ image_counter = 0
254
+ audio_counter = 0
255
+ for part in re.split(r"(<image>|<audio>)", text):
256
+ if part == "<image>":
257
+ info_list.append(Image.open(image_path[image_counter]))
258
+ image_counter += 1
259
+ elif part == "<audio>":
260
+ info_list.append({"mime_type": "audio/wav", "data": pathlib.Path(audio_path[audio_counter]).read_bytes()})
261
+ audio_counter += 1
262
+ else:
263
+ if part == " ":
264
+ continue
265
+ info_list.append(part)
266
+
267
+ return info_list
268
+
269
+ def get_video_audio_text_interleaved_message(self, video_path, audio_path, question):
270
+ # image_path for list of image path
271
+ # audio_path for list of audio path
272
+ # question for question
273
+
274
+ # fixed video token and no audio in text
275
+ for index in range(1, 1 + len(video_path)):
276
+ question = question.replace(f"[video{index}]", "<video>")
277
+ for index in range(1, 1 + len(audio_path)):
278
+ question = question.replace(f"[audio{index}]", "<audio>")
279
+
280
+ text = question
281
+
282
+ info_list = []
283
+ video_counter = 0
284
+ audio_counter = 0
285
+ for part in re.split(r"(<video>|<audio>)", text):
286
+ if part == "<video>":
287
+ current_video_file_name = video_path[video_counter]
288
+ current_video_file = genai.upload_file(path=current_video_file_name)
289
+ while current_video_file.state.name == "processing":
290
+ print("uploading file")
291
+ time.sleep(5)
292
+ current_video_file = genai.get_file(current_video_file.name)
293
+ if current_video_file.state.name == "FAILED":
294
+ print("uploading file failed, next question")
295
+ return 0
296
+ info_list.append(current_video_file)
297
+ video_counter += 1
298
+ elif part == "<audio>":
299
+ info_list.append({"mime_type": "audio/wav", "data": pathlib.Path(audio_path[audio_counter]).read_bytes()})
300
+ audio_counter += 1
301
+ else:
302
+ if part == " ":
303
+ continue
304
+ info_list.append(part)
305
+
306
+ return info_list
LaViDa/eval/lmms_eval/models/gpt4v.py ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import json
3
+ import os
4
+ import time
5
+ from copy import deepcopy
6
+ from io import BytesIO
7
+ from typing import List, Tuple, Union
8
+
9
+ import numpy as np
10
+ import requests as url_requests
11
+ from accelerate import Accelerator, DistributedType
12
+ from openai import AzureOpenAI, OpenAI
13
+ from tqdm import tqdm
14
+
15
+ from lmms_eval.api.instance import Instance
16
+ from lmms_eval.api.model import lmms
17
+ from lmms_eval.api.registry import register_model
18
+
19
+ try:
20
+ from decord import VideoReader, cpu
21
+ except ImportError:
22
+ pass
23
+
24
+ from loguru import logger as eval_logger
25
+ from PIL import Image
26
+
27
+ API_TYPE = os.getenv("API_TYPE", "openai")
28
+ NUM_SECONDS_TO_SLEEP = 10
29
+ if API_TYPE == "openai":
30
+ API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
31
+ API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
32
+
33
+ elif API_TYPE == "azure":
34
+ API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
35
+ API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
36
+ API_VERSION = os.getenv("AZURE_API_VERSION", "2023-07-01-preview")
37
+
38
+
39
+ @register_model("gpt4v")
40
+ class GPT4V(lmms):
41
+ def __init__(
42
+ self,
43
+ model_version: str = "gpt-4-vision-preview",
44
+ modality: str = "video",
45
+ max_frames_num: int = 10,
46
+ timeout: int = 120,
47
+ continual_mode: bool = False,
48
+ response_persistent_folder: str = None,
49
+ max_size_in_mb: int = 20,
50
+ **kwargs,
51
+ ) -> None:
52
+ super().__init__()
53
+ # Manually set a image token for GPT4V so that we can search for it
54
+ # and split the text and image
55
+ # Here we just use the same token as llava for convenient
56
+ self.model_version = model_version
57
+ self.modality = modality
58
+ self.max_frames_num = max_frames_num
59
+ self.image_token = "<image>"
60
+ self.timeout = timeout
61
+ self.continual_mode = continual_mode
62
+ if self.continual_mode:
63
+ if response_persistent_folder is None:
64
+ raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
65
+
66
+ os.makedirs(response_persistent_folder, exist_ok=True)
67
+ self.response_persistent_folder = response_persistent_folder
68
+ self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
69
+
70
+ if os.path.exists(self.response_persistent_file):
71
+ with open(self.response_persistent_file, "r") as f:
72
+ self.response_cache = json.load(f)
73
+ self.cache_mode = "resume"
74
+ else:
75
+ self.response_cache = {}
76
+ self.cache_mode = "start"
77
+
78
+ if API_TYPE == "openai":
79
+ self.client = OpenAI(api_key=API_KEY)
80
+ elif API_TYPE == "azure":
81
+ self.client = AzureOpenAI(api_key=API_KEY, azure_endpoint=API_URL, api_version=API_VERSION)
82
+
83
+ accelerator = Accelerator()
84
+ # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
85
+ if accelerator.num_processes > 1:
86
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
87
+ self.accelerator = accelerator
88
+ if self.accelerator.is_local_main_process:
89
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
90
+ self._rank = self.accelerator.local_process_index
91
+ self._world_size = self.accelerator.num_processes
92
+ else:
93
+ self.accelerator = accelerator
94
+ self._rank = self.accelerator.local_process_index
95
+ self._world_size = self.accelerator.num_processes
96
+
97
+ self.max_size_in_mb = max_size_in_mb
98
+ self.device = self.accelerator.device
99
+
100
+ # Function to encode the image
101
+ def encode_image(self, image: Union[Image.Image, str]):
102
+ max_size = self.max_size_in_mb * 1024 * 1024 # 20MB in bytes
103
+ if isinstance(image, str):
104
+ img = Image.open(image).convert("RGB")
105
+ else:
106
+ img = image.copy()
107
+
108
+ output_buffer = BytesIO()
109
+ img.save(output_buffer, format="PNG")
110
+ byte_data = output_buffer.getvalue()
111
+
112
+ # If image is too large, resize it while maintaining aspect ratio
113
+ while len(byte_data) > max_size and img.size[0] > 100 and img.size[1] > 100:
114
+ new_size = (int(img.size[0] * 0.75), int(img.size[1] * 0.75))
115
+ img = img.resize(new_size, Image.Resampling.LANCZOS)
116
+
117
+ output_buffer = BytesIO()
118
+ img.save(output_buffer, format="PNG")
119
+ byte_data = output_buffer.getvalue()
120
+
121
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
122
+ return base64_str
123
+
124
+ # Function to encode the video
125
+ def encode_video(self, video_path, for_get_frames_num):
126
+ vr = VideoReader(video_path, ctx=cpu(0))
127
+ total_frame_num = len(vr)
128
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int)
129
+
130
+ # Ensure the last frame is included
131
+ if total_frame_num - 1 not in uniform_sampled_frames:
132
+ uniform_sampled_frames = np.append(uniform_sampled_frames, total_frame_num - 1)
133
+
134
+ frame_idx = uniform_sampled_frames.tolist()
135
+ frames = vr.get_batch(frame_idx).asnumpy()
136
+
137
+ base64_frames = []
138
+ for frame in frames:
139
+ img = Image.fromarray(frame)
140
+ output_buffer = BytesIO()
141
+ img.save(output_buffer, format="PNG")
142
+ byte_data = output_buffer.getvalue()
143
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
144
+ base64_frames.append(base64_str)
145
+
146
+ return base64_frames
147
+
148
+ def flatten(self, input):
149
+ new_list = []
150
+ for i in input:
151
+ for j in i:
152
+ new_list.append(j)
153
+ return new_list
154
+
155
+ def generate_until(self, requests) -> List[str]:
156
+ res = []
157
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
158
+
159
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
160
+ if self.continual_mode is True and self.cache_mode == "resume":
161
+ doc_uuid = f"{task}___{split}___{doc_id}"
162
+ if doc_uuid in self.response_cache:
163
+ response_text = self.response_cache[doc_uuid]
164
+ if response_text:
165
+ res.append(response_text)
166
+ pbar.update(1)
167
+ continue
168
+
169
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
170
+ if None in visuals:
171
+ visuals = []
172
+ imgs = []
173
+ else:
174
+ visuals = self.flatten(visuals)
175
+ imgs = [] # multiple images or frames for video
176
+ for visual in visuals:
177
+ if isinstance(visual, str) and (".mp4" in visual or ".avi" in visual or ".mov" in visual or ".flv" in visual or ".wmv" in visual):
178
+ frames = self.encode_video(visual, self.max_frames_num)
179
+ imgs.extend(frames)
180
+ elif isinstance(visual, str) and (".jpg" in visual or ".jpeg" in visual or ".png" in visual or ".gif" in visual or ".bmp" in visual or ".tiff" in visual or ".webp" in visual):
181
+ img = self.encode_image(visual)
182
+ imgs.append(img)
183
+ elif isinstance(visual, Image.Image):
184
+ img = self.encode_image(visual)
185
+ imgs.append(img)
186
+
187
+ payload = {"messages": []}
188
+ payload["model"] = self.model_version
189
+
190
+ payload["messages"].append({"role": "user", "content": []})
191
+ payload["messages"][0]["content"].append({"type": "text", "text": contexts})
192
+ for img in imgs:
193
+ payload["messages"][0]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}})
194
+
195
+ if "max_new_tokens" not in gen_kwargs:
196
+ gen_kwargs["max_new_tokens"] = 1024
197
+ if gen_kwargs["max_new_tokens"] > 4096:
198
+ gen_kwargs["max_new_tokens"] = 4096
199
+ if "temperature" not in gen_kwargs:
200
+ gen_kwargs["temperature"] = 0
201
+ if "top_p" not in gen_kwargs:
202
+ gen_kwargs["top_p"] = None
203
+ if "num_beams" not in gen_kwargs:
204
+ gen_kwargs["num_beams"] = 1
205
+
206
+ payload["max_tokens"] = gen_kwargs["max_new_tokens"]
207
+ payload["temperature"] = gen_kwargs["temperature"]
208
+
209
+ MAX_RETRIES = 5
210
+ for attempt in range(MAX_RETRIES):
211
+ try:
212
+ response = self.client.chat.completions.create(**payload)
213
+ response_text = response.choices[0].message.content
214
+ break # If successful, break out of the loop
215
+
216
+ except Exception as e:
217
+ error_msg = str(e)
218
+ eval_logger.info(f"Attempt {attempt + 1}/{MAX_RETRIES} failed with error: {error_msg}")
219
+
220
+ # On last attempt, log error and set empty response
221
+ if attempt == MAX_RETRIES - 1:
222
+ eval_logger.error(f"All {MAX_RETRIES} attempts failed. Last error: {error_msg}")
223
+ response_text = ""
224
+ else:
225
+ time.sleep(NUM_SECONDS_TO_SLEEP)
226
+
227
+ res.append(response_text)
228
+ pbar.update(1)
229
+
230
+ if self.continual_mode is True: # Cache the response
231
+ doc_uuid = f"{task}___{split}___{doc_id}"
232
+ self.response_cache[doc_uuid] = response_text
233
+ with open(self.response_persistent_file, "w") as f:
234
+ json.dump(self.response_cache, f)
235
+
236
+ pbar.close()
237
+ return res
238
+
239
+ def generate_until_multi_round(self, requests) -> List[str]:
240
+ raise NotImplementedError("TODO: Implement multi-round generation for GPT4V")
241
+
242
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
243
+ # TODO
244
+ assert False, "GPT4V not support"
LaViDa/eval/lmms_eval/models/idefics2.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ from accelerate import Accelerator, DistributedType
6
+ from accelerate.state import AcceleratorState
7
+ from torchvision.transforms.functional import to_pil_image
8
+ from tqdm import tqdm
9
+ from transformers import AutoProcessor, Idefics2ForConditionalGeneration
10
+
11
+ from lmms_eval import utils
12
+ from lmms_eval.api.instance import Instance
13
+ from lmms_eval.api.model import lmms
14
+ from lmms_eval.api.registry import register_model
15
+ from lmms_eval.models.model_utils.load_video import load_video_decord
16
+
17
+ warnings.filterwarnings("ignore")
18
+
19
+ from loguru import logger as eval_logger
20
+
21
+ DEFAULT_IMAGE_TOKEN = "<image>"
22
+ try:
23
+ import flash_attn
24
+
25
+ best_fit_attn_implementation = "flash_attention_2"
26
+ except ImportError:
27
+ best_fit_attn_implementation = "eager"
28
+
29
+
30
+ @register_model("idefics2")
31
+ class Idefics2(lmms):
32
+ """
33
+ Idefics2 Model for Hugging Face Transformers: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py
34
+
35
+ Example usage:
36
+
37
+ accelerate launch --num_processes=8 -m lmms_eval \
38
+ --model idefics2 \
39
+ --model_args pretrained=HuggingFaceM4/idefics2-8b \
40
+ --tasks mme \
41
+ --batch_size 1 \
42
+ --output_path ./logs/ \
43
+ --log_samples
44
+ """
45
+
46
+ def __init__(
47
+ self,
48
+ pretrained: str = "HuggingFaceM4/idefics2-8b",
49
+ revision: str = "main",
50
+ device: str = "cuda",
51
+ dtype: Optional[Union[str, torch.dtype]] = "float16",
52
+ batch_size: int = 1,
53
+ trust_remote_code: Optional[bool] = False,
54
+ attn_implementation: Optional[str] = best_fit_attn_implementation,
55
+ device_map: str = "",
56
+ use_cache: bool = True,
57
+ do_image_splitting: bool = False,
58
+ max_frames_num: int = 16,
59
+ **kwargs,
60
+ ) -> None:
61
+ super().__init__()
62
+ # Do not use kwargs for now
63
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
64
+
65
+ accelerator = Accelerator()
66
+ if accelerator.num_processes > 1 and device_map == "":
67
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
68
+ self.device_map = f"cuda:{accelerator.local_process_index}"
69
+ else:
70
+ self._device = torch.device(device)
71
+ self.device_map = device_map
72
+ if isinstance(dtype, str) and dtype != "auto":
73
+ dtype = getattr(torch, dtype)
74
+ self._model = Idefics2ForConditionalGeneration.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
75
+ self._processor = AutoProcessor.from_pretrained(pretrained, do_image_splitting=do_image_splitting, revision=revision, trust_remote_code=trust_remote_code)
76
+ self.max_frames_num = max_frames_num
77
+
78
+ self._tokenizer = self._processor.tokenizer
79
+ self._config = self._model.config
80
+ self.batch_size_per_gpu = int(batch_size)
81
+ self.use_cache = use_cache
82
+ if accelerator.num_processes > 1 and device_map == "":
83
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
84
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
85
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
86
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
87
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
88
+ kwargs = {
89
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
90
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
91
+ }
92
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
93
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
94
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
95
+ self._model = accelerator.prepare(self.model)
96
+ else:
97
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
98
+ self.accelerator = accelerator
99
+ if self.accelerator.is_local_main_process:
100
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
101
+ self._rank = self.accelerator.local_process_index
102
+ self._world_size = self.accelerator.num_processes
103
+ elif accelerator.num_processes == 1 and device_map == "auto":
104
+ eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
105
+ self._rank = 0
106
+ self._world_size = 1
107
+ else:
108
+ eval_logger.info(f"Using single device: {self._device}")
109
+ self.model.to(self._device)
110
+ self._rank = 0
111
+ self._world_size = 1
112
+
113
+ @property
114
+ def config(self):
115
+ # return the associated transformers.AutoConfig for the given pretrained model.
116
+ return self._config
117
+
118
+ @property
119
+ def tokenizer(self):
120
+ return self._tokenizer
121
+
122
+ @property
123
+ def model(self):
124
+ # returns the model, unwrapping it if using Accelerate
125
+ if hasattr(self, "accelerator"):
126
+ return self.accelerator.unwrap_model(self._model)
127
+ else:
128
+ return self._model
129
+
130
+ @property
131
+ def eot_token_id(self):
132
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
133
+ return self.tokenizer.eos_token_id
134
+
135
+ @property
136
+ def max_length(self):
137
+ return self._max_length
138
+
139
+ @property
140
+ def batch_size(self):
141
+ return self.batch_size_per_gpu
142
+
143
+ @property
144
+ def device(self):
145
+ return self._device
146
+
147
+ @property
148
+ def rank(self):
149
+ return self._rank
150
+
151
+ @property
152
+ def world_size(self):
153
+ return self._world_size
154
+
155
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
156
+ """ """
157
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
158
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
159
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
160
+ if left_truncate_len:
161
+ encoding = encoding[-left_truncate_len:]
162
+ return encoding
163
+
164
+ def tok_decode(self, tokens):
165
+ return self.tokenizer.decode(tokens)
166
+
167
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
168
+ raise NotImplementedError("Loglikelihood is not implemented for Idefics2 model")
169
+
170
+ def flatten(self, input):
171
+ new_list = []
172
+ for i in input:
173
+ for j in i:
174
+ new_list.append(j)
175
+ return new_list
176
+
177
+ def generate_until(self, requests: List[Instance]) -> List[str]:
178
+ res = []
179
+
180
+ def _collate(x):
181
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
182
+ # - time estimates will always be over not underestimates, which is more useful for planning
183
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
184
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
185
+ # automatic adaptive batches much much easier to implement
186
+ # - any OOMs will happen right away rather than near the end
187
+ toks = self.tok_encode(x[0])
188
+ return -len(toks), x[0]
189
+
190
+ # we group requests by their generation_kwargs,
191
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
192
+ # in the same batch.
193
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
194
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
195
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
196
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
197
+ for chunk in chunks:
198
+ contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
199
+ visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
200
+ # we assume all gen kwargs in the batch are the same
201
+ # this is safe to assume because the `grouper` object ensures it.
202
+ gen_kwargs = all_gen_kwargs[0]
203
+ #
204
+ until = gen_kwargs.pop("until", None)
205
+ image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio", None)
206
+ if "max_new_tokens" not in gen_kwargs:
207
+ gen_kwargs["max_new_tokens"] = 1024
208
+ if "temperature" not in gen_kwargs:
209
+ gen_kwargs["temperature"] = 0
210
+
211
+ prompts = []
212
+ videos = None
213
+ for context, visual in zip(contexts, visuals):
214
+ content = []
215
+ if isinstance(visual[0], str):
216
+ videos = load_video_decord(visual[0], max_frames_num=self.max_frames_num)
217
+ for _ in range(videos.shape[0]):
218
+ content.append({"type": "image"})
219
+ elif DEFAULT_IMAGE_TOKEN not in context:
220
+ for image in visual:
221
+ content.append({"type": "image"})
222
+ content.append({"type": "text", "text": context})
223
+ message = [{"role": "user", "content": content}]
224
+ prompt = self._processor.apply_chat_template(message, add_generation_prompt=True)
225
+ prompts.append(prompt)
226
+ if videos is not None:
227
+ images = []
228
+ for frame in videos:
229
+ images.append(to_pil_image(frame))
230
+ inputs = self._processor(text=prompts, images=images, padding=True, return_tensors="pt")
231
+ else:
232
+ inputs = self._processor(text=prompts, images=visuals, padding=True, return_tensors="pt")
233
+ inputs = {k: v.to(self.device) for k, v in inputs.items()}
234
+ output_ids = self.model.generate(**inputs, **gen_kwargs)
235
+ # only retain the generated text
236
+ for output_id, input_id in zip(output_ids, inputs["input_ids"]):
237
+ generated_id = output_id[len(input_id) :]
238
+ generated_text = self.tokenizer.decode(generated_id, skip_special_tokens=True)
239
+
240
+ res.append(generated_text)
241
+ pbar.update(1)
242
+ # reorder this group of results back to original unsorted form
243
+ res = re_ords.get_original(res)
244
+
245
+ pbar.close()
246
+ return res
247
+
248
+ def generate_until_multi_round(self, requests) -> List[str]:
249
+ raise NotImplementedError("TODO: Implement multi-round generation for Idefics2")
LaViDa/eval/lmms_eval/models/instructblip.py ADDED
@@ -0,0 +1,230 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import warnings
3
+ from typing import List, Optional, Tuple, Union
4
+
5
+ import torch
6
+ import transformers
7
+ from accelerate import Accelerator, DistributedType
8
+ from accelerate.state import AcceleratorState
9
+ from tqdm import tqdm
10
+ from transformers import InstructBlipForConditionalGeneration, InstructBlipProcessor
11
+
12
+ from lmms_eval import utils
13
+ from lmms_eval.api.instance import Instance
14
+ from lmms_eval.api.model import lmms
15
+ from lmms_eval.api.registry import register_model
16
+ from lmms_eval.tasks.mmmu.utils_group_img import process_images
17
+ from lmms_eval.utils import stop_sequences_criteria
18
+
19
+ warnings.filterwarnings("ignore")
20
+
21
+ from loguru import logger as eval_logger
22
+
23
+
24
+ @register_model("instructblip")
25
+ class InstructBLIP(lmms):
26
+ """
27
+ InstructBLIP Model
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ pretrained: str = "Salesforce/instructblip-vicuna-7b",
33
+ device: Optional[str] = "cuda",
34
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
35
+ batch_size: Optional[Union[int, str]] = 1,
36
+ **kwargs,
37
+ ) -> None:
38
+ super().__init__()
39
+ # Do not use kwargs for now
40
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
41
+
42
+ accelerator = Accelerator()
43
+ if accelerator.num_processes > 1:
44
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
45
+ else:
46
+ self._device = device
47
+ self._model = InstructBlipForConditionalGeneration.from_pretrained(pretrained, device_map=self._device)
48
+ self._image_processor = InstructBlipProcessor.from_pretrained(pretrained)
49
+ self._tokenizer = self._image_processor.tokenizer
50
+ self._config = self._model.config
51
+ self.model.eval()
52
+ self.model.tie_weights()
53
+ self.batch_size_per_gpu = int(batch_size)
54
+ if accelerator.num_processes > 1:
55
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
56
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
57
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
58
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
59
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
60
+ kwargs = {
61
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
62
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
63
+ }
64
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
65
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
66
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
67
+ self._model = accelerator.prepare(self.model)
68
+ else:
69
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
70
+ self.accelerator = accelerator
71
+ if self.accelerator.is_local_main_process:
72
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
73
+ self._rank = self.accelerator.local_process_index
74
+ self._world_size = self.accelerator.num_processes
75
+ else:
76
+ self.model.to(self._device)
77
+ self._rank = 0
78
+ self._world_size = 1
79
+
80
+ @property
81
+ def config(self):
82
+ # return the associated transformers.AutoConfig for the given pretrained model.
83
+ return self._config
84
+
85
+ @property
86
+ def tokenizer(self):
87
+ return self._tokenizer
88
+
89
+ @property
90
+ def model(self):
91
+ # returns the model, unwrapping it if using Accelerate
92
+ if hasattr(self, "accelerator"):
93
+ return self.accelerator.unwrap_model(self._model)
94
+ else:
95
+ return self._model
96
+
97
+ @property
98
+ def eot_token_id(self):
99
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
100
+ return self.tokenizer.eos_token_id
101
+
102
+ @property
103
+ def max_length(self):
104
+ return self._max_length
105
+
106
+ @property
107
+ def batch_size(self):
108
+ return self.batch_size_per_gpu
109
+
110
+ @property
111
+ def device(self):
112
+ return self._device
113
+
114
+ @property
115
+ def rank(self):
116
+ return self._rank
117
+
118
+ @property
119
+ def world_size(self):
120
+ return self._world_size
121
+
122
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
123
+ """ """
124
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
125
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
126
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
127
+ if left_truncate_len:
128
+ encoding = encoding[-left_truncate_len:]
129
+ return encoding
130
+
131
+ def tok_decode(self, tokens):
132
+ return self.tokenizer.decode(tokens)
133
+
134
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
135
+ # TODO
136
+ assert False, "We have not implemented this function for InstructBLIP yet"
137
+
138
+ def flatten(self, input):
139
+ new_list = []
140
+ for i in input:
141
+ for j in i:
142
+ new_list.append(j)
143
+ return new_list
144
+
145
+ def generate_until(self, requests: List[Instance]) -> List[str]:
146
+ res = []
147
+
148
+ def _collate(x):
149
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
150
+ # - time estimates will always be over not underestimates, which is more useful for planning
151
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
152
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
153
+ # automatic adaptive batches much much easier to implement
154
+ # - any OOMs will happen right away rather than near the end
155
+ toks = self.tok_encode(x[0])
156
+ return -len(toks), x[0]
157
+
158
+ # we group requests by their generation_kwargs,
159
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
160
+ # in the same batch.
161
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
162
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
163
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
164
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
165
+ for chunk in chunks:
166
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
167
+ task = task[0]
168
+ split = split[0]
169
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
170
+ visuals = self.flatten(visuals)
171
+ # we assume all gen kwargs in the batch are the same
172
+ # this is safe to assume because the `grouper` object ensures it.
173
+ gen_kwargs = all_gen_kwargs[0]
174
+
175
+ # Set default values for until and max_new_tokens
176
+ until = [self.tok_decode(self.eot_token_id)]
177
+
178
+ # Update values from gen_kwargs if present
179
+ if "until" in gen_kwargs:
180
+ until = gen_kwargs.pop("until")
181
+ if isinstance(until, str):
182
+ until = [until]
183
+ elif not isinstance(until, list):
184
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
185
+ assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
186
+ context = contexts[0]
187
+ if "<image>" in context:
188
+ # instruct blip does not expect the <image> tag
189
+ context = context.replace("<image>", "")
190
+ # Set trunction equals true here, the max length for qformer tokenizer is 512
191
+ # if not truncate, some questions will cause size mismatch
192
+ # The transformer implementation can't handle multi images for blip
193
+ # Concat it into one image
194
+ if len(visuals) > 1:
195
+ visuals = [process_images(visuals)]
196
+ inputs = self._image_processor(images=visuals, text=context, return_tensors="pt", truncation=True).to(self.device)
197
+
198
+ gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
199
+ if "max_new_tokens" not in gen_kwargs:
200
+ gen_kwargs["max_new_tokens"] = 1024
201
+ if "temperature" not in gen_kwargs:
202
+ gen_kwargs["temperature"] = 0
203
+ if "top_p" not in gen_kwargs:
204
+ gen_kwargs["top_p"] = None
205
+ if "num_beams" not in gen_kwargs:
206
+ gen_kwargs["num_beams"] = 1
207
+ try:
208
+ cont = self.model.generate(
209
+ **inputs,
210
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
211
+ temperature=gen_kwargs["temperature"],
212
+ top_p=gen_kwargs["top_p"],
213
+ num_beams=gen_kwargs["num_beams"],
214
+ max_new_tokens=gen_kwargs["max_new_tokens"],
215
+ )
216
+ except Exception as e:
217
+ eval_logger.error(f"Error {e} in generating")
218
+ cont = ""
219
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0].strip()
220
+ res.append(text_outputs)
221
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
222
+ pbar.update(1)
223
+ # reorder this group of results back to original unsorted form
224
+ res = re_ords.get_original(res)
225
+
226
+ pbar.close()
227
+ return res
228
+
229
+ def generate_until_multi_round(self, requests) -> List[str]:
230
+ raise NotImplementedError("TODO: Implement multi-round generation for InstructBlip")
LaViDa/eval/lmms_eval/models/internvideo2.py ADDED
@@ -0,0 +1,366 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import os
3
+ from typing import List, Tuple
4
+
5
+ import decord
6
+ import numpy as np
7
+ import torch
8
+ import torchvision.transforms as T
9
+ from accelerate import Accelerator, DistributedType
10
+ from decord import VideoReader, cpu
11
+
12
+ decord.bridge.set_bridge("torch")
13
+ import torch.nn.functional as F
14
+ from PIL import Image
15
+ from tqdm import tqdm
16
+ from transformers import AutoModel, AutoTokenizer
17
+
18
+ from lmms_eval.api.instance import Instance
19
+ from lmms_eval.api.model import lmms
20
+ from lmms_eval.api.registry import register_model
21
+
22
+ eval_logger = logging.getLogger("eval_logger")
23
+
24
+
25
+ from datetime import timedelta
26
+
27
+ from accelerate.state import AcceleratorState
28
+ from accelerate.utils import InitProcessGroupKwargs
29
+
30
+ DEFAULT_GEN_KWARGS = dict(
31
+ num_beams=1,
32
+ max_new_tokens=1024,
33
+ do_sample=False,
34
+ )
35
+
36
+ # def get_index(num_frames, num_segments):
37
+ # seg_size = float(num_frames - 1) / num_segments
38
+ # start = int(seg_size / 2)
39
+ # offsets = np.array([
40
+ # start + int(np.round(seg_size * idx)) for idx in range(num_segments)
41
+ # ])
42
+ # return offsets
43
+
44
+
45
+ def get_index(max_frame, num_segments, fps, first_idx=0, bound=None):
46
+ if bound:
47
+ start, end = bound[0], bound[1]
48
+ if start is None:
49
+ start, end = -100000, 100000
50
+ else:
51
+ start, end = -100000, 100000
52
+ start_idx = max(first_idx, round(start * fps))
53
+ end_idx = min(round(end * fps), max_frame)
54
+ seg_size = float(end_idx - start_idx) / num_segments
55
+ frame_indices = np.array([int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)])
56
+ return frame_indices
57
+
58
+
59
+ def load_image(image_path, resolution=224, hd_num=6):
60
+ image = Image.open(image_path).convert("RGB")
61
+ image_tensor = T.PILToTensor()(image).unsqueeze(0)
62
+ image_tensor = HD_transform_no_padding(image_tensor.float(), image_size=resolution, hd_num=hd_num)
63
+ T_, C, H, W = image_tensor.shape
64
+
65
+ mean = (0.485, 0.456, 0.406)
66
+ std = (0.229, 0.224, 0.225)
67
+
68
+ transform = T.Compose([T.Lambda(lambda x: x.float().div(255.0)), T.Normalize(mean, std)])
69
+ image_tensor = transform(image_tensor).cuda()
70
+
71
+ sub_img = image_tensor.reshape(1, T_, 3, H // resolution, resolution, W // resolution, resolution).permute(0, 3, 5, 1, 2, 4, 6).reshape(-1, T_, 3, resolution, resolution).contiguous()
72
+
73
+ glb_img = F.interpolate(image_tensor.float(), size=(resolution, resolution), mode="bicubic", align_corners=False).to(sub_img.dtype).unsqueeze(0)
74
+
75
+ image_tensor = torch.cat([sub_img, glb_img]) # .unsqueeze(0)
76
+ return image_tensor
77
+
78
+
79
+ def load_video(video_path, num_segments=16, return_msg=False, resolution=224, hd_num=6, padding=False):
80
+ vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
81
+ num_frames = len(vr) - 1
82
+
83
+ frame_indices = get_index(max_frame=num_frames, num_segments=num_segments, fps=float(vr.get_avg_fps()), first_idx=0, bound=None)
84
+ mean = (0.485, 0.456, 0.406)
85
+ std = (0.229, 0.224, 0.225)
86
+
87
+ transform = T.Compose([T.Lambda(lambda x: x.float().div(255.0)), T.Normalize(mean, std)])
88
+
89
+ frames = vr.get_batch(frame_indices)
90
+ frames = frames.permute(0, 3, 1, 2)
91
+
92
+ if padding:
93
+ frames = HD_transform_padding(frames.float(), image_size=resolution, hd_num=hd_num)
94
+ else:
95
+ frames = HD_transform_no_padding(frames.float(), image_size=resolution, hd_num=hd_num)
96
+
97
+ frames = transform(frames)
98
+ T_, C, H, W = frames.shape
99
+
100
+ sub_img = frames.reshape(1, T_, 3, H // resolution, resolution, W // resolution, resolution).permute(0, 3, 5, 1, 2, 4, 6).reshape(-1, T_, 3, resolution, resolution).contiguous()
101
+
102
+ glb_img = F.interpolate(frames.float(), size=(resolution, resolution), mode="bicubic", align_corners=False).to(sub_img.dtype).unsqueeze(0)
103
+
104
+ frames = torch.cat([sub_img, glb_img]).unsqueeze(0)
105
+
106
+ if return_msg:
107
+ fps = float(vr.get_avg_fps())
108
+ sec = ", ".join([str(round(f / fps, 1)) for f in frame_indices])
109
+ # " " should be added in the start and end
110
+ msg = f"The video contains {len(frame_indices)} frames sampled at {sec} seconds."
111
+ return frames, msg
112
+ else:
113
+ return frames
114
+
115
+
116
+ def HD_transform_padding(frames, image_size=224, hd_num=6):
117
+ def _padding_224(frames):
118
+ _, _, H, W = frames.shape
119
+ tar = int(np.ceil(H / 224) * 224)
120
+ top_padding = (tar - H) // 2
121
+ bottom_padding = tar - H - top_padding
122
+ left_padding = 0
123
+ right_padding = 0
124
+
125
+ padded_frames = F.pad(frames, pad=[left_padding, right_padding, top_padding, bottom_padding], mode="constant", value=255)
126
+ return padded_frames
127
+
128
+ _, _, H, W = frames.shape
129
+ trans = False
130
+ if W < H:
131
+ frames = frames.flip(-2, -1)
132
+ trans = True
133
+ width, height = H, W
134
+ else:
135
+ width, height = W, H
136
+
137
+ ratio = width / height
138
+ scale = 1
139
+ while scale * np.ceil(scale / ratio) <= hd_num:
140
+ scale += 1
141
+ scale -= 1
142
+ new_w = int(scale * image_size)
143
+ new_h = int(new_w / ratio)
144
+
145
+ resized_frames = F.interpolate(frames, size=(new_h, new_w), mode="bicubic", align_corners=False)
146
+ padded_frames = _padding_224(resized_frames)
147
+
148
+ if trans:
149
+ padded_frames = padded_frames.flip(-2, -1)
150
+
151
+ return padded_frames
152
+
153
+
154
+ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
155
+ best_ratio_diff = float("inf")
156
+ best_ratio = (1, 1)
157
+ area = width * height
158
+ for ratio in target_ratios:
159
+ target_aspect_ratio = ratio[0] / ratio[1]
160
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
161
+ if ratio_diff < best_ratio_diff:
162
+ best_ratio_diff = ratio_diff
163
+ best_ratio = ratio
164
+ elif ratio_diff == best_ratio_diff:
165
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
166
+ best_ratio = ratio
167
+ return best_ratio
168
+
169
+
170
+ def HD_transform_no_padding(frames, image_size=224, hd_num=6, fix_ratio=(2, 1)):
171
+ min_num = 1
172
+ max_num = hd_num
173
+ _, _, orig_height, orig_width = frames.shape
174
+ aspect_ratio = orig_width / orig_height
175
+
176
+ # calculate the existing video aspect ratio
177
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
178
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
179
+
180
+ # find the closest aspect ratio to the target
181
+ if fix_ratio:
182
+ target_aspect_ratio = fix_ratio
183
+ else:
184
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
185
+
186
+ # calculate the target width and height
187
+ target_width = image_size * target_aspect_ratio[0]
188
+ target_height = image_size * target_aspect_ratio[1]
189
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
190
+
191
+ # resize the frames
192
+ resized_frame = F.interpolate(frames, size=(target_height, target_width), mode="bicubic", align_corners=False)
193
+ return resized_frame
194
+
195
+
196
+ @register_model("InternVideo2")
197
+ class InternVideo2(lmms):
198
+ def __init__(
199
+ self,
200
+ pretrained: str = "OpenGVLab/InternVideo2_chat_8B_HD",
201
+ modality: str = "video",
202
+ device: str = "cuda:0",
203
+ device_map: str = "cuda:0",
204
+ batch_size: str = "1",
205
+ num_segments: str = "8",
206
+ hd_num: str = "6",
207
+ **kwargs,
208
+ ):
209
+ super().__init__()
210
+ self.path = pretrained
211
+ self.instruction = "Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons.\n"
212
+
213
+ self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True, use_fast=False)
214
+ self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, trust_remote_code=True).eval().cuda()
215
+ batch_size = int(batch_size)
216
+ self.num_segments = int(num_segments)
217
+ self.hd_num = int(hd_num)
218
+ assert batch_size == 1, f"Batch size should be 1 for InternVideo2, but got {batch_size}."
219
+ self.batch_size_per_gpu = batch_size
220
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
221
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
222
+ self.accelerator = accelerator
223
+ if accelerator.num_processes > 1:
224
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
225
+ self.device_map = f"cuda:{accelerator.local_process_index}"
226
+ elif accelerator.num_processes == 1 and device_map == "auto":
227
+ self._device = torch.device(device)
228
+ self.device_map = device_map
229
+ else:
230
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
231
+ self.device_map = f"cuda:{accelerator.local_process_index}"
232
+
233
+ if accelerator.num_processes > 1:
234
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
235
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
236
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
237
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
238
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
239
+ kwargs = {
240
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
241
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
242
+ }
243
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
244
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
245
+
246
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
247
+ self._model = accelerator.prepare(self.model)
248
+ else:
249
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
250
+ self.accelerator = accelerator
251
+ if self.accelerator.is_local_main_process:
252
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
253
+ self._rank = self.accelerator.local_process_index
254
+ self._world_size = self.accelerator.num_processes
255
+ elif accelerator.num_processes == 1 and device_map == "auto":
256
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
257
+ self._rank = 0
258
+ self._world_size = 1
259
+ else:
260
+ eval_logger.info(f"Using single device: {self._device}")
261
+ self.model.to(self._device)
262
+ self._rank = 0
263
+ self._world_size = 1
264
+
265
+ self.modality = modality
266
+
267
+ @property
268
+ def config(self):
269
+ # return the associated transformers.AutoConfig for the given pretrained model.
270
+ return self._config
271
+
272
+ @property
273
+ def tokenizer(self):
274
+ return self._tokenizer
275
+
276
+ @property
277
+ def model(self):
278
+ # returns the model, unwrapping it if using Accelerate
279
+ if hasattr(self, "accelerator"):
280
+ return self.accelerator.unwrap_model(self._model)
281
+ else:
282
+ return self._model
283
+
284
+ @property
285
+ def batch_size(self):
286
+ return self.batch_size_per_gpu
287
+
288
+ @property
289
+ def device(self):
290
+ return self._device
291
+
292
+ @property
293
+ def rank(self):
294
+ return self._rank
295
+
296
+ @property
297
+ def world_size(self):
298
+ return self._world_size
299
+
300
+ def flatten(self, input):
301
+ new_list = []
302
+ for i in input:
303
+ for j in i:
304
+ new_list.append(j)
305
+ return new_list
306
+
307
+ def generate_until(self, requests) -> List[str]:
308
+ res = []
309
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
310
+
311
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
312
+ if "until" in gen_kwargs:
313
+ gen_kwargs.pop("until")
314
+ for k, v in DEFAULT_GEN_KWARGS.items():
315
+ if k not in gen_kwargs:
316
+ gen_kwargs[k] = v
317
+
318
+ pop_keys = []
319
+ for k, v in gen_kwargs.items():
320
+ if k not in DEFAULT_GEN_KWARGS:
321
+ pop_keys.append(k)
322
+
323
+ for k in pop_keys:
324
+ gen_kwargs.pop(k)
325
+
326
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
327
+ visuals = self.flatten(visuals)
328
+ if self.modality == "image":
329
+ image_path = visuals[0]
330
+ pixel_values = load_image(image_path, resolution=224, hd_num=self.hd_num)
331
+ pixel_values = pixel_values.to(torch.bfloat16).cuda()
332
+ question = contexts
333
+ response, history = self.model.chat(self.tokenizer, msg="", user_prompt=question, media_type="image", media_tensor=pixel_values, instruction=None, chat_history=[], return_history=True, **gen_kwargs)
334
+ elif self.modality == "video":
335
+ assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos. [META-INFO]{visuals}"
336
+ video_path = visuals[0]
337
+ if "mvbench" in task:
338
+ answer_prompt = "Best Option:("
339
+ else:
340
+ answer_prompt = None
341
+ pixel_values = load_video(video_path, num_segments=self.num_segments, return_msg=False, resolution=224, hd_num=self.hd_num)
342
+ pixel_values = pixel_values.to(torch.bfloat16).cuda()
343
+ question = self.instruction + contexts
344
+ response, history = self.model.chat(
345
+ self.tokenizer,
346
+ msg="",
347
+ user_prompt=question,
348
+ media_type="video",
349
+ media_tensor=pixel_values,
350
+ instruction=self.instruction,
351
+ chat_history=[],
352
+ return_history=True,
353
+ generation_config=gen_kwargs,
354
+ answer_prompt=answer_prompt,
355
+ debug_conv=False,
356
+ )
357
+ res.append(response)
358
+ pbar.update(1)
359
+ pbar.close()
360
+ return res
361
+
362
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
363
+ assert False, "Not implemented yet."
364
+
365
+ def generate_until_multi_round(self, requests) -> List[str]:
366
+ raise NotImplementedError("TODO: Implement multi-round generation for InternVideo2")
LaViDa/eval/lmms_eval/models/internvideo2_5.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ from datetime import timedelta
3
+ from typing import List, Tuple
4
+
5
+ import numpy as np
6
+ import torch
7
+ import torchvision.transforms as T
8
+ from accelerate import Accelerator, DistributedType
9
+ from accelerate.state import AcceleratorState
10
+ from accelerate.utils import InitProcessGroupKwargs
11
+ from decord import VideoReader, cpu
12
+ from PIL import Image
13
+ from torchvision.transforms.functional import InterpolationMode
14
+ from tqdm import tqdm
15
+ from transformers import AutoModel, AutoTokenizer
16
+
17
+ from lmms_eval.api.instance import Instance
18
+ from lmms_eval.api.model import lmms
19
+ from lmms_eval.api.registry import register_model
20
+
21
+ eval_logger = logging.getLogger("eval_logger")
22
+
23
+ IMAGENET_MEAN = (0.485, 0.456, 0.406)
24
+ IMAGENET_STD = (0.229, 0.224, 0.225)
25
+
26
+ DEFAULT_GEN_KWARGS = dict(
27
+ num_beams=1,
28
+ max_new_tokens=1024,
29
+ do_sample=False,
30
+ )
31
+
32
+
33
+ def build_transform(input_size):
34
+ MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
35
+ transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
36
+ return transform
37
+
38
+
39
+ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
40
+ best_ratio_diff = float("inf")
41
+ best_ratio = (1, 1)
42
+ area = width * height
43
+ for ratio in target_ratios:
44
+ target_aspect_ratio = ratio[0] / ratio[1]
45
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
46
+ if ratio_diff < best_ratio_diff:
47
+ best_ratio_diff = ratio_diff
48
+ best_ratio = ratio
49
+ elif ratio_diff == best_ratio_diff:
50
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
51
+ best_ratio = ratio
52
+ return best_ratio
53
+
54
+
55
+ def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
56
+ orig_width, orig_height = image.size
57
+ aspect_ratio = orig_width / orig_height
58
+
59
+ # calculate the existing image aspect ratio
60
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
61
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
62
+
63
+ # find the closest aspect ratio to the target
64
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
65
+
66
+ # calculate the target width and height
67
+ target_width = image_size * target_aspect_ratio[0]
68
+ target_height = image_size * target_aspect_ratio[1]
69
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
70
+
71
+ # resize the image
72
+ resized_img = image.resize((target_width, target_height))
73
+ processed_images = []
74
+ for i in range(blocks):
75
+ box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
76
+ # split the image
77
+ split_img = resized_img.crop(box)
78
+ processed_images.append(split_img)
79
+ assert len(processed_images) == blocks
80
+ if use_thumbnail and len(processed_images) != 1:
81
+ thumbnail_img = image.resize((image_size, image_size))
82
+ processed_images.append(thumbnail_img)
83
+ return processed_images
84
+
85
+
86
+ def load_image(image, input_size=448, max_num=6):
87
+ transform = build_transform(input_size=input_size)
88
+ images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
89
+ pixel_values = [transform(image) for image in images]
90
+ pixel_values = torch.stack(pixel_values)
91
+ return pixel_values
92
+
93
+
94
+ def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
95
+ if bound:
96
+ start, end = bound[0], bound[1]
97
+ else:
98
+ start, end = -100000, 100000
99
+ start_idx = max(first_idx, round(start * fps))
100
+ end_idx = min(round(end * fps), max_frame)
101
+ seg_size = float(end_idx - start_idx) / num_segments
102
+ frame_indices = [int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)]
103
+ return frame_indices
104
+
105
+
106
+ def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32, media_dict=None):
107
+ if type(video_path) == str:
108
+ vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
109
+ else:
110
+ vr = VideoReader(video_path[0], ctx=cpu(0), num_threads=1)
111
+
112
+ max_frame = len(vr) - 1
113
+ fps = float(vr.get_avg_fps())
114
+
115
+ pixel_values_list, num_patches_list = [], []
116
+ transform = build_transform(input_size=input_size)
117
+ frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
118
+ if media_dict is not None and media_dict["video_read_type"] == "decord_last":
119
+ frame_indices = frame_indices + [max_frame for _ in range(4)] # add last 4 frames
120
+
121
+ frame_indices = np.array(frame_indices)
122
+
123
+ for frame_index in frame_indices:
124
+ img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
125
+ img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=False, max_num=max_num)
126
+ pixel_values = [transform(tile) for tile in img]
127
+ pixel_values = torch.stack(pixel_values)
128
+ num_patches_list.append(pixel_values.shape[0])
129
+ pixel_values_list.append(pixel_values)
130
+ pixel_values = torch.cat(pixel_values_list)
131
+ return pixel_values, num_patches_list
132
+
133
+
134
+ @register_model("internvideo2_5")
135
+ class InternVideo2_5(lmms):
136
+ def __init__(
137
+ self,
138
+ pretrained: str = "OpenGVLab/InternVideo2_5_Chat_8B",
139
+ modality: str = "video",
140
+ device: str = "cuda:0",
141
+ device_map: str = "cuda:0",
142
+ batch_size: str = "1",
143
+ max_frames_num: int = 32,
144
+ **kwargs,
145
+ ):
146
+ super().__init__()
147
+ self.max_frames_num = max_frames_num
148
+ self.path = pretrained
149
+ self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, use_flash_attn=True, trust_remote_code=True).eval().cuda()
150
+ self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True)
151
+
152
+ batch_size = int(batch_size)
153
+ assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}."
154
+ self.batch_size_per_gpu = batch_size
155
+
156
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
157
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
158
+ self.accelerator = accelerator
159
+ if accelerator.num_processes > 1:
160
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
161
+ self.device_map = f"cuda:{accelerator.local_process_index}"
162
+ elif accelerator.num_processes == 1 and device_map == "auto":
163
+ self._device = torch.device(device)
164
+ self.device_map = device_map
165
+ else:
166
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
167
+ self.device_map = f"cuda:{accelerator.local_process_index}"
168
+
169
+ if accelerator.num_processes > 1:
170
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
171
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
172
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
173
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
174
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
175
+ kwargs = {
176
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
177
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
178
+ }
179
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
180
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
181
+
182
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
183
+ self._model = accelerator.prepare(self.model)
184
+ else:
185
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
186
+ self.accelerator = accelerator
187
+ if self.accelerator.is_local_main_process:
188
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
189
+ self._rank = self.accelerator.local_process_index
190
+ self._world_size = self.accelerator.num_processes
191
+ elif accelerator.num_processes == 1 and device_map == "auto":
192
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
193
+ self._rank = 0
194
+ self._word_size = 1
195
+ else:
196
+ eval_logger.info(f"Using single device: {self._device}")
197
+ self.model.to(self._device)
198
+ self._rank = 0
199
+ self._world_size = 1
200
+
201
+ self.modality = modality
202
+
203
+ @property
204
+ def config(self):
205
+ # return the associated transformers.AutoConfig for the given pretrained model.
206
+ return self._config
207
+
208
+ @property
209
+ def tokenizer(self):
210
+ return self._tokenizer
211
+
212
+ @property
213
+ def model(self):
214
+ # returns the model, unwrapping it if using Accelerate
215
+ if hasattr(self, "accelerator"):
216
+ return self.accelerator.unwrap_model(self._model)
217
+ else:
218
+ return self._model
219
+
220
+ @property
221
+ def batch_size(self):
222
+ return self.batch_size_per_gpu
223
+
224
+ @property
225
+ def device(self):
226
+ return self._device
227
+
228
+ @property
229
+ def rank(self):
230
+ return self._rank
231
+
232
+ @property
233
+ def world_size(self):
234
+ return self._world_size
235
+
236
+ def flatten(self, input):
237
+ new_list = []
238
+ for i in input:
239
+ for j in i:
240
+ new_list.append(j)
241
+ return new_list
242
+
243
+ def generate_until(self, requests) -> List[str]:
244
+ res = []
245
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
246
+
247
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
248
+ if "until" in gen_kwargs:
249
+ gen_kwargs.pop("until")
250
+ for k, v in DEFAULT_GEN_KWARGS.items():
251
+ if k not in gen_kwargs:
252
+ gen_kwargs[k] = v
253
+
254
+ pop_keys = []
255
+ for k, v in gen_kwargs.items():
256
+ if k not in DEFAULT_GEN_KWARGS:
257
+ pop_keys.append(k)
258
+
259
+ for k in pop_keys:
260
+ gen_kwargs.pop(k)
261
+
262
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
263
+ visuals = self.flatten(visuals)
264
+ if self.modality == "image":
265
+ if visuals:
266
+ visuals = [load_image(visual).to(torch.bfloat16).cuda() for visual in visuals]
267
+ pixel_values = torch.cat(visuals, dim=0)
268
+ num_patches_list = [visual.size(0) for visual in visuals]
269
+ image_tokens = ["<image>"] * len(visuals)
270
+ image_tokens = " ".join(image_tokens)
271
+ contexts = image_tokens + "\n" + contexts
272
+ else:
273
+ pixel_values = None
274
+ num_patch_list = None
275
+ response, history = self.model.chat(self.tokenizer, pixel_values, contexts, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
276
+ elif self.modality == "video":
277
+ # assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos. {visuals}"
278
+ video_path = visuals[0]
279
+ if len(visuals) > 1:
280
+ assert len(visuals) == 2, visuals
281
+ media_dict = visuals[1]
282
+ else:
283
+ media_dict = {"video_read_type": "decord"}
284
+ pixel_values, num_patches_list = load_video(video_path, num_segments=self.max_frames_num, max_num=1, media_dict=media_dict)
285
+ pixel_values = pixel_values.to(torch.bfloat16).cuda()
286
+ video_prefix = "".join([f"Frame{i+1}: <image>\n" for i in range(len(num_patches_list))])
287
+ question = video_prefix + contexts
288
+ response, history = self.model.chat(self.tokenizer, pixel_values, question, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
289
+ res.append(response)
290
+ pbar.update(1)
291
+ pbar.close()
292
+ return res
293
+
294
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
295
+ assert False, "Not implemented yet."
LaViDa/eval/lmms_eval/models/internvl.py ADDED
@@ -0,0 +1,496 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+ import subprocess
4
+ from datetime import timedelta
5
+ from pathlib import Path
6
+ from typing import List, Optional, Tuple, Union
7
+
8
+ import numpy as np
9
+ import requests
10
+ import torch
11
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
12
+ from accelerate.state import AcceleratorState
13
+ from huggingface_hub import snapshot_download
14
+ from PIL import Image
15
+ from tqdm import tqdm
16
+ from transformers import AutoConfig
17
+
18
+ from lmms_eval import utils
19
+ from lmms_eval.api.instance import Instance
20
+ from lmms_eval.api.model import lmms
21
+ from lmms_eval.api.registry import register_model
22
+ from lmms_eval.utils import stop_sequences_criteria
23
+
24
+ wd = Path(__file__).parent.parent.parent.resolve()
25
+ import sys
26
+
27
+ sys.path.append(os.path.join(str(wd), "InternVL", "internvl_chat"))
28
+ from loguru import logger as eval_logger
29
+
30
+ if not hasattr(eval_logger, "internvl_warning_logged"):
31
+ eval_logger.internvl_warning_logged = False
32
+
33
+ try:
34
+ from internvl.model.internlm2.modeling_internlm2 import InternLM2ForCausalLM
35
+ from internvl.model.internvl_chat import InternVLChatModel
36
+ from internvl.model.internvl_chat.configuration_internvl_chat import (
37
+ InternVLChatConfig,
38
+ )
39
+ from internvl.model.internvl_chat.modeling_intern_vit import InternVisionModel
40
+ from internvl.train.dataset import build_transform, dynamic_preprocess
41
+ except ImportError:
42
+ eval_logger.debug("InternVL is not installed. Please install InternVL to use this model.")
43
+ if not eval_logger.internvl_warning_logged:
44
+ eval_logger.debug("InternVL is not installed. Please install InternVL to use this model.")
45
+ eval_logger.internvl_warning_logged = True
46
+
47
+ import re
48
+ import warnings
49
+ from typing import Any, List, Optional, Tuple, Union
50
+
51
+ import torch.utils.checkpoint
52
+ from huggingface_hub import snapshot_download
53
+ from peft import LoraConfig, get_peft_model
54
+ from torch import nn
55
+ from torch.nn import CrossEntropyLoss
56
+ from transformers import (
57
+ AutoModel,
58
+ AutoTokenizer,
59
+ GenerationConfig,
60
+ LlamaForCausalLM,
61
+ LlamaTokenizer,
62
+ )
63
+ from transformers.modeling_outputs import CausalLMOutputWithPast
64
+ from transformers.modeling_utils import PreTrainedModel
65
+
66
+
67
+ @register_model("internvl")
68
+ class InternVLChat(lmms):
69
+ # config_class = InternVLChatConfig
70
+ main_input_name = "pixel_values"
71
+ _no_split_modules = ["InternVisionEncoderLayer", "LlamaDecoderLayer"]
72
+
73
+ """
74
+ 0. Install lmms-eval
75
+ cd lmms-eval
76
+ pip install -e .
77
+
78
+ How to Install InternVL:
79
+ 1. Clone the InternVL repository:
80
+ git clone https://github.com/OpenGVLab/InternVL.git
81
+
82
+ 2. Install the requirements:
83
+ pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu118
84
+
85
+ 3. Install flash-attn==2.3.6:
86
+ pip install flash-attn==2.3.6 --no-build-isolation
87
+ """
88
+
89
+ """
90
+ How to download the pretrained model:
91
+ 1. Download the pretrained model from hugginface:
92
+ cd pretrained/
93
+ # pip install -U huggingface_hub
94
+ huggingface-cli download --resume-download --local-dir-use-symlinks False OpenGVLab/InternVL-Chat-V1-5 --local-dir InternVL-Chat-V1-5
95
+
96
+ 2. the pretrained model should be in the following directory:
97
+ pretrained
98
+ └── InternVL-Chat-V1-5
99
+ """
100
+
101
+ #
102
+ # The above steps can be optional, I add snapshot download, so now can just use hf repo_id
103
+ # model_args pretrained=OpenGVLab/InternVL-Chat-V1-5
104
+ #
105
+
106
+ """
107
+ InternVL-Chat-V1-5 Model for OpenGVLab https://github.com/OpenGVLab/InternVL/blob/main/internvl_chat/internvl/model/internvl_chat/modeling_internvl_chat.py
108
+ Example usage:
109
+
110
+ accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
111
+ --model internvl \
112
+ --model_args pretrained=OpenGVLab/InternVL-Chat-V1-5 \
113
+ --tasks llava_wilder_small \
114
+ --batch_size 1 \
115
+ --output_path ./logs/ \
116
+ --log_samples
117
+ """
118
+
119
+ def __init__(
120
+ self,
121
+ config=None,
122
+ pretrained: str = "OpenGVLab/InternVL-Chat-V1-5",
123
+ truncation: Optional[bool] = True,
124
+ device: Optional[str] = "cuda:0",
125
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
126
+ batch_size: Optional[Union[int, str]] = 1,
127
+ trust_remote_code: Optional[bool] = False,
128
+ revision=None,
129
+ device_map="cuda:0",
130
+ conv_template="vicuna_v1",
131
+ use_cache=True,
132
+ truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
133
+ customized_config=None, # ends in json
134
+ dynamic=True,
135
+ load_in_8bit=False,
136
+ vision_model=None,
137
+ language_model=None,
138
+ max_num=12,
139
+ **kwargs,
140
+ ) -> None:
141
+ super().__init__()
142
+
143
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
144
+
145
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
146
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
147
+ if accelerator.num_processes > 1:
148
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
149
+ self.device_map = f"cuda:{accelerator.local_process_index}"
150
+ elif accelerator.num_processes == 1 and device_map == "auto":
151
+ self._device = torch.device(device)
152
+ self.device_map = device_map
153
+ else:
154
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
155
+ self.device_map = f"cuda:{accelerator.local_process_index}"
156
+
157
+ self.dynamic = dynamic # dynamic image_size
158
+ self.max_num = max_num
159
+ if accelerator.is_main_process:
160
+ cache_dir = snapshot_download(repo_id=pretrained, cache_dir="cache_dir", local_dir="cache_dir", local_dir_use_symlinks=False)
161
+ accelerator.wait_for_everyone()
162
+ # So what I did is that I let main process to download the repo, and then
163
+ # other process can just simply read from this repo
164
+ cache_dir = snapshot_download(repo_id=pretrained, cache_dir="cache_dir", local_dir="cache_dir", local_dir_use_symlinks=False)
165
+ config = InternVLChatConfig.from_pretrained(cache_dir)
166
+ tokenizer = AutoTokenizer.from_pretrained(cache_dir, trust_remote_code=True, use_fast=False)
167
+ model = InternVLChatModel.from_pretrained(cache_dir, low_cpu_mem_usage=True, config=config, torch_dtype=torch.bfloat16, load_in_8bit=load_in_8bit).eval()
168
+ if not load_in_8bit:
169
+ model = model.cuda()
170
+ # self.model=model
171
+ # self.device=self._device
172
+ self._tokenizer = tokenizer
173
+ # self.tokenizer=tokenizer
174
+ self._model = model
175
+ self._config = self._model.config
176
+ self.use_thumbnail = self.model.config.use_thumbnail
177
+ self.model.eval()
178
+ self.model.tie_weights()
179
+ self.truncation = truncation
180
+ self.batch_size_per_gpu = int(batch_size)
181
+ self.conv_template = conv_template
182
+ self.use_cache = use_cache
183
+ self.truncate_context = truncate_context
184
+ if accelerator.num_processes > 1:
185
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
186
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
187
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
188
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
189
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
190
+ kwargs = {
191
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
192
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
193
+ }
194
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
195
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
196
+
197
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
198
+ self._model = accelerator.prepare(self.model)
199
+ else:
200
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
201
+ self.accelerator = accelerator
202
+ if self.accelerator.is_local_main_process:
203
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
204
+ self._rank = self.accelerator.local_process_index
205
+ self._world_size = self.accelerator.num_processes
206
+ elif accelerator.num_processes == 1 and device_map == "auto":
207
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
208
+ self._rank = 0
209
+ self._world_size = 1
210
+ else:
211
+ eval_logger.info(f"Using single device: {self._device}")
212
+ self.model.to(self._device)
213
+ self._rank = 0
214
+ self._world_size = 1
215
+
216
+ # from internvl model
217
+
218
+ self.image_size = config.force_image_size or config.vision_config.image_size
219
+
220
+ def wrap_backbone_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):
221
+ lora_config = LoraConfig(
222
+ r=r,
223
+ target_modules=["attn.qkv", "attn.proj", "mlp.fc1", "mlp.fc2"],
224
+ lora_alpha=lora_alpha,
225
+ lora_dropout=lora_dropout,
226
+ )
227
+ self.vision_model = get_peft_model(self.vision_model, lora_config)
228
+ self.vision_model.print_trainable_parameters()
229
+
230
+ def wrap_llm_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):
231
+ lora_config = LoraConfig(
232
+ r=r, target_modules=["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.down_proj", "mlp.up_proj"], lora_alpha=lora_alpha, lora_dropout=lora_dropout, task_type="CAUSAL_LM"
233
+ )
234
+ self.language_model = get_peft_model(self.language_model, lora_config)
235
+ self.language_model.enable_input_require_grads()
236
+ self.language_model.print_trainable_parameters()
237
+
238
+ def pixel_shuffle(self, x, scale_factor=0.5):
239
+ n, w, h, c = x.size()
240
+ # N, W, H, C --> N, W, H * scale, C // scale
241
+ x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
242
+ # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
243
+ x = x.permute(0, 2, 1, 3).contiguous()
244
+ # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
245
+ x = x.view(n, int(h * scale_factor), int(w * scale_factor), int(c / (scale_factor * scale_factor)))
246
+ if self.ps_version == "v1":
247
+ warnings.warn("In ps_version 'v1', the height and width have not been swapped back, " "which results in a transposed image.")
248
+ else:
249
+ x = x.permute(0, 2, 1, 3).contiguous()
250
+ return x
251
+
252
+ def noised_embed(self, vit_embeds, noise_alpha=5):
253
+ dims = torch.tensor(vit_embeds.size(1) * vit_embeds.size(2))
254
+ mag_norm = noise_alpha / torch.sqrt(dims)
255
+ noise = torch.zeros_like(vit_embeds).uniform_(-mag_norm, mag_norm)
256
+ return vit_embeds + noise
257
+
258
+ def extract_feature(self, pixel_values):
259
+ if self.select_layer == -1:
260
+ vit_embeds = self.vision_model(pixel_values=pixel_values, output_hidden_states=False, return_dict=True).last_hidden_state
261
+ else:
262
+ vit_embeds = self.vision_model(pixel_values=pixel_values, output_hidden_states=True, return_dict=True).hidden_states[self.select_layer]
263
+ vit_embeds = vit_embeds[:, 1:, :]
264
+
265
+ if self.training and self.neftune_alpha is not None:
266
+ vit_embeds = self.noised_embed(vit_embeds, self.neftune_alpha)
267
+
268
+ h = w = int(vit_embeds.shape[1] ** 0.5)
269
+ vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
270
+ vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
271
+ vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
272
+ vit_embeds = self.mlp1(vit_embeds) # .to(pixel_values.device)
273
+ return vit_embeds
274
+
275
+ def multi_image_chat(self, tokenizer, pixel_values, image_counts, question, generation_config, history=None, return_history=False, IMG_START_TOKEN="<img>", IMG_END_TOKEN="</img>", IMG_CONTEXT_TOKEN="<IMG_CONTEXT>"):
276
+ img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
277
+ self.img_context_token_id = img_context_token_id
278
+ if tokenizer.convert_tokens_to_ids("<|im_end|>") != 0:
279
+ eos_token_id = tokenizer.convert_tokens_to_ids("<|im_end|>") # 92542, InternLM2
280
+ else:
281
+ eos_token_id = tokenizer.eos_token_id
282
+
283
+ from internvl.conversation import get_conv_template
284
+
285
+ template = get_conv_template(self.template)
286
+
287
+ if history is None:
288
+ history = []
289
+ image_tokens = ""
290
+ image_bs = pixel_values.shape[0]
291
+ # print(f"dynamic ViT batch size: {image_bs}, image_counts: {image_counts}")
292
+ for idx, image_count in enumerate(image_counts):
293
+ image_tokens += f"<image {idx+1}> (图{idx+1}):" + IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * image_count + IMG_END_TOKEN
294
+ question = image_tokens + "\n" + question
295
+ else:
296
+ for old_question, old_answer in history:
297
+ template.append_message(template.roles[0], old_question)
298
+ template.append_message(template.roles[1], old_answer)
299
+ template.append_message(template.roles[0], question)
300
+ template.append_message(template.roles[1], None)
301
+ query = template.get_prompt()
302
+ model_inputs = tokenizer(query, return_tensors="pt")
303
+ input_ids = model_inputs["input_ids"].cuda()
304
+ attention_mask = model_inputs["attention_mask"].cuda()
305
+ generation_config["eos_token_id"] = eos_token_id
306
+
307
+ generation_output = self.generate(pixel_values=pixel_values, input_ids=input_ids, attention_mask=attention_mask, **generation_config)
308
+ response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
309
+ response = response.split("<|im_end|>")[0].strip() # for InternLM2
310
+ history.append((question, response))
311
+ if return_history:
312
+ return response, history
313
+ else:
314
+ query_to_print = query.replace(image_tokens, "<image>")
315
+ # print(query_to_print, response)
316
+ return response
317
+ return response
318
+
319
+ @property
320
+ def tokenizer(self):
321
+ return self._tokenizer
322
+
323
+ @property
324
+ def model(self):
325
+ # returns the model, unwrapping it if using Accelerate
326
+ if hasattr(self, "accelerator"):
327
+ return self.accelerator.unwrap_model(self._model)
328
+ else:
329
+ return self._model
330
+
331
+ @property
332
+ def batch_size(self):
333
+ return self.batch_size_per_gpu
334
+
335
+ @property
336
+ def device(self):
337
+ return self._device
338
+
339
+ @property
340
+ def rank(self):
341
+ return self._rank
342
+
343
+ @property
344
+ def world_size(self):
345
+ return self._world_size
346
+
347
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
348
+ """ """
349
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
350
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
351
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
352
+ if left_truncate_len:
353
+ encoding = encoding[-left_truncate_len:]
354
+ return encoding
355
+
356
+ def tok_decode(self, tokens):
357
+ try:
358
+ return self.tokenizer.decode(tokens)
359
+ except:
360
+ return self.tokenizer.decode([tokens])
361
+
362
+ def post_processing(self, response):
363
+ response = response.replace("\n", "").replace("不是", "No").replace("是", "Yes").replace("否", "No")
364
+ response = response.lower().replace("true", "yes").replace("false", "no")
365
+ pattern = re.compile(r"[\u4e00-\u9fa5]")
366
+ response = re.sub(pattern, "", response)
367
+ return response
368
+
369
+ @torch.no_grad()
370
+ def generate(
371
+ self,
372
+ pixel_values: Optional[torch.FloatTensor] = None,
373
+ input_ids: Optional[torch.FloatTensor] = None,
374
+ attention_mask: Optional[torch.LongTensor] = None,
375
+ visual_features: Optional[torch.FloatTensor] = None,
376
+ generation_config: Optional[GenerationConfig] = None,
377
+ output_hidden_states: Optional[bool] = None,
378
+ return_dict: Optional[bool] = None,
379
+ **generate_kwargs,
380
+ ) -> torch.LongTensor:
381
+ assert self.img_context_token_id is not None
382
+ if pixel_values is not None:
383
+ if visual_features is not None:
384
+ vit_embeds = visual_features
385
+ else:
386
+ vit_embeds = self.extract_feature(pixel_values)
387
+
388
+ input_embeds = self.language_model.get_input_embeddings()(input_ids)
389
+ B, N, C = input_embeds.shape
390
+ input_embeds = input_embeds.reshape(B * N, C)
391
+
392
+ input_ids = input_ids.reshape(B * N)
393
+ selected = input_ids == self.img_context_token_id
394
+ assert selected.sum() != 0
395
+ input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
396
+
397
+ input_embeds = input_embeds.reshape(B, N, C)
398
+ else:
399
+ input_embeds = self.language_model.get_input_embeddings()(input_ids)
400
+
401
+ outputs = self.language_model.generate(
402
+ inputs_embeds=input_embeds,
403
+ attention_mask=attention_mask,
404
+ generation_config=generation_config,
405
+ output_hidden_states=output_hidden_states,
406
+ return_dict=return_dict,
407
+ use_cache=True,
408
+ **generate_kwargs,
409
+ )
410
+
411
+ return outputs
412
+
413
+ def flatten(self, input):
414
+ new_list = []
415
+ for i in input:
416
+ for j in i:
417
+ new_list.append(j)
418
+ return new_list
419
+
420
+ def load_image(self, flattened_visuals, input_size=224):
421
+ assert flattened_visuals[0].mode == "RGB"
422
+ image = flattened_visuals[0].convert("RGB")
423
+ transform = build_transform(is_train=False, input_size=input_size)
424
+ if self.dynamic:
425
+ images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=self.use_thumbnail, max_num=self.max_num)
426
+ else:
427
+ images = [image]
428
+ pixel_values = [transform(image) for image in images]
429
+ pixel_values = torch.stack(pixel_values)
430
+ return pixel_values
431
+
432
+ def generate_until(self, requests: List[Instance]) -> List[str]:
433
+ res = []
434
+
435
+ def _collate(x):
436
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
437
+ # - time estimates will always be over not underestimates, which is more useful for planning
438
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
439
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
440
+ # automatic adaptive batches much much easier to implement
441
+ # - any OOMs will happen right away rather than near the end
442
+ toks = self.tok_encode(x[0])
443
+ return -len(toks), x[0]
444
+
445
+ # we group requests by their generation_kwargs,
446
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
447
+ # in the same batch.
448
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
449
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
450
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
451
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
452
+ for chunk in chunks:
453
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
454
+ task = task[0]
455
+ split = split[0]
456
+ batched_visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # [B, N]
457
+ flattened_visuals = self.flatten(batched_visuals)
458
+ try:
459
+ pixel_values = self.load_image(flattened_visuals, self.image_size).cuda().to(torch.bfloat16)
460
+ except IndexError:
461
+ pixel_values = None
462
+ gen_kwargs = all_gen_kwargs[0]
463
+
464
+ if "max_new_tokens" not in gen_kwargs:
465
+ gen_kwargs["max_new_tokens"] = 1024
466
+ if "temperature" not in gen_kwargs:
467
+ gen_kwargs["temperature"] = 0
468
+ if "top_p" not in gen_kwargs:
469
+ gen_kwargs["top_p"] = None
470
+ if "num_beams" not in gen_kwargs:
471
+ gen_kwargs["num_beams"] = 1
472
+
473
+ generation_config = dict(
474
+ do_sample=False,
475
+ top_k=50,
476
+ top_p=gen_kwargs["top_p"],
477
+ num_beams=gen_kwargs["num_beams"],
478
+ max_new_tokens=gen_kwargs["max_new_tokens"],
479
+ eos_token_id=self.tokenizer.eos_token_id,
480
+ )
481
+ question = contexts[0]
482
+ response = self.model.chat(tokenizer=self.tokenizer, pixel_values=pixel_values, question=question, generation_config=generation_config)
483
+ # TODO(choiszt) try batch_chat for multiple inputs
484
+ response = self.post_processing(response)
485
+ res.append(response)
486
+ self.cache_hook.add_partial("generate_until", (question, gen_kwargs), response)
487
+ pbar.update(1)
488
+ res = re_ords.get_original(res)
489
+ return res
490
+ # print(chunk)
491
+
492
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
493
+ pass
494
+
495
+ def generate_until_multi_round(self, requests) -> List[str]:
496
+ raise NotImplementedError("TODO: Implement multi-round generation for InternVL")
LaViDa/eval/lmms_eval/models/internvl2.py ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ from typing import List, Tuple
3
+
4
+ import numpy as np
5
+ import torch
6
+ import torchvision.transforms as T
7
+ from accelerate import Accelerator, DistributedType
8
+ from decord import VideoReader, cpu
9
+ from PIL import Image
10
+ from torchvision.transforms.functional import InterpolationMode
11
+ from tqdm import tqdm
12
+ from transformers import AutoModel, AutoTokenizer
13
+
14
+ from lmms_eval.api.instance import Instance
15
+ from lmms_eval.api.model import lmms
16
+ from lmms_eval.api.registry import register_model
17
+
18
+ eval_logger = logging.getLogger("eval_logger")
19
+
20
+ IMAGENET_MEAN = (0.485, 0.456, 0.406)
21
+ IMAGENET_STD = (0.229, 0.224, 0.225)
22
+
23
+ DEFAULT_GEN_KWARGS = dict(
24
+ num_beams=1,
25
+ max_new_tokens=1024,
26
+ do_sample=False,
27
+ )
28
+
29
+
30
+ def build_transform(input_size):
31
+ MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
32
+ transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
33
+ return transform
34
+
35
+
36
+ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
37
+ best_ratio_diff = float("inf")
38
+ best_ratio = (1, 1)
39
+ area = width * height
40
+ for ratio in target_ratios:
41
+ target_aspect_ratio = ratio[0] / ratio[1]
42
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
43
+ if ratio_diff < best_ratio_diff:
44
+ best_ratio_diff = ratio_diff
45
+ best_ratio = ratio
46
+ elif ratio_diff == best_ratio_diff:
47
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
48
+ best_ratio = ratio
49
+ return best_ratio
50
+
51
+
52
+ def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
53
+ orig_width, orig_height = image.size
54
+ aspect_ratio = orig_width / orig_height
55
+
56
+ # calculate the existing image aspect ratio
57
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
58
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
59
+
60
+ # find the closest aspect ratio to the target
61
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
62
+
63
+ # calculate the target width and height
64
+ target_width = image_size * target_aspect_ratio[0]
65
+ target_height = image_size * target_aspect_ratio[1]
66
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
67
+
68
+ # resize the image
69
+ resized_img = image.resize((target_width, target_height))
70
+ processed_images = []
71
+ for i in range(blocks):
72
+ box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
73
+ # split the image
74
+ split_img = resized_img.crop(box)
75
+ processed_images.append(split_img)
76
+ assert len(processed_images) == blocks
77
+ if use_thumbnail and len(processed_images) != 1:
78
+ thumbnail_img = image.resize((image_size, image_size))
79
+ processed_images.append(thumbnail_img)
80
+ return processed_images
81
+
82
+
83
+ def load_image(image, input_size=448, max_num=6):
84
+ transform = build_transform(input_size=input_size)
85
+ images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
86
+ pixel_values = [transform(image) for image in images]
87
+ pixel_values = torch.stack(pixel_values)
88
+ return pixel_values
89
+
90
+
91
+ def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
92
+ if bound:
93
+ start, end = bound[0], bound[1]
94
+ else:
95
+ start, end = -100000, 100000
96
+ start_idx = max(first_idx, round(start * fps))
97
+ end_idx = min(round(end * fps), max_frame)
98
+ seg_size = float(end_idx - start_idx) / num_segments
99
+ frame_indices = np.array([int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)])
100
+ return frame_indices
101
+
102
+
103
+ def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32):
104
+ vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
105
+ max_frame = len(vr) - 1
106
+ fps = float(vr.get_avg_fps())
107
+
108
+ pixel_values_list, num_patches_list = [], []
109
+ transform = build_transform(input_size=input_size)
110
+ frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
111
+ for frame_index in frame_indices:
112
+ img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
113
+ img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=True, max_num=max_num)
114
+ pixel_values = [transform(tile) for tile in img]
115
+ pixel_values = torch.stack(pixel_values)
116
+ num_patches_list.append(pixel_values.shape[0])
117
+ pixel_values_list.append(pixel_values)
118
+ pixel_values = torch.cat(pixel_values_list)
119
+ return pixel_values, num_patches_list
120
+
121
+
122
+ import math
123
+ from datetime import timedelta
124
+
125
+ from accelerate.state import AcceleratorState
126
+ from accelerate.utils import InitProcessGroupKwargs
127
+
128
+
129
+ # The reason for writing the code this way is to avoid errors that occur during multi-GPU inference due to tensors not being on the same device. By ensuring that the first and last layers of the large language model (LLM) are on the same device, we prevent such errors.
130
+ def split_model(model_name, num_layers=None):
131
+ device_map = {}
132
+ world_size = torch.cuda.device_count()
133
+ if num_layers is None:
134
+ num_layers = {
135
+ "InternVL2_5-1B": 24,
136
+ "InternVL2_5-2B": 24,
137
+ "InternVL2_5-4B": 36,
138
+ "InternVL2_5-8B": 32,
139
+ "InternVL2_5-26B": 48,
140
+ "InternVL2_5-38B": 64,
141
+ "InternVL2_5-78B": 80,
142
+ "InternVL2-1B": 24,
143
+ "InternVL2-2B": 24,
144
+ "InternVL2-4B": 32,
145
+ "InternVL2-8B": 32,
146
+ "InternVL2-26B": 48,
147
+ "InternVL2-40B": 60,
148
+ "InternVL2-Llama3-76B": 80,
149
+ }[model_name]
150
+ # Since the first GPU will be used for ViT, treat it as half a GPU.
151
+ num_layers_per_gpu = math.ceil(num_layers / (world_size - 0.5))
152
+ num_layers_per_gpu = [num_layers_per_gpu] * world_size
153
+ num_layers_per_gpu[0] = math.ceil(num_layers_per_gpu[0] * 0.5)
154
+ layer_cnt = 0
155
+ for i, num_layer in enumerate(num_layers_per_gpu):
156
+ for j in range(num_layer):
157
+ device_map[f"language_model.model.layers.{layer_cnt}"] = i
158
+ layer_cnt += 1
159
+ device_map["vision_model"] = 0
160
+ device_map["mlp1"] = 0
161
+ device_map["language_model.model.tok_embeddings"] = 0
162
+ device_map["language_model.model.embed_tokens"] = 0
163
+ device_map["language_model.output"] = 0
164
+ device_map["language_model.model.norm"] = 0
165
+ device_map["language_model.lm_head"] = 0
166
+ device_map[f"language_model.model.layers.{num_layers - 1}"] = 0
167
+
168
+ return device_map
169
+
170
+
171
+ @register_model("internvl2")
172
+ class InternVL2(lmms):
173
+ def __init__(
174
+ self,
175
+ pretrained: str = "OpenGVLab/InternVL2-2B",
176
+ modality: str = "image",
177
+ device: str = "cuda:0",
178
+ device_map: str = "cuda:0",
179
+ batch_size: str = "1",
180
+ num_frame: int = 32,
181
+ num_layers=None,
182
+ **kwargs,
183
+ ):
184
+ super().__init__()
185
+
186
+ self.path = pretrained
187
+ self.num_frame = num_frame
188
+
189
+ batch_size = int(batch_size)
190
+ assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}."
191
+ self.batch_size_per_gpu = batch_size
192
+
193
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
194
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
195
+ self.accelerator = accelerator
196
+ if accelerator.num_processes > 1:
197
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
198
+ self.device_map = f"cuda:{accelerator.local_process_index}"
199
+ elif accelerator.num_processes == 1 and device_map == "auto":
200
+ self._device = torch.device(device)
201
+ device_map = split_model(pretrained.split("/")[-1], num_layers=num_layers)
202
+ self.device_map = device_map
203
+ else:
204
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
205
+ self.device_map = f"cuda:{accelerator.local_process_index}"
206
+
207
+ self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True, device_map=self.device_map).eval()
208
+ self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True, device_map=self.device_map)
209
+
210
+ if accelerator.num_processes > 1:
211
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
212
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
213
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
214
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
215
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
216
+ kwargs = {
217
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
218
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
219
+ }
220
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
221
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
222
+
223
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
224
+ self._model = accelerator.prepare(self.model)
225
+ else:
226
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
227
+ self.accelerator = accelerator
228
+ if self.accelerator.is_local_main_process:
229
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
230
+ self._rank = self.accelerator.local_process_index
231
+ self._world_size = self.accelerator.num_processes
232
+ elif accelerator.num_processes == 1 and device_map == "auto":
233
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
234
+ self._rank = 0
235
+ self._world_size = 1
236
+ else:
237
+ eval_logger.info(f"Using single device: {self._device}")
238
+ self.model.to(self._device)
239
+ self._rank = 0
240
+ self._world_size = 1
241
+
242
+ self.modality = modality
243
+
244
+ @property
245
+ def config(self):
246
+ # return the associated transformers.AutoConfig for the given pretrained model.
247
+ return self._config
248
+
249
+ @property
250
+ def tokenizer(self):
251
+ return self._tokenizer
252
+
253
+ @property
254
+ def model(self):
255
+ # returns the model, unwrapping it if using Accelerate
256
+ if hasattr(self, "accelerator"):
257
+ return self.accelerator.unwrap_model(self._model)
258
+ else:
259
+ return self._model
260
+
261
+ @property
262
+ def batch_size(self):
263
+ return self.batch_size_per_gpu
264
+
265
+ @property
266
+ def device(self):
267
+ return self._device
268
+
269
+ @property
270
+ def rank(self):
271
+ return self._rank
272
+
273
+ @property
274
+ def world_size(self):
275
+ return self._world_size
276
+
277
+ def flatten(self, input):
278
+ new_list = []
279
+ for i in input:
280
+ for j in i:
281
+ new_list.append(j)
282
+ return new_list
283
+
284
+ def generate_until(self, requests) -> List[str]:
285
+ res = []
286
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
287
+
288
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
289
+ if "until" in gen_kwargs:
290
+ gen_kwargs.pop("until")
291
+ for k, v in DEFAULT_GEN_KWARGS.items():
292
+ if k not in gen_kwargs:
293
+ gen_kwargs[k] = v
294
+
295
+ pop_keys = []
296
+ for k, v in gen_kwargs.items():
297
+ if k not in DEFAULT_GEN_KWARGS:
298
+ pop_keys.append(k)
299
+
300
+ for k in pop_keys:
301
+ gen_kwargs.pop(k)
302
+
303
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
304
+ visuals = self.flatten(visuals)
305
+ if self.modality == "image":
306
+ if visuals:
307
+ visuals = [load_image(visual).to(torch.bfloat16).cuda() for visual in visuals]
308
+ pixel_values = torch.cat(visuals, dim=0)
309
+ num_patches_list = [visual.size(0) for visual in visuals]
310
+ image_tokens = ["<image>"] * len(visuals)
311
+ image_tokens = " ".join(image_tokens)
312
+ contexts = image_tokens + "\n" + contexts
313
+ else:
314
+ pixel_values = None
315
+ num_patches_list = None
316
+ response, history = self.model.chat(self.tokenizer, pixel_values, contexts, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
317
+ elif self.modality == "video":
318
+ assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos."
319
+ video_path = visuals[0]
320
+ pixel_values, num_patches_list = load_video(video_path, num_segments=self.num_frame)
321
+ pixel_values = pixel_values.to(torch.bfloat16).cuda()
322
+ video_prefix = "".join([f"Frame{i+1}: <image>\n" for i in range(len(num_patches_list))])
323
+ question = video_prefix + contexts
324
+ response, history = self.model.chat(self.tokenizer, pixel_values, question, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True)
325
+ res.append(response)
326
+ pbar.update(1)
327
+ pbar.close()
328
+ return res
329
+
330
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
331
+ assert False, "Not implemented yet."
332
+
333
+ def generate_until_multi_round(self, requests) -> List[str]:
334
+ raise NotImplementedError("TODO: Implement multi-round generation for InternVL2")
LaViDa/eval/lmms_eval/models/llama_vid.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+ import subprocess
4
+ from datetime import timedelta
5
+ from typing import List, Optional, Tuple, Union
6
+
7
+ import numpy as np
8
+ import requests
9
+ import torch
10
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
11
+ from accelerate.state import AcceleratorState
12
+ from decord import VideoReader, cpu
13
+ from huggingface_hub import snapshot_download
14
+ from loguru import logger as eval_logger
15
+ from tqdm import tqdm
16
+ from transformers import AutoConfig
17
+
18
+ from lmms_eval import utils
19
+ from lmms_eval.api.instance import Instance
20
+ from lmms_eval.api.model import lmms
21
+ from lmms_eval.api.registry import register_model
22
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
23
+ from lmms_eval.utils import stop_sequences_criteria
24
+
25
+ try:
26
+ from llamavid.constants import (
27
+ DEFAULT_IM_END_TOKEN,
28
+ DEFAULT_IM_START_TOKEN,
29
+ DEFAULT_IMAGE_TOKEN,
30
+ IMAGE_TOKEN_INDEX,
31
+ )
32
+ from llamavid.conversation import SeparatorStyle, conv_templates
33
+ from llamavid.model.builder import load_pretrained_model
34
+ from llava.mm_utils import (
35
+ KeywordsStoppingCriteria,
36
+ get_model_name_from_path,
37
+ tokenizer_image_token,
38
+ )
39
+ except ImportError:
40
+ eval_logger.debug("LLaMA-Video is not installed. Please install LLaMA-Video to use this model.")
41
+
42
+
43
+ @register_model("llama_vid")
44
+ class LLaMAVid(lmms):
45
+ def __init__(
46
+ self,
47
+ pretrained: str = "YanweiLi/llama-vid-7b-full-224-video-fps-1",
48
+ truncation: Optional[bool] = True,
49
+ device: Optional[str] = "cuda:0",
50
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
51
+ batch_size: Optional[Union[int, str]] = 1,
52
+ trust_remote_code: Optional[bool] = False,
53
+ revision=None,
54
+ attn_implementation=(
55
+ "sdpa" if torch.__version__ > "2.1.2" else "eager"
56
+ ), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
57
+ device_map="cuda:0",
58
+ conv_template="vicuna_v1",
59
+ use_cache=True,
60
+ truncate_context=False,
61
+ num_frames: int = 100,
62
+ **kwargs,
63
+ ) -> None:
64
+ super().__init__()
65
+
66
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
67
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
68
+ if accelerator.num_processes > 1:
69
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
70
+ self.device_map = f"cuda:{accelerator.local_process_index}"
71
+ elif accelerator.num_processes == 1 and device_map == "auto":
72
+ self._device = torch.device(device)
73
+ self.device_map = device_map
74
+ else:
75
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
76
+ self.device_map = f"cuda:{accelerator.local_process_index}"
77
+
78
+ self.pretrained = pretrained
79
+ self.model_path = snapshot_download(self.pretrained)
80
+ self.model_name = get_model_name_from_path(pretrained)
81
+ self.num_frames = num_frames
82
+ if not os.path.exists("./model_zoo/LAVIS/eva_vit_g.pth") and accelerator.is_main_process:
83
+ eval_logger.info("\n\n Eva Encoder is not found for LLaMA-VID. Download automatically to the folder ./model_zoo/LAVIS")
84
+ cache_path = "model_zoo/LAVIS"
85
+ os.makedirs(cache_path, exist_ok=True)
86
+ subprocess.run(["wget https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/BLIP2/eva_vit_g.pth -O ./model_zoo/LAVIS/eva_vit_g.pth"], shell=True)
87
+
88
+ accelerator.wait_for_everyone()
89
+ self._tokenizer, self._model, self.image_processor, self._max_length = load_pretrained_model(
90
+ self.model_path,
91
+ None,
92
+ self.model_name,
93
+ device_map=self.device_map,
94
+ )
95
+
96
+ self._config = self._model.config
97
+ self.model.eval()
98
+ self.model.tie_weights()
99
+ self.truncation = truncation
100
+ self.batch_size_per_gpu = int(batch_size)
101
+ self.conv_template = conv_template
102
+ self.use_cache = use_cache
103
+ self.truncate_context = truncate_context
104
+ # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
105
+ if accelerator.num_processes > 1:
106
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
107
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
108
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
109
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
110
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
111
+ kwargs = {
112
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
113
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
114
+ }
115
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
116
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
117
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
118
+ self._model = accelerator.prepare(self.model)
119
+ else:
120
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
121
+ self.accelerator = accelerator
122
+ if self.accelerator.is_local_main_process:
123
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
124
+ self._rank = self.accelerator.local_process_index
125
+ self._world_size = self.accelerator.num_processes
126
+ elif accelerator.num_processes == 1 and device_map == "auto":
127
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
128
+ self._rank = 0
129
+ self._world_size = 1
130
+ else:
131
+ eval_logger.info(f"Using single device: {self._device}")
132
+ self.model.to(self._device)
133
+ self._rank = 0
134
+ self._world_size = 1
135
+
136
+ def download_file(self, url, folder_path):
137
+ # Create the folder if it doesn't exist
138
+ if not os.path.exists(folder_path):
139
+ os.makedirs(folder_path)
140
+
141
+ # Extract filename from URL
142
+ filename = url.split("/")[-1]
143
+
144
+ # Define path to save the file
145
+ file_path = os.path.join(folder_path, filename)
146
+
147
+ # Send a GET request to the URL
148
+ response = requests.get(url)
149
+
150
+ # Check if request was successful (status code 200)
151
+ if response.status_code == 200:
152
+ # Save the file to the specified folder
153
+ with open(file_path, "wb") as f:
154
+ f.write(response.content)
155
+ print(f"File downloaded successfully to {file_path}")
156
+ else:
157
+ print(f"Failed to download file. Status code: {response.status_code}")
158
+
159
+ @property
160
+ def config(self):
161
+ # return the associated transformers.AutoConfig for the given pretrained model.
162
+ return self._config
163
+
164
+ @property
165
+ def tokenizer(self):
166
+ return self._tokenizer
167
+
168
+ @property
169
+ def model(self):
170
+ # returns the model, unwrapping it if using Accelerate
171
+ if hasattr(self, "accelerator"):
172
+ return self.accelerator.unwrap_model(self._model)
173
+ else:
174
+ return self._model
175
+
176
+ @property
177
+ def eot_token_id(self):
178
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
179
+ return self.tokenizer.eos_token_id
180
+
181
+ @property
182
+ def max_length(self):
183
+ return self._max_length
184
+
185
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
186
+ """ """
187
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
188
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
189
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
190
+ if left_truncate_len:
191
+ encoding = encoding[-left_truncate_len:]
192
+ return encoding
193
+
194
+ def tok_decode(self, tokens):
195
+ return self.tokenizer.decode(tokens)
196
+
197
+ def load_video(self, video_path):
198
+ vr = VideoReader(video_path, ctx=cpu(0))
199
+ total_frame_num = len(vr)
200
+ fps = round(vr.get_avg_fps())
201
+ frame_idx = [i for i in range(0, len(vr), fps)]
202
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
203
+ return spare_frames
204
+
205
+ def flatten(self, input):
206
+ new_list = []
207
+ for i in input:
208
+ for j in i:
209
+ new_list.append(j)
210
+ return new_list
211
+
212
+ def generate_until(self, requests) -> List[str]:
213
+ res = []
214
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
215
+
216
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
217
+ # encode, pad, and truncate contexts for this batch
218
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
219
+ visuals = self.flatten(visuals)
220
+ videos = []
221
+ for visual in visuals:
222
+ video = read_video_pyav(visual, num_frm=self.num_frames)
223
+ video = self.image_processor.preprocess(video, return_tensors="pt")["pixel_values"].half().cuda()
224
+ video = [video]
225
+ videos += video
226
+ qs = contexts
227
+ if self.model.config.mm_use_im_start_end:
228
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
229
+ else:
230
+ qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
231
+
232
+ conv = conv_templates[self.conv_template].copy()
233
+ conv.append_message(conv.roles[0], qs)
234
+ conv.append_message(conv.roles[1], None)
235
+ prompt = conv.get_prompt()
236
+
237
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
238
+
239
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
240
+ keywords = [stop_str]
241
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
242
+
243
+ cur_prompt = contexts
244
+ with torch.inference_mode():
245
+ self.model.update_prompt([[cur_prompt]])
246
+ output_ids = self.model.generate(input_ids, images=video, do_sample=True, temperature=0.2, max_new_tokens=1024, use_cache=True, stopping_criteria=[stopping_criteria])
247
+
248
+ input_token_len = input_ids.shape[1]
249
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
250
+ if n_diff_input_output > 0:
251
+ print(f"[Warning] {n_diff_input_output} output_ids are not the same as the input_ids")
252
+ outputs = self.tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
253
+ outputs = outputs.strip()
254
+ if outputs.endswith(stop_str):
255
+ outputs = outputs[: -len(stop_str)]
256
+ outputs = outputs.strip()
257
+ pbar.update(1)
258
+ res.append(outputs)
259
+
260
+ return res
261
+
262
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
263
+ return super().loglikelihood(requests)
264
+
265
+ @property
266
+ def batch_size(self):
267
+ return self.batch_size_per_gpu
268
+
269
+ @property
270
+ def device(self):
271
+ return self._device
272
+
273
+ @property
274
+ def rank(self):
275
+ return self._rank
276
+
277
+ @property
278
+ def world_size(self):
279
+ return self._world_size
280
+
281
+ def generate_until_multi_round(self, requests) -> List[str]:
282
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaMAVid")
LaViDa/eval/lmms_eval/models/llama_vision.py ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import numpy as np
5
+ import PIL
6
+ import torch
7
+ from accelerate import Accelerator, DistributedType
8
+ from accelerate.state import AcceleratorState
9
+ from decord import VideoReader, cpu
10
+ from torchvision.transforms.functional import to_pil_image
11
+ from tqdm import tqdm
12
+ from transformers import AutoConfig, AutoProcessor, MllamaForConditionalGeneration
13
+
14
+ from lmms_eval import utils
15
+ from lmms_eval.api.instance import Instance
16
+ from lmms_eval.api.model import lmms
17
+ from lmms_eval.api.registry import register_model
18
+
19
+ warnings.filterwarnings("ignore")
20
+
21
+ from loguru import logger as eval_logger
22
+
23
+ DEFAULT_IMAGE_TOKEN = "<|image|>"
24
+
25
+
26
+ @register_model("llama_vision")
27
+ class LlamaVision(lmms):
28
+ def __init__(
29
+ self,
30
+ pretrained: str = "meta-llama/Llama-3.2-11B-Vision",
31
+ revision: str = "main",
32
+ device: str = "cuda",
33
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
34
+ batch_size: int = 1,
35
+ trust_remote_code: Optional[bool] = False,
36
+ attn_implementation: Optional[str] = None,
37
+ device_map: str = "",
38
+ max_frames_num: Optional[int] = 32,
39
+ **kwargs,
40
+ ) -> None:
41
+ super().__init__()
42
+ # Do not use kwargs for now
43
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
44
+
45
+ accelerator = Accelerator()
46
+ if accelerator.num_processes > 1 and device_map == "":
47
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
48
+ self.device_map = f"cuda:{accelerator.local_process_index}"
49
+ else:
50
+ self._device = torch.device(device)
51
+ self.device_map = device_map
52
+ if isinstance(dtype, str) and dtype != "auto":
53
+ dtype = getattr(torch, dtype)
54
+
55
+ self.max_frames_num = max_frames_num
56
+ self._model = MllamaForConditionalGeneration.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
57
+ self.model.eval()
58
+ self.processor = AutoProcessor.from_pretrained(pretrained)
59
+ if accelerator.num_processes > 1 and device_map == "":
60
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
61
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
62
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
63
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
64
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
65
+ kwargs = {
66
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
67
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
68
+ }
69
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
70
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
71
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
72
+ self._model = accelerator.prepare(self.model)
73
+ else:
74
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
75
+ self.accelerator = accelerator
76
+ if self.accelerator.is_local_main_process:
77
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
78
+ self._rank = self.accelerator.local_process_index
79
+ self._world_size = self.accelerator.num_processes
80
+ elif accelerator.num_processes == 1 and device_map == "auto":
81
+ eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
82
+ self._rank = 0
83
+ self._world_size = 1
84
+ else:
85
+ eval_logger.info(f"Using single device: {self._device}")
86
+ self.model.to(self._device)
87
+ self._rank = 0
88
+ self._world_size = 1
89
+ self.accelerator = accelerator
90
+
91
+ @property
92
+ def config(self):
93
+ # return the associated transformers.AutoConfig for the given pretrained model.
94
+ return self._config
95
+
96
+ @property
97
+ def tokenizer(self):
98
+ return self._tokenizer
99
+
100
+ @property
101
+ def model(self):
102
+ # returns the model, unwrapping it if using Accelerate
103
+ if hasattr(self, "accelerator"):
104
+ return self.accelerator.unwrap_model(self._model)
105
+ else:
106
+ return self._model
107
+
108
+ @property
109
+ def eot_token_id(self):
110
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
111
+ return self.tokenizer.eos_token_id
112
+
113
+ @property
114
+ def max_length(self):
115
+ return self._max_length
116
+
117
+ @property
118
+ def batch_size(self):
119
+ return self.batch_size_per_gpu
120
+
121
+ @property
122
+ def device(self):
123
+ return self._device
124
+
125
+ @property
126
+ def rank(self):
127
+ return self._rank
128
+
129
+ @property
130
+ def world_size(self):
131
+ return self._world_size
132
+
133
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
134
+ """ """
135
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
136
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
137
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
138
+ if left_truncate_len:
139
+ encoding = encoding[-left_truncate_len:]
140
+ return encoding
141
+
142
+ def tok_decode(self, tokens):
143
+ return self.tokenizer.decode(tokens)
144
+
145
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
146
+ assert False, "Not implemented"
147
+
148
+ def flatten(self, input):
149
+ new_list = []
150
+ for i in input:
151
+ for j in i:
152
+ new_list.append(j)
153
+ return new_list
154
+
155
+ def load_video(self, video_path, max_frames_num):
156
+ if type(video_path) == str:
157
+ vr = VideoReader(video_path, ctx=cpu(0))
158
+ else:
159
+ vr = VideoReader(video_path[0], ctx=cpu(0))
160
+ total_frame_num = len(vr)
161
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
162
+ frame_idx = uniform_sampled_frames.tolist()
163
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
164
+ return spare_frames # (frames, height, width, channels)
165
+
166
+ def generate_until(self, requests: List[Instance]) -> List[str]:
167
+ res = []
168
+
169
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
170
+
171
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
172
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
173
+ visuals = self.flatten(visuals)
174
+
175
+ messages = [{"role": "user", "content": []}]
176
+ images = []
177
+
178
+ for visual in visuals:
179
+ if isinstance(visual, str):
180
+ frames = self.load_video(visual, self.max_frames_num)
181
+ frames = torch.from_numpy(frames).permute(0, 3, 1, 2)
182
+ images.extend([to_pil_image(frame) for frame in frames])
183
+ elif isinstance(visual, PIL.Image.Image):
184
+ images.append(visual)
185
+
186
+ for _ in range(len(images)):
187
+ messages[-1]["content"].append({"type": "image"})
188
+ messages[-1]["content"].append({"type": "text", "text": contexts})
189
+ prompt = self.processor.apply_chat_template(messages, add_generation_prompt=True)
190
+ inputs = self.processor(images, prompt, add_special_tokens=False, return_tensors="pt").to(self.model.device)
191
+
192
+ if "max_new_tokens" not in gen_kwargs:
193
+ gen_kwargs["max_new_tokens"] = 1024
194
+ if "temperature" not in gen_kwargs:
195
+ gen_kwargs["temperature"] = 0
196
+ if "top_p" not in gen_kwargs:
197
+ gen_kwargs["top_p"] = None
198
+ if "num_beams" not in gen_kwargs:
199
+ gen_kwargs["num_beams"] = 1
200
+ if "do_sample" not in gen_kwargs:
201
+ gen_kwargs["do_sample"] = False
202
+
203
+ with torch.no_grad():
204
+ output = self.model.generate(
205
+ **inputs,
206
+ max_new_tokens=gen_kwargs["max_new_tokens"],
207
+ temperature=gen_kwargs["temperature"],
208
+ do_sample=gen_kwargs["do_sample"],
209
+ )
210
+ output = output[:, inputs["input_ids"].shape[-1] :]
211
+ res.append(self.processor.decode(output[0], skip_special_tokens=True))
212
+
213
+ pbar.update(1)
214
+ pbar.close()
215
+ return res
216
+
217
+ def generate_until_multi_round(self, requests) -> List[str]:
218
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
LaViDa/eval/lmms_eval/models/llava.py ADDED
@@ -0,0 +1,442 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ torch.backends.cuda.matmul.allow_tf32 = True
4
+
5
+
6
+ import copy
7
+ import warnings
8
+ from datetime import timedelta
9
+ from typing import List, Optional, Tuple, Union
10
+
11
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
12
+ from accelerate.state import AcceleratorState
13
+ from packaging import version
14
+ from tqdm import tqdm
15
+
16
+ from lmms_eval import utils
17
+ from lmms_eval.api.instance import Instance
18
+ from lmms_eval.api.model import lmms
19
+ from lmms_eval.api.registry import register_model
20
+ from lmms_eval.utils import stop_sequences_criteria
21
+ import time
22
+ warnings.filterwarnings("ignore")
23
+
24
+ from loguru import logger as eval_logger
25
+
26
+ try:
27
+ from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
28
+ from llava.conversation import conv_templates
29
+ from llava.mm_utils import (
30
+ get_model_name_from_path,
31
+ process_images,
32
+ tokenizer_image_token,
33
+ )
34
+ from llava.model.builder import load_pretrained_model
35
+ except Exception as e:
36
+ eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
37
+
38
+ # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
39
+ # if is_flash_attn_2_available:
40
+ # best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
41
+
42
+ if version.parse(torch.__version__) >= version.parse("2.1.2"):
43
+ best_fit_attn_implementation = "sdpa"
44
+ else:
45
+ best_fit_attn_implementation = "eager"
46
+
47
+
48
+ @register_model("llava")
49
+ class Llava(lmms):
50
+ """
51
+ Llava Model
52
+ """
53
+
54
+ def __init__(
55
+ self,
56
+ pretrained: str = "liuhaotian/llava-v1.5-7b",
57
+ truncation: Optional[bool] = True,
58
+ device: Optional[str] = "cuda:0",
59
+ batch_size: Optional[Union[int, str]] = 1,
60
+ model_name=None,
61
+ attn_implementation=best_fit_attn_implementation,
62
+ device_map="cuda:0",
63
+ conv_template="vicuna_v1",
64
+ use_cache=True,
65
+ tie_weights: bool = True,
66
+ truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
67
+ customized_config=None, # ends in json
68
+ **kwargs,
69
+ ) -> None:
70
+ super().__init__()
71
+ # Do not use kwargs for now
72
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
73
+
74
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
75
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
76
+ self.accelerator = accelerator
77
+ if accelerator.num_processes > 1:
78
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
79
+ self.device_map = f"cuda:{accelerator.local_process_index}"
80
+ elif accelerator.num_processes == 1 and device_map == "auto":
81
+ self._device = torch.device(device)
82
+ self.device_map = device_map
83
+ else:
84
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
85
+ self.device_map = f"cuda:{accelerator.local_process_index}"
86
+
87
+ llava_model_args = {
88
+ "multimodal": True,
89
+ }
90
+ if customized_config is not None:
91
+ llava_model_args["customized_config"] = customized_config
92
+ if attn_implementation is not None:
93
+ llava_model_args["attn_implementation"] = attn_implementation
94
+ if "use_flash_attention_2" in kwargs:
95
+ llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
96
+ model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
97
+
98
+ try:
99
+ # Try to load the model with the multimodal argument
100
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map='cpu', **llava_model_args)
101
+ self._model.to(torch.float16).to('cuda')
102
+ except TypeError:
103
+ # for older versions of LLaVA that don't have multimodal argument
104
+ llava_model_args.pop("multimodal", None)
105
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
106
+ self._config = self._model.config
107
+ self.model.eval()
108
+ if tie_weights:
109
+ self.model.tie_weights()
110
+
111
+ self.truncation = truncation
112
+ self.batch_size_per_gpu = int(batch_size)
113
+ self.conv_template = conv_template
114
+ self.use_cache = use_cache
115
+ self.truncate_context = truncate_context
116
+ # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
117
+ if accelerator.num_processes > 1:
118
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
119
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
120
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
121
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
122
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
123
+ kwargs = {
124
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
125
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
126
+ }
127
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
128
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
129
+
130
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
131
+ self._model = accelerator.prepare(self.model)
132
+ else:
133
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
134
+ self.accelerator = accelerator
135
+ if self.accelerator.is_local_main_process:
136
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
137
+ self._rank = self.accelerator.local_process_index
138
+ self._world_size = self.accelerator.num_processes
139
+ elif accelerator.num_processes == 1 and device_map == "auto":
140
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
141
+ self._rank = 0
142
+ self._world_size = 1
143
+ else:
144
+ eval_logger.info(f"Using single device: {self._device}")
145
+ self.model.to(self._device)
146
+ self._rank = 0
147
+ self._world_size = 1
148
+
149
+ @property
150
+ def config(self):
151
+ # return the associated transformers.AutoConfig for the given pretrained model.
152
+ return self._config
153
+
154
+ @property
155
+ def tokenizer(self):
156
+ return self._tokenizer
157
+
158
+ @property
159
+ def model(self):
160
+ # returns the model, unwrapping it if using Accelerate
161
+ if hasattr(self, "accelerator"):
162
+ return self.accelerator.unwrap_model(self._model)
163
+ else:
164
+ return self._model
165
+
166
+ @property
167
+ def eot_token_id(self):
168
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
169
+ return self.tokenizer.eos_token_id
170
+
171
+ @property
172
+ def max_length(self):
173
+ return self._max_length
174
+
175
+ def pad_sequence(self, input_ids, batch_first, padding_value):
176
+ if self.tokenizer.padding_side == "left":
177
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
178
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
179
+ if self.tokenizer.padding_side == "left":
180
+ input_ids = torch.flip(input_ids, [1])
181
+ return input_ids
182
+
183
+ @property
184
+ def batch_size(self):
185
+ return self.batch_size_per_gpu
186
+
187
+ @property
188
+ def device(self):
189
+ return self._device
190
+
191
+ @property
192
+ def rank(self):
193
+ return self._rank
194
+
195
+ @property
196
+ def world_size(self):
197
+ return self._world_size
198
+
199
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
200
+ """ """
201
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
202
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
203
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
204
+ if left_truncate_len:
205
+ encoding = encoding[-left_truncate_len:]
206
+ return encoding
207
+
208
+ def tok_decode(self, tokens):
209
+ try:
210
+ return self.tokenizer.decode(tokens)
211
+ except:
212
+ return self.tokenizer.decode([tokens])
213
+
214
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
215
+ # TODO
216
+ res = []
217
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
218
+
219
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
220
+ # encode, pad, and truncate contexts for this batch
221
+ if type(doc_to_target) == str:
222
+ continuation = doc_to_target
223
+ else:
224
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
225
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
226
+ visuals = self.flatten(visuals)
227
+ image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
228
+ if visuals:
229
+ image = process_images(visuals, self._image_processor, self._config)
230
+ if type(image) is list:
231
+ image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
232
+ else:
233
+ image = image.to(dtype=torch.float16, device=self.device)
234
+ else:
235
+ image = None
236
+
237
+ prompts_input = contexts[0] if isinstance(contexts, list) else contexts
238
+
239
+ if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
240
+ """
241
+ Three senarios:
242
+ 1. No image, and there for, no image token should be added.
243
+ 2. image token is already specified in the context, so we don't need to add it.
244
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
245
+ """
246
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
247
+ image_tokens = " ".join(image_tokens)
248
+ prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts)
249
+
250
+ # This is much safer for llama3, as we now have some object type in it
251
+ if "llama_3" in self.conv_template:
252
+ conv = copy.deepcopy(conv_templates[self.conv_template])
253
+ else:
254
+ conv = conv_templates[self.conv_template].copy()
255
+ conv.append_message(conv.roles[0], prompts_input)
256
+ conv.append_message(conv.roles[1], None)
257
+ prompt = conv.get_prompt()
258
+ pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
259
+ contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
260
+ # Add the answer of the second role
261
+ conv.messages[1][1] = continuation
262
+
263
+ prompt = conv.get_prompt()
264
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
265
+ labels = input_ids.clone()
266
+ # Context part no need to calculate for loss
267
+ labels[0, : contxt_id.shape[1]] = -100
268
+ with torch.inference_mode():
269
+ outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
270
+ loss = outputs["loss"]
271
+ # loss = torch.exp(loss)
272
+ logits = outputs["logits"]
273
+ greedy_tokens = logits.argmax(dim=-1)
274
+ cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
275
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
276
+ max_equal = (greedy_tokens == cont_toks).all()
277
+ res.append((float(loss.item()), bool(max_equal)))
278
+ pbar.update(1)
279
+ pbar.close()
280
+ return res
281
+
282
+ def flatten(self, input):
283
+ if not input or any(i is None for i in input):
284
+ return []
285
+ new_list = []
286
+ for i in input:
287
+ if i:
288
+ for j in i:
289
+ new_list.append(j)
290
+ return new_list
291
+
292
+ def generate_until(self, requests: List[Instance]) -> List[str]:
293
+ res = []
294
+
295
+ def _collate(x):
296
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
297
+ # - time estimates will always be over not underestimates, which is more useful for planning
298
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
299
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
300
+ # automatic adaptive batches much much easier to implement
301
+ # - any OOMs will happen right away rather than near the end
302
+ toks = self.tok_encode(x[0])
303
+ return -len(toks), x[0]
304
+
305
+ # we group requests by their generation_kwargs,
306
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
307
+ # in the same batch.
308
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
309
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
310
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
311
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
312
+ delta_t = 0
313
+ num_generated = 0
314
+ for chunk in chunks:
315
+ t0 = time.time()
316
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
317
+ task = task[0]
318
+ split = split[0]
319
+ batched_visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id] # [B, N]
320
+ flattened_visuals = self.flatten(batched_visuals) # [B*N]
321
+ # we assume all gen kwargs in the batch are the same
322
+ # this is safe to assume because the `grouper` object ensures it.
323
+ gen_kwargs = all_gen_kwargs[0]
324
+
325
+ # Set default values for until and max_new_tokens
326
+ until = [self.tok_decode(self.eot_token_id)]
327
+
328
+ # Update values from gen_kwargs if present
329
+ if "until" in gen_kwargs:
330
+ until = gen_kwargs.pop("until")
331
+ if isinstance(until, str):
332
+ until = [until]
333
+ elif not isinstance(until, list):
334
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
335
+ # self._config.image_aspect_ratio = 'baseline'
336
+ if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
337
+ # here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
338
+ self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
339
+ eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
340
+ # encode, pad, and truncate contexts for this batch
341
+ if flattened_visuals:
342
+ image_tensor = process_images(flattened_visuals, self._image_processor, self._config)
343
+ if type(image_tensor) is list:
344
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
345
+ else:
346
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
347
+ else:
348
+ image_tensor = None
349
+
350
+ # prompts_input = contexts[0]
351
+
352
+ question_input = []
353
+
354
+ for visual, context in zip(batched_visuals, contexts):
355
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
356
+ """
357
+ Three senarios:
358
+ 1. No image, and there for, no image token should be added.
359
+ 2. image token is already specified in the context, so we don't need to add it.
360
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
361
+ """
362
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
363
+ image_tokens = " ".join(image_tokens)
364
+ question = image_tokens + "\n" + context
365
+ else:
366
+ question = context
367
+ # This is much safer for llama3, as we now have some object type in it
368
+ if "llama_3" in self.conv_template:
369
+ conv = copy.deepcopy(conv_templates[self.conv_template])
370
+ else:
371
+ conv = conv_templates[self.conv_template].copy()
372
+ conv.append_message(conv.roles[0], question)
373
+ conv.append_message(conv.roles[1], None)
374
+ prompt_question = conv.get_prompt()
375
+ question_input.append(prompt_question)
376
+
377
+ # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
378
+ # preconfigure gen_kwargs with defaults
379
+ gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
380
+ if "max_new_tokens" not in gen_kwargs:
381
+ gen_kwargs["max_new_tokens"] = 1024
382
+ if "temperature" not in gen_kwargs:
383
+ gen_kwargs["temperature"] = 0
384
+ if "top_p" not in gen_kwargs:
385
+ gen_kwargs["top_p"] = None
386
+ if "num_beams" not in gen_kwargs:
387
+ gen_kwargs["num_beams"] = 1
388
+
389
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
390
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
391
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
392
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
393
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
394
+ # TODO: attention to this major generation step...
395
+ try:
396
+ cont = self.model.generate(
397
+ input_ids,
398
+ attention_mask=attention_masks,
399
+ pad_token_id=pad_token_ids,
400
+ images=image_tensor,
401
+ image_sizes=gen_kwargs["image_sizes"],
402
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
403
+ temperature=gen_kwargs["temperature"],
404
+ top_p=gen_kwargs["top_p"],
405
+ num_beams=gen_kwargs["num_beams"],
406
+ max_new_tokens=gen_kwargs["max_new_tokens"],
407
+ use_cache=self.use_cache,
408
+ )
409
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
410
+ except Exception as e:
411
+ raise e
412
+ eval_logger.error(f"Error {e} in generating")
413
+ cont = ""
414
+ text_outputs = [""]
415
+
416
+ # cont_toks_list = cont.tolist()
417
+ # for cont_toks, context in zip(cont_toks_list, contexts):
418
+ # discard context + left-padding toks if using causal decoder-only LMM
419
+ # if self.truncate_context:
420
+ # cont_toks = cont_toks[input_ids.shape[1] :]
421
+ # use secondary stop seqs to cut off should-have-been-stopped content post-hoc
422
+ # if self.truncate_context:
423
+ # for term in until:
424
+ # if len(term) > 0:
425
+ # # ignore '' separator,
426
+ # # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
427
+ # text_outputs = text_outputs.split(term)[0]
428
+ res.extend(text_outputs)
429
+ t1 = time.time()
430
+ delta_t += t1-t0
431
+ num_generated +=1
432
+ print(f"Avg Latenc (of {num_generated}): {delta_t/num_generated}")
433
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
434
+ pbar.update(1)
435
+ # reorder this group of results back to original unsorted form
436
+ res = re_ords.get_original(res)
437
+
438
+ pbar.close()
439
+ return res
440
+
441
+ def generate_until_multi_round(self, requests) -> List[str]:
442
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVA")
LaViDa/eval/lmms_eval/models/llava_dream.py ADDED
@@ -0,0 +1,855 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import json
3
+ import logging
4
+ import math
5
+ import re
6
+ import warnings
7
+ from datetime import timedelta
8
+ from typing import List, Optional, Tuple, Union
9
+
10
+ import numpy as np
11
+ import os
12
+ import PIL
13
+ import torch
14
+ import accelerate
15
+ import transformers
16
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
17
+ from accelerate.state import AcceleratorState
18
+ from decord import VideoReader, cpu
19
+ from packaging import version
20
+ from tqdm import tqdm
21
+ from transformers import AutoConfig
22
+
23
+ from lmms_eval import utils
24
+ from lmms_eval.api.instance import Instance
25
+ from lmms_eval.api.model import lmms
26
+ from lmms_eval.api.registry import register_model
27
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
28
+
29
+ # Suppress warnings
30
+ warnings.filterwarnings("ignore")
31
+
32
+ # Configure logging
33
+ eval_logger = logging.getLogger("lmms-eval")
34
+
35
+ # Enable TF32 for CUDA
36
+ torch.backends.cuda.matmul.allow_tf32 = True
37
+
38
+ # Import LLaVA modules
39
+ try:
40
+ from llava.constants import (
41
+ DEFAULT_IM_END_TOKEN,
42
+ DEFAULT_IM_START_TOKEN,
43
+ DEFAULT_IMAGE_TOKEN,
44
+ IGNORE_INDEX,
45
+ IMAGE_TOKEN_INDEX,
46
+ )
47
+ from llava.conversation import SeparatorStyle, conv_templates
48
+ from llava.mm_utils import (
49
+ KeywordsStoppingCriteria,
50
+ get_model_name_from_path,
51
+ process_images,
52
+ tokenizer_image_token,
53
+ )
54
+ from llava.model.builder import load_pretrained_model
55
+ except ImportError as e:
56
+ eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
57
+
58
+
59
+ # Determine best attention implementation
60
+ if version.parse(torch.__version__) >= version.parse("2.1.2"):
61
+ best_fit_attn_implementation = "sdpa"
62
+ else:
63
+ best_fit_attn_implementation = "eager"
64
+
65
+
66
+ @register_model("llava_llada")
67
+ class Llava_Dream(lmms):
68
+ """
69
+ Llava Model
70
+ """
71
+
72
+ def __init__(
73
+ self,
74
+ pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
75
+ truncation: Optional[bool] = True,
76
+ device: Optional[str] = "cuda:0",
77
+ batch_size: Optional[Union[int, str]] = 1,
78
+ model_name: Optional[str] = None,
79
+ attn_implementation: Optional[str] = best_fit_attn_implementation,
80
+ device_map: Optional[str] = "cuda:0",
81
+ conv_template: Optional[str] = "dream",
82
+ use_cache: Optional[bool] = True,
83
+ truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
84
+ customized_config: Optional[str] = None, # ends in json
85
+ max_frames_num: Optional[int] = 32,
86
+ mm_spatial_pool_stride: Optional[int] = 2,
87
+ mm_spatial_pool_mode: Optional[str] = "bilinear",
88
+ token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
89
+ video_decode_backend: str = "decord",
90
+ mc_num=16,
91
+ **kwargs,
92
+ ) -> None:
93
+ super().__init__()
94
+ # Do not use kwargs for now
95
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
96
+
97
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
98
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
99
+ if accelerator.num_processes > 1:
100
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
101
+ self.device_map = f"cuda:{accelerator.local_process_index}"
102
+ elif accelerator.num_processes == 1 and device_map == "auto":
103
+ self._device = torch.device(device)
104
+ self.device_map = device_map
105
+ else:
106
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
107
+ self.device_map = f"cuda:{accelerator.local_process_index}"
108
+ self.mc_num = mc_num
109
+ llava_model_args = {
110
+ "multimodal": True,
111
+ }
112
+ if customized_config is not None:
113
+ llava_model_args["customized_config"] = customized_config
114
+ if attn_implementation is not None:
115
+ llava_model_args["attn_implementation"] = attn_implementation
116
+ if "use_flash_attention_2" in kwargs:
117
+ llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
118
+ model_name = 'llava_dream'# if model_name is not None else get_model_name_from_path(pretrained)
119
+
120
+ self.pretrained = pretrained
121
+ self.token_strategy = token_strategy
122
+ self.max_frames_num = max_frames_num
123
+ self.mm_spatial_pool_stride = mm_spatial_pool_stride
124
+ self.mm_spatial_pool_mode = mm_spatial_pool_mode
125
+ self.video_decode_backend = video_decode_backend
126
+
127
+ overwrite_config = {}
128
+ overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
129
+ overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
130
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained,trust_remote_code=True)
131
+
132
+ llava_model_args["overwrite_config"] = overwrite_config
133
+ # try:
134
+ # Try to load the model with the multimodal argument
135
+
136
+ if os.path.exists('/data1/jacklishufan/siglip-so400m-patch14-384'):
137
+ vision_tower_path = "/data1/jacklishufan/siglip-so400m-patch14-384"
138
+ else:
139
+ vision_tower_path="/data0/jacklishufan/siglip-so400m-patch14-384"
140
+ # print(vision_tower_path)
141
+ vision_kwargs = dict(
142
+ mm_vision_tower=os.environ.get('LLADA_VISION_ENCODER',vision_tower_path),
143
+ mm_resampler_type=None,
144
+ mm_projector_type='mlp2x_gelu',
145
+ mm_hidden_size=1152,
146
+ use_mm_proj=True
147
+
148
+ )
149
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args,vision_kwargs=vision_kwargs)
150
+ assert self._tokenizer is not None
151
+ # except TypeError:
152
+ # # for older versions of LLaVA that don't have multimodal argument
153
+ # llava_model_args.pop("multimodal", None)
154
+ # self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
155
+
156
+ self._config = self._model.config
157
+ self.model.eval()
158
+ # self.model.model.set_activation_checkpointing(None)
159
+ self.model.requires_grad_(False)
160
+ self.truncation = truncation
161
+ self.batch_size_per_gpu = int(batch_size)
162
+ self.conv_template = conv_template
163
+ self.use_cache = use_cache
164
+ self.truncate_context = truncate_context
165
+ assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
166
+
167
+ if accelerator.num_processes > 1:
168
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
169
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
170
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
171
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
172
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
173
+ kwargs = {
174
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
175
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
176
+ }
177
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
178
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
179
+
180
+ # if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
181
+ # self._model = accelerator.prepare(self.model)
182
+ # else:
183
+ # self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
184
+ self.model.to(self._device).to(torch.bfloat16)
185
+ #self._model.model.transformer = accelerator.prepare(self.model.model.transformer)
186
+
187
+ self.accelerator = accelerator
188
+ if self.accelerator.is_local_main_process:
189
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
190
+ self._rank = self.accelerator.local_process_index
191
+ self._world_size = self.accelerator.num_processes
192
+
193
+ elif accelerator.num_processes == 1 and device_map == "auto":
194
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
195
+ self._rank = 0
196
+ self._world_size = 1
197
+
198
+ else:
199
+ eval_logger.info(f"Using single device: {self._device}")
200
+ self.model.to(self._device).to(torch.bfloat16)
201
+ self._rank = 0
202
+ self._world_size = 1
203
+ #self.model.model.transformer = accelerate.cpu_offload(self.model.model.transformer)
204
+
205
+ @property
206
+ def config(self):
207
+ # return the associated transformers.AutoConfig for the given pretrained model.
208
+ return self._config
209
+
210
+ @property
211
+ def tokenizer(self):
212
+ return self._tokenizer
213
+
214
+ @property
215
+ def model(self):
216
+ # returns the model, unwrapping it if using Accelerate
217
+ if hasattr(self, "accelerator"):
218
+ return self.accelerator.unwrap_model(self._model)
219
+ else:
220
+ return self._model
221
+
222
+ @property
223
+ def eot_token_id(self):
224
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
225
+ return self.tokenizer.eos_token_id
226
+
227
+ @property
228
+ def max_length(self):
229
+ return self._max_length
230
+
231
+ def pad_sequence(self, input_ids, batch_first, padding_value):
232
+ if self.tokenizer.padding_side == "left":
233
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
234
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
235
+ if self.tokenizer.padding_side == "left":
236
+ input_ids = torch.flip(input_ids, [1])
237
+ return input_ids
238
+
239
+ @property
240
+ def batch_size(self):
241
+ return self.batch_size_per_gpu
242
+
243
+ @property
244
+ def device(self):
245
+ return self._device
246
+
247
+ @property
248
+ def rank(self):
249
+ return self._rank
250
+
251
+ @property
252
+ def world_size(self):
253
+ return self._world_size
254
+
255
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
256
+ """ """
257
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
258
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
259
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
260
+ if left_truncate_len:
261
+ encoding = encoding[-left_truncate_len:]
262
+ return encoding
263
+
264
+ def tok_decode(self, tokens):
265
+ try:
266
+ return self.tokenizer.decode(tokens)
267
+ except:
268
+ return self.tokenizer.decode([tokens])
269
+
270
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
271
+ res = []
272
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
273
+
274
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
275
+
276
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
277
+ visual = doc_to_visual(self.task_dict[task][split][doc_id])
278
+
279
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
280
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
281
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
282
+
283
+ if visual is None or visual == []:
284
+ visual = None
285
+ task_type = "text"
286
+ image_tensor = None
287
+ else:
288
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__:
289
+ self._config.image_aspect_ratio = "pad"
290
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
291
+
292
+ if "task_type" in self.metadata and self.metadata["task_type"] == "video" and "sample_frames" in self.metadata:
293
+ assert type(visual) == list, "sample_frames must be specified for video task"
294
+ sample_indices = np.linspace(0, len(visual) - 1, self.metadata["sample_frames"], dtype=int)
295
+ visual = [visual[i] for i in sample_indices]
296
+ assert len(visual) == self.metadata["sample_frames"]
297
+
298
+ image_tensor = process_images(visual, self._image_processor, self._config)
299
+ if type(image_tensor) is list:
300
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
301
+ else:
302
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
303
+
304
+ task_type = "video"
305
+
306
+ # elif type(visual[0]) == PIL.Image.Image:
307
+ elif isinstance(visual[0], PIL.Image.Image):
308
+ image_tensor = process_images(visual, self._image_processor, self._config)
309
+ if type(image_tensor) is list:
310
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
311
+ else:
312
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
313
+
314
+ task_type = "image"
315
+
316
+ elif type(visual[0]) == str:
317
+ image_tensor = []
318
+ try:
319
+ if self.video_decode_backend == "decord":
320
+ frames = self.load_video(visual, self.max_frames_num)
321
+ elif self.video_decode_backend == "pyav":
322
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
323
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
324
+ image_tensor.append(frames)
325
+ except Exception as e:
326
+ eval_logger.error(f"Error {e} in loading video")
327
+ image_tensor = None
328
+
329
+ task_type = "video"
330
+
331
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in contexts:
332
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
333
+ if task_type == "video":
334
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
335
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
336
+ image_tokens = " ".join(image_tokens)
337
+ prompts_input = image_tokens + "\n" + contexts
338
+ else:
339
+ prompts_input = contexts
340
+
341
+
342
+ assert self.conv_template=="dream"
343
+
344
+ if "llama_3" in self.conv_template or 'llada' in self.conv_template or 'dream' in self.conv_template:
345
+ conv = copy.deepcopy(conv_templates[self.conv_template])
346
+ else:
347
+ conv = conv_templates[self.conv_template].copy()
348
+
349
+
350
+ conv.append_message(conv.roles[0], prompts_input)
351
+ conv.append_message(conv.roles[1], None)
352
+ prompt = conv.get_prompt()
353
+ # breakpoint()
354
+
355
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
356
+
357
+ if type(doc_to_target) == str:
358
+ continuation = doc_to_target
359
+ else:
360
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
361
+
362
+ # conv.messages[-1][1] = continuation
363
+ # full_prompt = conv.get_prompt()
364
+ # full_input_ids = tokenizer_image_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
365
+
366
+ # labels = full_input_ids.clone()
367
+ # labels[0, : input_ids.shape[1]] = -100
368
+ input_prompt = conv.get_prompt()
369
+ input_ids = tokenizer_image_token(input_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
370
+ answers = continuation
371
+ answer_ids = self.tokenizer(continuation)['input_ids']
372
+ answer_ids = torch.tensor(continuation).to(input_ids.device).unsqueeze(0)
373
+
374
+
375
+ kwargs = {}
376
+ if task_type == "image":
377
+ kwargs["image_sizes"] = [[v.size[0], v.size[1]] for v in visual] if isinstance(visual, list) else [[visual.size[0], visual.size[1]]]
378
+ elif task_type == "video":
379
+ kwargs["modalities"] = ["video"]
380
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
381
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
382
+
383
+ torch.cuda.empty_cache()
384
+ # with torch.inference_mode():
385
+ #outputs = self.model(input_ids=full_input_ids, labels=labels, images=image_tensor, use_cache=True, **kwargs)
386
+ likelyhoods = self.model.log_likelyhood_inference(
387
+ input_ids,
388
+ images=image_tensor.to(torch.bfloat16),
389
+ image_sizes=None,
390
+ verbose=True,
391
+ answer=answer_ids,
392
+ mc_num=self.mc_num,
393
+ )
394
+
395
+ # loss = outputs["loss"]
396
+ # logits = outputs["logits"]
397
+ # greedy_tokens = logits.argmax(dim=-1)
398
+ # cont_toks = full_input_ids[:, input_ids.shape[1] :]
399
+ # greedy_tokens = greedy_tokens[:, input_ids.shape[1] : full_input_ids.shape[1]]
400
+ # max_equal = (greedy_tokens == cont_toks).all()
401
+ # lmms eval return loss
402
+ res.append((float(-likelyhoods.item()), False))
403
+ pbar.update(1)
404
+
405
+ pbar.close()
406
+ return res
407
+
408
+ def flatten(self, input):
409
+ if not input or any(i is None for i in input):
410
+ return []
411
+ new_list = []
412
+ for i in input:
413
+ if i:
414
+ for j in i:
415
+ new_list.append(j)
416
+ return new_list
417
+
418
+ def load_video(self, video_path, max_frames_num):
419
+ if type(video_path) == str:
420
+ vr = VideoReader(video_path, ctx=cpu(0))
421
+ else:
422
+ vr = VideoReader(video_path[0], ctx=cpu(0))
423
+ total_frame_num = len(vr)
424
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
425
+ frame_idx = uniform_sampled_frames.tolist()
426
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
427
+ return spare_frames # (frames, height, width, channels)
428
+
429
+ def generate_until(self, requests: List[Instance]) -> List[str]:
430
+ res = []
431
+
432
+ def _collate(x):
433
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
434
+ # - time estimates will always be over not underestimates, which is more useful for planning
435
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
436
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
437
+ # automatic adaptive batches much much easier to implement
438
+ # - any OOMs will happen right away rather than near the end
439
+ toks = self.tok_encode(x[0])
440
+ return -len(toks), x[0]
441
+
442
+ # we group requests by their generation_kwargs,
443
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
444
+ # in the same batch.
445
+ metadata = requests[0].metadata
446
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
447
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
448
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
449
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
450
+
451
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
452
+
453
+ for chunk in chunks:
454
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
455
+ task = batched_task[0]
456
+ split = batched_split[0]
457
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
458
+ assert len(batched_visuals) == 1
459
+
460
+ # we assume all gen kwargs in the batch are the same
461
+ # this is safe to assume because the `grouper` object ensures it.
462
+ gen_kwargs = all_gen_kwargs[0]
463
+ if "until" in gen_kwargs:
464
+ gen_kwargs.pop("until")
465
+
466
+ question_input = []
467
+ # import ipdb; ipdb.set_trace()
468
+ for visual, context in zip(batched_visuals, batched_contexts):
469
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
470
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
471
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
472
+
473
+ if visual is None or visual == []: # for text-only tasks.
474
+ visual = None
475
+ task_type = "text"
476
+ placeholder_count = 0
477
+ image_tensor = None
478
+ else:
479
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
480
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
481
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
482
+ print(self._config.image_aspect_ratio )
483
+ if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
484
+ assert type(visual) == list, "sample_frames must be specified for video task"
485
+ sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
486
+ visual = [visual[i] for i in sample_indices]
487
+ assert len(visual) == metadata["sample_frames"]
488
+
489
+ image_tensor = process_images(visual, self._image_processor, self._config)
490
+ if type(image_tensor) is list:
491
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
492
+ else:
493
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
494
+
495
+ task_type = "video"
496
+ placeholder_count = 1
497
+
498
+ elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
499
+ image_tensor = process_images(visual, self._image_processor, self._config)
500
+ if type(image_tensor) is list:
501
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
502
+ else:
503
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
504
+
505
+ task_type = "image"
506
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
507
+
508
+ elif type(visual[0]) == str: # For video task
509
+ image_tensor = []
510
+ try:
511
+ if self.video_decode_backend == "decord":
512
+ frames = self.load_video(visual, self.max_frames_num)
513
+ elif self.video_decode_backend == "pyav":
514
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
515
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
516
+ image_tensor.append(frames)
517
+ except Exception as e:
518
+ eval_logger.error(f"Error {e} in loading video")
519
+ image_tensor = None
520
+
521
+ task_type = "video"
522
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
523
+
524
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
525
+ """
526
+ Three senarios:
527
+ 1. No image, and there for, no image token should be added.
528
+ 2. image token is already specified in the context, so we don't need to add it.
529
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
530
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
531
+ """
532
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
533
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
534
+ # elif task_type == "video":
535
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
536
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
537
+ image_tokens = " ".join(image_tokens)
538
+ question = image_tokens + "\n" + context
539
+ else:
540
+ question = context
541
+
542
+ # This is much safer for llama3, as we now have some object type in it
543
+ # print(self.conv_template)
544
+ if "llama_3" in self.conv_template or 'llada' in self.conv_template or 'dream' in self.conv_template:
545
+ conv = copy.deepcopy(conv_templates[self.conv_template])
546
+ else:
547
+ conv = conv_templates[self.conv_template].copy()
548
+
549
+ if utils.is_json(question): # conversational question input
550
+ question = json.loads(question)
551
+ for idx, item in enumerate(question):
552
+ role = conv.roles[idx % 2]
553
+ message = item["value"]
554
+ conv.append_message(role, message)
555
+
556
+ assert len(conv.messages) % 2 == 1
557
+ conv.append_message(conv.roles[1], None)
558
+ prompt_question = conv.get_prompt()
559
+ question_input.append(prompt_question)
560
+ else: # only simple string for question
561
+ conv.append_message(conv.roles[0], question)
562
+ conv.append_message(conv.roles[1], None)
563
+ # breakpoint()
564
+ prompt_question = conv.get_prompt()
565
+ question_input.append(prompt_question)
566
+ # breakpoint()
567
+ # preconfigure gen_kwargs with defaults
568
+ if "max_new_tokens" not in gen_kwargs:
569
+ gen_kwargs["max_new_tokens"] = 256
570
+
571
+ if "temperature" not in gen_kwargs:
572
+ gen_kwargs["temperature"] = 0
573
+ if "do_sample" not in gen_kwargs:
574
+ gen_kwargs["do_sample"] = False
575
+ if "top_p" not in gen_kwargs:
576
+ gen_kwargs["top_p"] = None
577
+ if "num_beams" not in gen_kwargs:
578
+ gen_kwargs["num_beams"] = 1
579
+ schedule_kwargs = {}
580
+ for key in list(gen_kwargs.keys()):
581
+ if key.startswith('schedule__'):
582
+ value = gen_kwargs.pop(key)
583
+ schedule_kwargs[key.replace('schedule__','')] = value
584
+
585
+ if 'block_length' not in gen_kwargs:
586
+ gen_kwargs['block_length'] = min(128,gen_kwargs["max_new_tokens"])
587
+ if 'step_per_block' not in gen_kwargs:
588
+ gen_kwargs['step_per_block'] = gen_kwargs['block_length']
589
+ gen_kwargs["temperature"] = 0
590
+
591
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
592
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
593
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
594
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
595
+
596
+ if task_type == "image":
597
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
598
+ elif task_type == "video":
599
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
600
+ keywords = [stop_str]
601
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
602
+ gen_kwargs["modalities"] = ["video"]
603
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
604
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
605
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
606
+
607
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
608
+ # TODO: attention to this major generation step...
609
+ if "image_aspect_ratio" in gen_kwargs.keys():
610
+ gen_kwargs.pop("image_aspect_ratio")
611
+ # breakpoint()
612
+ # breakpoint()
613
+ if len(schedule_kwargs) > 0:
614
+ gen_kwargs['schedule_kwargs'] = schedule_kwargs
615
+ try:
616
+ with torch.inference_mode():
617
+ cont = self.model.generate(input_ids,
618
+ attention_mask=attention_masks,
619
+ pad_token_id=pad_token_ids,
620
+ images=image_tensor,
621
+ use_cache=self.use_cache,
622
+ **gen_kwargs).sequences
623
+ # cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
624
+
625
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
626
+ text_outputs = [text_output.lstrip('!').replace('<|im_end|>\n','') for text_output in text_outputs]
627
+ # breakpoint()
628
+ except Exception as e:
629
+ raise e
630
+
631
+ text_outputs = [response.strip() for response in text_outputs]
632
+ res.extend(text_outputs)
633
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
634
+ pbar.update(1)
635
+ # reorder this group of results back to original unsorted form
636
+ res = re_ords.get_original(res)
637
+
638
+ pbar.close()
639
+ return res
640
+
641
+ def generate_until_multi_round(self, requests: List[Instance]) -> List[str]:
642
+ res = []
643
+ raise NotImplementedError()
644
+
645
+ def _collate(x):
646
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
647
+ # - time estimates will always be over not underestimates, which is more useful for planning
648
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
649
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
650
+ # automatic adaptive batches much much easier to implement
651
+ # - any OOMs will happen right away rather than near the end
652
+ toks = self.tok_encode(x[0])
653
+ return -len(toks), x[0]
654
+
655
+ # we group requests by their generation_kwargs,
656
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
657
+ # in the same batch.
658
+ metadata = requests[0].metadata
659
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
660
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
661
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
662
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
663
+
664
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
665
+
666
+ for chunk in chunks:
667
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_to_text, batched_doc_id, batched_task, batched_split = zip(*chunk)
668
+ task = batched_task[0]
669
+ split = batched_split[0]
670
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
671
+ assert len(batched_visuals) == 1
672
+
673
+ # we assume all gen kwargs in the batch are the same
674
+ # this is safe to assume because the `grouper` object ensures it.
675
+ gen_kwargs = all_gen_kwargs[0]
676
+ if "until" in gen_kwargs:
677
+ gen_kwargs.pop("until")
678
+
679
+ # multi round inference: terminate when receiving signal from the doc_to_text
680
+ round_idx = 0
681
+ batched_round_res = []
682
+ batched_previous_round_info = None
683
+ while True:
684
+ question_input = []
685
+
686
+ if round_idx != 0: # get current round visual and context from doc_to_text function
687
+ batched_visuals, batched_contexts, batched_terminal_singal, batched_round_res, batched_previous_round_info = list(
688
+ zip(
689
+ *[
690
+ batched_doc_to_text[0](
691
+ self.task_dict[task][split][ids],
692
+ previous_output=[round_res[ids_idx] for round_res in batched_round_res],
693
+ round_idx=round_idx,
694
+ previous_round_info=batched_previous_round_info[ids_idx] if batched_previous_round_info is not None else None,
695
+ )
696
+ for ids_idx, ids in enumerate(batched_doc_id)
697
+ ]
698
+ )
699
+ )
700
+ # import ipdb; ipdb.set_trace()
701
+ batched_round_res = list(zip(*batched_round_res)) # [(r1_1, r1_2), (r2_1, r2_2), ...]
702
+ if batched_terminal_singal[0]: # terminal signal from doc_to_text function
703
+ break
704
+
705
+ for visual, context in zip(batched_visuals, batched_contexts):
706
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
707
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
708
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
709
+
710
+ if visual is None or visual == []: # for text-only tasks.
711
+ visual = None
712
+ task_type = "text"
713
+ placeholder_count = 0
714
+ image_tensor = None
715
+ else:
716
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
717
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
718
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
719
+
720
+ if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
721
+ assert type(visual) == list, "sample_frames must be specified for video task"
722
+ sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
723
+ visual = [visual[i] for i in sample_indices]
724
+ assert len(visual) == metadata["sample_frames"]
725
+
726
+ image_tensor = process_images(visual, self._image_processor, self._config)
727
+ if type(image_tensor) is list:
728
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
729
+ else:
730
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
731
+
732
+ task_type = "video"
733
+ placeholder_count = 1
734
+
735
+ elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
736
+ image_tensor = process_images(visual, self._image_processor, self._config)
737
+ if type(image_tensor) is list:
738
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
739
+ else:
740
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
741
+
742
+ task_type = "image"
743
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
744
+
745
+ elif type(visual[0]) == str: # For video task
746
+ image_tensor = []
747
+ try:
748
+ if self.video_decode_backend == "decord":
749
+ frames = self.load_video(visual, self.max_frames_num)
750
+ elif self.video_decode_backend == "pyav":
751
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
752
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
753
+ image_tensor.append(frames)
754
+ except Exception as e:
755
+ eval_logger.error(f"Error {e} in loading video")
756
+ image_tensor = None
757
+
758
+ task_type = "video"
759
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
760
+
761
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
762
+ """
763
+ Three senarios:
764
+ 1. No image, and there for, no image token should be added.
765
+ 2. image token is already specified in the context, so we don't need to add it.
766
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
767
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
768
+ """
769
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
770
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
771
+ # elif task_type == "video":
772
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
773
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
774
+ image_tokens = " ".join(image_tokens)
775
+ question = image_tokens + "\n" + context
776
+ else:
777
+ question = context
778
+
779
+ # This is much safer for llama3, as we now have some object type in it
780
+ if "llama_3" in self.conv_template:
781
+ conv = copy.deepcopy(conv_templates[self.conv_template])
782
+ else:
783
+ conv = conv_templates[self.conv_template].copy()
784
+
785
+ if utils.is_json(question): # conversational question input
786
+ question = json.loads(question)
787
+ for idx, item in enumerate(question):
788
+ role = conv.roles[idx % 2]
789
+ message = item["value"]
790
+ conv.append_message(role, message)
791
+
792
+ assert len(conv.messages) % 2 == 1
793
+ conv.append_message(conv.roles[1], None)
794
+ prompt_question = conv.get_prompt()
795
+ question_input.append(prompt_question)
796
+ else: # only simple string for question
797
+ conv.append_message(conv.roles[0], question)
798
+ conv.append_message(conv.roles[1], None)
799
+ prompt_question = conv.get_prompt()
800
+ question_input.append(prompt_question)
801
+
802
+ # preconfigure gen_kwargs with defaults
803
+ if "max_new_tokens" not in gen_kwargs:
804
+ gen_kwargs["max_new_tokens"] = 1024
805
+ if "temperature" not in gen_kwargs:
806
+ gen_kwargs["temperature"] = 0
807
+ if "do_sample" not in gen_kwargs:
808
+ gen_kwargs["do_sample"] = False
809
+ if "top_p" not in gen_kwargs:
810
+ gen_kwargs["top_p"] = None
811
+ if "num_beams" not in gen_kwargs:
812
+ gen_kwargs["num_beams"] = 1
813
+
814
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
815
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
816
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
817
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
818
+
819
+ if task_type == "image":
820
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
821
+ elif task_type == "video":
822
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
823
+ keywords = [stop_str]
824
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
825
+ gen_kwargs["modalities"] = ["video"]
826
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
827
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
828
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
829
+
830
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
831
+ # TODO: attention to this major generation step...
832
+ if "image_aspect_ratio" in gen_kwargs.keys():
833
+ gen_kwargs.pop("image_aspect_ratio")
834
+ try:
835
+ with torch.inference_mode():
836
+ cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
837
+ # cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
838
+
839
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
840
+ except Exception as e:
841
+ raise e
842
+
843
+ text_outputs = [response.strip() for response in text_outputs]
844
+ batched_round_res.append(text_outputs)
845
+
846
+ round_idx += 1
847
+
848
+ res.extend(list(zip(*batched_round_res)))
849
+ self.cache_hook.add_partial("generate_until_multi_round", (context, gen_kwargs), batched_round_res)
850
+ pbar.update(1)
851
+ # reorder this group of results back to original unsorted form
852
+ res = re_ords.get_original(res)
853
+
854
+ pbar.close()
855
+ return res
LaViDa/eval/lmms_eval/models/llava_hf.py ADDED
@@ -0,0 +1,390 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import numpy as np
5
+ import PIL
6
+ import torch
7
+ from accelerate import Accelerator, DistributedType
8
+ from accelerate.state import AcceleratorState
9
+ from decord import VideoReader, cpu
10
+ from tqdm import tqdm
11
+ from transformers import (
12
+ AutoConfig,
13
+ AutoProcessor,
14
+ LlavaForConditionalGeneration,
15
+ LlavaNextForConditionalGeneration,
16
+ )
17
+
18
+ from lmms_eval import utils
19
+ from lmms_eval.api.instance import Instance
20
+ from lmms_eval.api.model import lmms
21
+ from lmms_eval.api.registry import register_model
22
+
23
+ warnings.filterwarnings("ignore")
24
+
25
+ from loguru import logger as eval_logger
26
+
27
+ DEFAULT_IMAGE_TOKEN = "<image>"
28
+ DEFAULT_VIDEO_TOKEN = "<video>"
29
+
30
+ # Default chat for llava-hf/llava-1.5 models: https://huggingface.co/collections/llava-hf/llava-15-65f762d5b6941db5c2ba07e0
31
+ VICUNA_CHAT_TEMPLATE = "{% for message in messages %}{% if loop.index0 == 0 %}A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {{ message['content'] }} {% elif message['role'] == 'user' %}USER: {{ message['content'] }} {% else %} ASSISTANT: {{ message['content'] }}{{ eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}"
32
+
33
+ model_map = {
34
+ "llava": LlavaForConditionalGeneration,
35
+ "llava_next": LlavaNextForConditionalGeneration,
36
+ }
37
+
38
+ try:
39
+ from transformers import LlavaOnevisionForConditionalGeneration
40
+
41
+ model_map["llava_onevision"] = LlavaOnevisionForConditionalGeneration
42
+ except Exception as e:
43
+ eval_logger.debug("Transformers version does not support llava-onevision. Skipping.")
44
+
45
+
46
+ @register_model("llava_hf")
47
+ class LlavaHf(lmms):
48
+ """
49
+ Llava Model for Hugging Face Transformers: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava
50
+
51
+ Adapted from the InstructBLIP model in lmms_eval/models/instructblip.py
52
+
53
+ Example usage:
54
+
55
+ accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
56
+ --model llava_hf \
57
+ --model_args pretrained=llava-hf/llava-1.5-7b-hf \
58
+ --tasks seedbench \
59
+ --batch_size 1 \
60
+ --output_path ./logs/ \
61
+ --log_samples
62
+ """
63
+
64
+ def __init__(
65
+ self,
66
+ pretrained: str = "llava-hf/llava-1.5-7b-hf",
67
+ revision: str = "main",
68
+ device: str = "cuda",
69
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
70
+ batch_size: int = 1,
71
+ trust_remote_code: Optional[bool] = False,
72
+ attn_implementation: Optional[str] = None,
73
+ device_map: str = "",
74
+ chat_template: Optional[str] = None,
75
+ use_cache: bool = True,
76
+ max_frames_num: Optional[int] = 32,
77
+ **kwargs,
78
+ ) -> None:
79
+ super().__init__()
80
+ # Do not use kwargs for now
81
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
82
+
83
+ accelerator = Accelerator()
84
+ if accelerator.num_processes > 1 and device_map == "":
85
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
86
+ self.device_map = f"cuda:{accelerator.local_process_index}"
87
+ else:
88
+ self._device = torch.device(device)
89
+ self.device_map = device_map
90
+ if isinstance(dtype, str) and dtype != "auto":
91
+ dtype = getattr(torch, dtype)
92
+
93
+ config = AutoConfig.from_pretrained(pretrained)
94
+ self.max_frames_num = max_frames_num
95
+ model_type = getattr(config, "model_type", "llava")
96
+ model_type = model_map[model_type]
97
+ self._model = model_type.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
98
+
99
+ self.pretrained = pretrained
100
+ self._image_processor = AutoProcessor.from_pretrained(pretrained, revision=revision, trust_remote_code=trust_remote_code)
101
+ # Pad from left for batched generation: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava#usage-tips
102
+ self._image_processor.tokenizer.padding_side = "left"
103
+ self._tokenizer = self._image_processor.tokenizer
104
+ self._config = self._model.config
105
+ self.batch_size_per_gpu = int(batch_size)
106
+ self.chat_template = chat_template
107
+ self.use_cache = use_cache
108
+ if accelerator.num_processes > 1 and device_map == "":
109
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
110
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
111
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
112
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
113
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
114
+ kwargs = {
115
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
116
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
117
+ }
118
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
119
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
120
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
121
+ self._model = accelerator.prepare(self.model)
122
+ else:
123
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
124
+ self.accelerator = accelerator
125
+ if self.accelerator.is_local_main_process:
126
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
127
+ self._rank = self.accelerator.local_process_index
128
+ self._world_size = self.accelerator.num_processes
129
+ elif accelerator.num_processes == 1 and device_map == "auto":
130
+ eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
131
+ self._rank = 0
132
+ self._world_size = 1
133
+ else:
134
+ eval_logger.info(f"Using single device: {self._device}")
135
+ self.model.to(self._device)
136
+ self._rank = 0
137
+ self._world_size = 1
138
+ self.accelerator = accelerator
139
+
140
+ @property
141
+ def config(self):
142
+ # return the associated transformers.AutoConfig for the given pretrained model.
143
+ return self._config
144
+
145
+ @property
146
+ def tokenizer(self):
147
+ return self._tokenizer
148
+
149
+ @property
150
+ def model(self):
151
+ # returns the model, unwrapping it if using Accelerate
152
+ if hasattr(self, "accelerator"):
153
+ return self.accelerator.unwrap_model(self._model)
154
+ else:
155
+ return self._model
156
+
157
+ @property
158
+ def eot_token_id(self):
159
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
160
+ return self.tokenizer.eos_token_id
161
+
162
+ @property
163
+ def max_length(self):
164
+ return self._max_length
165
+
166
+ @property
167
+ def batch_size(self):
168
+ return self.batch_size_per_gpu
169
+
170
+ @property
171
+ def device(self):
172
+ return self._device
173
+
174
+ @property
175
+ def rank(self):
176
+ return self._rank
177
+
178
+ @property
179
+ def world_size(self):
180
+ return self._world_size
181
+
182
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
183
+ """ """
184
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
185
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
186
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
187
+ if left_truncate_len:
188
+ encoding = encoding[-left_truncate_len:]
189
+ return encoding
190
+
191
+ def tok_decode(self, tokens):
192
+ return self.tokenizer.decode(tokens)
193
+
194
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
195
+ res = []
196
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
197
+
198
+ for context, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
199
+ # encode, pad, and truncate contexts for this batch
200
+ if type(doc_to_target) == str:
201
+ continuation = doc_to_target
202
+ else:
203
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
204
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
205
+ visuals = self.flatten(visuals)
206
+
207
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
208
+ image_tokens = " ".join(image_tokens)
209
+ context = f"{image_tokens}\n{context}"
210
+ # Apply chat template
211
+ messages = [{"role": "user", "content": context}, {"role": "assistant", "content": continuation}]
212
+ if self.chat_template is not None:
213
+ self.tokenizer.chat_template = self.chat_template
214
+ prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
215
+ prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
216
+ elif self.tokenizer.chat_template is not None:
217
+ prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
218
+ prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
219
+ else:
220
+ self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
221
+ prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
222
+ prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
223
+
224
+ formatted_contexts = [prompt]
225
+ formatted_continuation = [prompt_and_continuation]
226
+ model_inputs = self._image_processor(text=formatted_continuation, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
227
+ labels = model_inputs["input_ids"].clone()
228
+ contxt_id = self._image_processor(text=formatted_contexts, return_tensors="pt")["input_ids"]
229
+ labels[:, : contxt_id.shape[1]] = -100
230
+
231
+ if self.accelerator.is_main_process and doc_id % 100 == 0:
232
+ eval_logger.debug(f"Prompt for doc ID {doc_id}:\n\n{formatted_contexts[0]}\n")
233
+ eval_logger.debug(f"Prompt and continuation for doc ID {doc_id}:\n\n{formatted_continuation[0]}\n")
234
+
235
+ with torch.inference_mode():
236
+ outputs = self.model(**model_inputs, labels=labels)
237
+ loss = outputs["loss"]
238
+ logits = outputs["logits"]
239
+ greedy_tokens = logits.argmax(dim=-1)
240
+ cont_toks = model_inputs["input_ids"][:, contxt_id.shape[1] :] # [1, seq]
241
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : model_inputs["input_ids"].shape[1]] # [1, seq]
242
+ max_equal = (greedy_tokens == cont_toks).all()
243
+ res.append((float(loss.item()), bool(max_equal)))
244
+ pbar.update(1)
245
+
246
+ pbar.close()
247
+ return res
248
+
249
+ def flatten(self, input):
250
+ new_list = []
251
+ for i in input:
252
+ for j in i:
253
+ new_list.append(j)
254
+ return new_list
255
+
256
+ def load_video(self, video_path, max_frames_num):
257
+ if type(video_path) == str:
258
+ vr = VideoReader(video_path, ctx=cpu(0))
259
+ else:
260
+ vr = VideoReader(video_path[0], ctx=cpu(0))
261
+ total_frame_num = len(vr)
262
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
263
+ frame_idx = uniform_sampled_frames.tolist()
264
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
265
+ return spare_frames # (frames, height, width, channels)
266
+
267
+ def generate_until(self, requests: List[Instance]) -> List[str]:
268
+ res = []
269
+
270
+ def _collate(x):
271
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
272
+ # - time estimates will always be over not underestimates, which is more useful for planning
273
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
274
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
275
+ # automatic adaptive batches much much easier to implement
276
+ # - any OOMs will happen right away rather than near the end
277
+ toks = self.tok_encode(x[0])
278
+ return -len(toks), x[0]
279
+
280
+ # we group requests by their generation_kwargs,
281
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
282
+ # in the same batch.
283
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
284
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
285
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
286
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
287
+ for chunk in chunks:
288
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
289
+ task = task[0]
290
+ split = split[0]
291
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
292
+ visuals = self.flatten(visuals)
293
+ if len(visuals) == 0:
294
+ task_type = "text"
295
+ elif isinstance(visuals[0], PIL.Image.Image):
296
+ task_type = "image"
297
+ elif isinstance(visuals[0], str):
298
+ task_type = "video"
299
+ # we assume all gen kwargs in the batch are the same
300
+ # this is safe to assume because the `grouper` object ensures it.
301
+ gen_kwargs = all_gen_kwargs[0]
302
+
303
+ # Set default values for until and max_new_tokens
304
+ until = [self.tok_decode(self.eot_token_id)]
305
+
306
+ # Update values from gen_kwargs if present
307
+ if "until" in gen_kwargs:
308
+ until = gen_kwargs.pop("until")
309
+ if isinstance(until, str):
310
+ until = [until]
311
+ elif not isinstance(until, list):
312
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
313
+ assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
314
+ context = contexts[0]
315
+
316
+ # Some benchmarks like MME do not contain image tokens, so we prepend them to the prompt.
317
+ if DEFAULT_IMAGE_TOKEN not in context:
318
+ if task_type == "image":
319
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
320
+ elif task_type == "video":
321
+ image_tokens = [DEFAULT_VIDEO_TOKEN] * len(visuals)
322
+ image_tokens = " ".join(image_tokens)
323
+ context = f"{image_tokens}\n{context}"
324
+ # Apply chat template
325
+ messages = [{"role": "user", "content": context}]
326
+ if self.chat_template is not None:
327
+ self.tokenizer.chat_template = self.chat_template
328
+ text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
329
+ elif self.tokenizer.chat_template is not None:
330
+ text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
331
+ else:
332
+ self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
333
+ text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
334
+
335
+ if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
336
+ eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
337
+
338
+ if task_type == "video":
339
+ try:
340
+ visuals = [self.load_video(visuals, self.max_frames_num)]
341
+ except Exception as e:
342
+ res.append("")
343
+ eval_logger.info(f"Error {e} when loading video : {visuals}")
344
+ pbar.update(1)
345
+
346
+ if task_type == "image":
347
+ inputs = self._image_processor(images=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
348
+ elif task_type == "video":
349
+ inputs = self._image_processor(videos=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
350
+
351
+ gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
352
+ if "max_new_tokens" not in gen_kwargs:
353
+ gen_kwargs["max_new_tokens"] = 1024
354
+ if "temperature" not in gen_kwargs:
355
+ gen_kwargs["temperature"] = 0
356
+ if "top_p" not in gen_kwargs:
357
+ gen_kwargs["top_p"] = None
358
+ if "num_beams" not in gen_kwargs:
359
+ gen_kwargs["num_beams"] = 1
360
+ try:
361
+ cont = self.model.generate(
362
+ **inputs,
363
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
364
+ temperature=gen_kwargs["temperature"],
365
+ top_p=gen_kwargs["top_p"],
366
+ num_beams=gen_kwargs["num_beams"],
367
+ max_new_tokens=gen_kwargs["max_new_tokens"],
368
+ use_cache=self.use_cache,
369
+ pad_token_id=self.eot_token_id,
370
+ eos_token_id=self.eot_token_id,
371
+ )
372
+ cont = cont[:, inputs["input_ids"].shape[-1] :]
373
+ except Exception as e:
374
+ eval_logger.error(f"Error {e} in generating")
375
+ cont = ""
376
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0]
377
+ if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
378
+ eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n")
379
+
380
+ res.append(text_outputs)
381
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
382
+ pbar.update(1)
383
+ # reorder this group of results back to original unsorted form
384
+ res = re_ords.get_original(res)
385
+
386
+ pbar.close()
387
+ return res
388
+
389
+ def generate_until_multi_round(self, requests) -> List[str]:
390
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
LaViDa/eval/lmms_eval/models/llava_llada.py ADDED
@@ -0,0 +1,894 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import json
3
+ import logging
4
+ import math
5
+ import re
6
+ import warnings
7
+ from datetime import timedelta
8
+ from typing import List, Optional, Tuple, Union
9
+
10
+ import numpy as np
11
+ import os
12
+ import PIL
13
+ import torch
14
+ import accelerate
15
+ import transformers
16
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
17
+ from accelerate.state import AcceleratorState
18
+ from decord import VideoReader, cpu
19
+ from packaging import version
20
+ from tqdm import tqdm
21
+ from transformers import AutoConfig
22
+
23
+ from lmms_eval import utils
24
+ from lmms_eval.api.instance import Instance
25
+ from lmms_eval.api.model import lmms
26
+ from lmms_eval.api.registry import register_model
27
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
28
+ import time
29
+ # Suppress warnings
30
+ warnings.filterwarnings("ignore")
31
+
32
+ # Configure logging
33
+ eval_logger = logging.getLogger("lmms-eval")
34
+
35
+ # Enable TF32 for CUDA
36
+ torch.backends.cuda.matmul.allow_tf32 = True
37
+ DEBUG_PRINT_OUTPUT = os.environ.get('DEBUG_PRINT_OUTPUT',False)
38
+
39
+
40
+ def _shuffle_video_tensor(video_tensor: torch.Tensor, seed: int | None = None) -> torch.Tensor:
41
+ if video_tensor.ndim < 1 or video_tensor.shape[0] <= 1:
42
+ return video_tensor.clone()
43
+ generator = torch.Generator(device="cpu")
44
+ if seed is not None:
45
+ generator.manual_seed(int(seed))
46
+ perm = torch.randperm(video_tensor.shape[0], generator=generator)
47
+ return video_tensor[perm.to(video_tensor.device)]
48
+ # Import LLaVA modules
49
+ DEBUG_LOAD_TRAINER = os.environ.get('DEBUG_LOAD_TRAINER',False)
50
+ try:
51
+ from llava.constants import (
52
+ DEFAULT_IM_END_TOKEN,
53
+ DEFAULT_IM_START_TOKEN,
54
+ DEFAULT_IMAGE_TOKEN,
55
+ IGNORE_INDEX,
56
+ IMAGE_TOKEN_INDEX,
57
+ )
58
+ from llava.conversation import SeparatorStyle, conv_templates
59
+ from llava.mm_utils import (
60
+ KeywordsStoppingCriteria,
61
+ get_model_name_from_path,
62
+ process_images,
63
+ tokenizer_image_token,
64
+ )
65
+ from llava.model.builder import load_pretrained_model
66
+ except ImportError as e:
67
+ eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
68
+
69
+
70
+ # Determine best attention implementation
71
+ if version.parse(torch.__version__) >= version.parse("2.1.2"):
72
+ best_fit_attn_implementation = "sdpa"
73
+ else:
74
+ best_fit_attn_implementation = "eager"
75
+
76
+
77
+ @register_model("llava_llada")
78
+ class Llava_Llada(lmms):
79
+ """
80
+ Llava Model
81
+ """
82
+
83
+ def __init__(
84
+ self,
85
+ pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
86
+ truncation: Optional[bool] = True,
87
+ device: Optional[str] = "cuda:0",
88
+ batch_size: Optional[Union[int, str]] = 1,
89
+ model_name: Optional[str] = None,
90
+ attn_implementation: Optional[str] = best_fit_attn_implementation,
91
+ device_map: Optional[str] = "cuda:0",
92
+ conv_template: Optional[str] = "llava_llada",
93
+ use_cache: Optional[bool] = True,
94
+ truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
95
+ customized_config: Optional[str] = None, # ends in json
96
+ max_frames_num: Optional[int] = 32,
97
+ mm_spatial_pool_stride: Optional[int] = 2,
98
+ mm_spatial_pool_mode: Optional[str] = "bilinear",
99
+ token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
100
+ video_decode_backend: str = "decord",
101
+ mc_num=16,
102
+ **kwargs,
103
+ ) -> None:
104
+ super().__init__()
105
+ # Do not use kwargs for now
106
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
107
+
108
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
109
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
110
+ if accelerator.num_processes > 1:
111
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
112
+ self.device_map = f"cuda:{accelerator.local_process_index}"
113
+ elif accelerator.num_processes == 1 and device_map == "auto":
114
+ self._device = torch.device(device)
115
+ self.device_map = device_map
116
+ else:
117
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
118
+ self.device_map = f"cuda:{accelerator.local_process_index}"
119
+ self.mc_num = mc_num
120
+ llava_model_args = {
121
+ "multimodal": True,
122
+ }
123
+ if customized_config is not None:
124
+ llava_model_args["customized_config"] = customized_config
125
+ if attn_implementation is not None:
126
+ llava_model_args["attn_implementation"] = attn_implementation
127
+ if "use_flash_attention_2" in kwargs:
128
+ llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
129
+ model_name = 'llava_llada'# if model_name is not None else get_model_name_from_path(pretrained)
130
+ self.overwrite_image_aspect = os.environ.get("LLAVA_OVERWRITE_IMAGE_ASPECT", None)
131
+ self.pretrained = pretrained
132
+ self.token_strategy = token_strategy
133
+ self.max_frames_num = max_frames_num
134
+ self.mm_spatial_pool_stride = mm_spatial_pool_stride
135
+ self.mm_spatial_pool_mode = mm_spatial_pool_mode
136
+ self.video_decode_backend = video_decode_backend
137
+
138
+ overwrite_config = {}
139
+ overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
140
+ overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
141
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
142
+
143
+ llava_model_args["overwrite_config"] = overwrite_config
144
+ # try:
145
+ # Try to load the model with the multimodal argument
146
+
147
+ if os.path.exists('/data1/jacklishufan/siglip-so400m-patch14-384'):
148
+ vision_tower_path = "/data1/jacklishufan/siglip-so400m-patch14-384"
149
+ else:
150
+ vision_tower_path="/data0/jacklishufan/siglip-so400m-patch14-384"
151
+ # print(vision_tower_path)
152
+ vision_kwargs = dict(
153
+ mm_vision_tower=os.environ.get('LLADA_VISION_ENCODER',vision_tower_path),
154
+ mm_resampler_type=None,
155
+ mm_projector_type=os.environ.get('LLADA_VISION_PROJECTOR','mlp2x_gelu'),
156
+ mm_hidden_size=int(os.environ.get('LLADA_VISION_ENCODER_HIDDEN_SIZE',1152)),
157
+ mm_pooler_ratio=int(os.environ.get('LLADA_MM_POOLER_RATIO',2)),
158
+ use_mm_proj=True,
159
+ mm_patch_merge_type='spatial_unpad',
160
+ )
161
+ resize_embeddings = True # default behavior
162
+ if DEBUG_LOAD_TRAINER:
163
+ resize_embeddings = False
164
+
165
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args,vision_kwargs=vision_kwargs,resize_embeddings=resize_embeddings)
166
+ # breakpoint()
167
+ assert self._tokenizer is not None
168
+ # except TypeError:
169
+ # # for older versions of LLaVA that don't have multimodal argument
170
+ # llava_model_args.pop("multimodal", None)
171
+ # self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
172
+
173
+ self._config = self._model.config
174
+ self.model.eval()
175
+ self.model.model.set_activation_checkpointing(None)
176
+ self.model.requires_grad_(False)
177
+ self.truncation = truncation
178
+ self.batch_size_per_gpu = int(batch_size)
179
+ self.conv_template = conv_template
180
+ self.use_cache = use_cache
181
+ self.truncate_context = truncate_context
182
+ assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
183
+
184
+ if accelerator.num_processes > 1:
185
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
186
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
187
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
188
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
189
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
190
+ kwargs = {
191
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
192
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
193
+ }
194
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
195
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
196
+
197
+ # if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
198
+ # self._model = accelerator.prepare(self.model)
199
+ # else:
200
+ # self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
201
+ self.model.to(self._device).to(torch.bfloat16)
202
+ self._model.model.transformer = accelerator.prepare(self.model.model.transformer)
203
+
204
+ self.accelerator = accelerator
205
+ if self.accelerator.is_local_main_process:
206
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
207
+ self._rank = self.accelerator.local_process_index
208
+ self._world_size = self.accelerator.num_processes
209
+
210
+ elif accelerator.num_processes == 1 and device_map == "auto":
211
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
212
+ self._rank = 0
213
+ self._world_size = 1
214
+
215
+ else:
216
+ eval_logger.info(f"Using single device: {self._device}")
217
+ self.model.to(self._device).to(torch.bfloat16)
218
+ self._rank = 0
219
+ self._world_size = 1
220
+ #self.model.model.transformer = accelerate.cpu_offload(self.model.model.transformer)
221
+
222
+ @property
223
+ def config(self):
224
+ # return the associated transformers.AutoConfig for the given pretrained model.
225
+ return self._config
226
+
227
+ @property
228
+ def tokenizer(self):
229
+ return self._tokenizer
230
+
231
+ @property
232
+ def model(self):
233
+ # returns the model, unwrapping it if using Accelerate
234
+ if hasattr(self, "accelerator"):
235
+ return self.accelerator.unwrap_model(self._model)
236
+ else:
237
+ return self._model
238
+
239
+ @property
240
+ def eot_token_id(self):
241
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
242
+ return self.tokenizer.eos_token_id
243
+
244
+ @property
245
+ def max_length(self):
246
+ return self._max_length
247
+
248
+ def pad_sequence(self, input_ids, batch_first, padding_value):
249
+ if self.tokenizer.padding_side == "left":
250
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
251
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
252
+ if self.tokenizer.padding_side == "left":
253
+ input_ids = torch.flip(input_ids, [1])
254
+ return input_ids
255
+
256
+ @property
257
+ def batch_size(self):
258
+ return self.batch_size_per_gpu
259
+
260
+ @property
261
+ def device(self):
262
+ return self._device
263
+
264
+ @property
265
+ def rank(self):
266
+ return self._rank
267
+
268
+ @property
269
+ def world_size(self):
270
+ return self._world_size
271
+
272
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
273
+ """ """
274
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
275
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
276
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
277
+ if left_truncate_len:
278
+ encoding = encoding[-left_truncate_len:]
279
+ return encoding
280
+
281
+ def tok_decode(self, tokens):
282
+ try:
283
+ return self.tokenizer.decode(tokens)
284
+ except:
285
+ return self.tokenizer.decode([tokens])
286
+
287
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
288
+ res = []
289
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
290
+
291
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
292
+
293
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
294
+ visual = doc_to_visual(self.task_dict[task][split][doc_id])
295
+
296
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
297
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
298
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
299
+
300
+ if visual is None or visual == []:
301
+ visual = None
302
+ task_type = "text"
303
+ image_tensor = None
304
+ else:
305
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__:
306
+ self._config.image_aspect_ratio = "pad"
307
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
308
+
309
+ if "task_type" in self.metadata and self.metadata["task_type"] == "video" and "sample_frames" in self.metadata:
310
+ assert type(visual) == list, "sample_frames must be specified for video task"
311
+ sample_indices = np.linspace(0, len(visual) - 1, self.metadata["sample_frames"], dtype=int)
312
+ visual = [visual[i] for i in sample_indices]
313
+ assert len(visual) == self.metadata["sample_frames"]
314
+
315
+ image_tensor = process_images(visual, self._image_processor, self._config)
316
+ if type(image_tensor) is list:
317
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
318
+ else:
319
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
320
+
321
+ task_type = "video"
322
+
323
+ # elif type(visual[0]) == PIL.Image.Image:
324
+ elif isinstance(visual[0], PIL.Image.Image):
325
+ image_tensor = process_images(visual, self._image_processor, self._config)
326
+ if type(image_tensor) is list:
327
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
328
+ else:
329
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
330
+
331
+ task_type = "image"
332
+
333
+ elif type(visual[0]) == str:
334
+ image_tensor = []
335
+ try:
336
+ if self.video_decode_backend == "decord":
337
+ frames = self.load_video(visual, self.max_frames_num)
338
+ elif self.video_decode_backend == "pyav":
339
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
340
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
341
+ image_tensor.append(frames)
342
+ except Exception as e:
343
+ eval_logger.error(f"Error {e} in loading video")
344
+ image_tensor = None
345
+
346
+ task_type = "video"
347
+
348
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in contexts:
349
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
350
+ if task_type == "video":
351
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
352
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
353
+ image_tokens = " ".join(image_tokens)
354
+ prompts_input = image_tokens + "\n" + contexts
355
+ else:
356
+ prompts_input = contexts
357
+
358
+
359
+ if "llama_3" in self.conv_template or 'llada' in self.conv_template:
360
+ conv = copy.deepcopy(conv_templates[self.conv_template])
361
+ else:
362
+ conv = conv_templates[self.conv_template].copy()
363
+
364
+ conv.append_message(conv.roles[0], prompts_input)
365
+ conv.append_message(conv.roles[1], None)
366
+ prompt = conv.get_prompt()
367
+
368
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
369
+
370
+ if type(doc_to_target) == str:
371
+ continuation = doc_to_target
372
+ else:
373
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
374
+
375
+ # conv.messages[-1][1] = continuation
376
+ # full_prompt = conv.get_prompt()
377
+ # full_input_ids = tokenizer_image_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
378
+
379
+ # labels = full_input_ids.clone()
380
+ # labels[0, : input_ids.shape[1]] = -100
381
+ input_prompt = conv.get_prompt()
382
+ input_ids = tokenizer_image_token(input_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
383
+ answers = continuation
384
+ answer_ids = self.tokenizer(continuation)['input_ids']
385
+ answer_ids = torch.tensor(continuation).to(input_ids.device).unsqueeze(0)
386
+
387
+
388
+ kwargs = {}
389
+ if task_type == "image":
390
+ kwargs["image_sizes"] = [[v.size[0], v.size[1]] for v in visual] if isinstance(visual, list) else [[visual.size[0], visual.size[1]]]
391
+ elif task_type == "video":
392
+ kwargs["modalities"] = ["video"]
393
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
394
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
395
+
396
+ torch.cuda.empty_cache()
397
+ # with torch.inference_mode():
398
+ #outputs = self.model(input_ids=full_input_ids, labels=labels, images=image_tensor, use_cache=True, **kwargs)
399
+ likelyhoods = self.model.log_likelyhood_inference(
400
+ input_ids,
401
+ images=image_tensor.to(torch.bfloat16),
402
+ image_sizes=None,
403
+ verbose=True,
404
+ answer=answer_ids,
405
+ mc_num=self.mc_num,
406
+ )
407
+
408
+ # loss = outputs["loss"]
409
+ # logits = outputs["logits"]
410
+ # greedy_tokens = logits.argmax(dim=-1)
411
+ # cont_toks = full_input_ids[:, input_ids.shape[1] :]
412
+ # greedy_tokens = greedy_tokens[:, input_ids.shape[1] : full_input_ids.shape[1]]
413
+ # max_equal = (greedy_tokens == cont_toks).all()
414
+ # lmms eval return loss
415
+ res.append((float(-likelyhoods.item()), False))
416
+ pbar.update(1)
417
+
418
+ pbar.close()
419
+ return res
420
+
421
+ def flatten(self, input):
422
+ if not input or any(i is None for i in input):
423
+ return []
424
+ new_list = []
425
+ for i in input:
426
+ if i:
427
+ for j in i:
428
+ new_list.append(j)
429
+ return new_list
430
+
431
+ def load_video(self, video_path, max_frames_num):
432
+ if type(video_path) == str:
433
+ vr = VideoReader(video_path, ctx=cpu(0))
434
+ else:
435
+ vr = VideoReader(video_path[0], ctx=cpu(0))
436
+ total_frame_num = len(vr)
437
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
438
+ frame_idx = uniform_sampled_frames.tolist()
439
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
440
+ return spare_frames # (frames, height, width, channels)
441
+
442
+ def generate_until(self, requests: List[Instance]) -> List[str]:
443
+ res = []
444
+
445
+ def _collate(x):
446
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
447
+ # - time estimates will always be over not underestimates, which is more useful for planning
448
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
449
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
450
+ # automatic adaptive batches much much easier to implement
451
+ # - any OOMs will happen right away rather than near the end
452
+ toks = self.tok_encode(x[0])
453
+ # breakpoint()
454
+ return -len(toks), x[0]
455
+
456
+ # we group requests by their generation_kwargs,
457
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
458
+ # in the same batch.
459
+ metadata = requests[0].metadata
460
+ if DEBUG_PRINT_OUTPUT:
461
+ # do not sort by length, instead using lambda x:x[-3]
462
+ re_ords = utils.Collator([reg.args for reg in requests], lambda x:x[-3], grouping=True)
463
+ else:
464
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
465
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
466
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
467
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
468
+
469
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
470
+ if DEBUG_LOAD_TRAINER:
471
+ ckpt1 = torch.load(DEBUG_LOAD_TRAINER, map_location='cpu')
472
+ ckpt1 = {k.replace('module.model','model'):v for k,v in ckpt1.items()}
473
+ _res = self.model.load_state_dict(ckpt1,strict=False)
474
+ print(f"DEBUG_LOAD_TRAINER:{DEBUG_LOAD_TRAINER} {_res}")
475
+ print("Something is broken if above line does not show all keys matched!!!")
476
+ del ckpt1
477
+ delta_t = 0
478
+ num_generated = 0
479
+ for chunk in chunks:
480
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
481
+ task = batched_task[0]
482
+ split = batched_split[0]
483
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
484
+ assert len(batched_visuals) == 1
485
+
486
+ # we assume all gen kwargs in the batch are the same
487
+ # this is safe to assume because the `grouper` object ensures it.
488
+ gen_kwargs = all_gen_kwargs[0]
489
+ if "until" in gen_kwargs:
490
+ gen_kwargs.pop("until")
491
+
492
+ question_input = []
493
+ # import ipdb; ipdb.set_trace()
494
+
495
+ for visual, context in zip(batched_visuals, batched_contexts):
496
+ t0 = time.time()
497
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
498
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
499
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
500
+ if self.overwrite_image_aspect:
501
+ self._config.image_aspect_ratio = self.overwrite_image_aspect
502
+ if visual is None or visual == []: # for text-only tasks.
503
+ visual = None
504
+ task_type = "text"
505
+ placeholder_count = 0
506
+ image_tensor = None
507
+ else:
508
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
509
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
510
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
511
+
512
+ if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
513
+ assert type(visual) == list, "sample_frames must be specified for video task"
514
+ sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
515
+ visual = [visual[i] for i in sample_indices]
516
+ assert len(visual) == metadata["sample_frames"]
517
+
518
+ image_tensor = process_images(visual, self._image_processor, self._config)
519
+ if type(image_tensor) is list:
520
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
521
+ else:
522
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
523
+
524
+ task_type = "video"
525
+ placeholder_count = 1
526
+
527
+ elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
528
+ image_tensor = process_images(visual, self._image_processor, self._config)
529
+ if type(image_tensor) is list:
530
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
531
+ else:
532
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
533
+
534
+ task_type = "image"
535
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
536
+
537
+ elif type(visual[0]) == str: # For video task
538
+ image_tensor = []
539
+ try:
540
+ if self.video_decode_backend == "decord":
541
+ frames = self.load_video(visual, self.max_frames_num)
542
+ elif self.video_decode_backend == "pyav":
543
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
544
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
545
+ image_tensor.append(frames)
546
+ except Exception as e:
547
+ eval_logger.error(f"Error {e} in loading video")
548
+ image_tensor = None
549
+
550
+ task_type = "video"
551
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
552
+
553
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
554
+ """
555
+ Three senarios:
556
+ 1. No image, and there for, no image token should be added.
557
+ 2. image token is already specified in the context, so we don't need to add it.
558
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
559
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
560
+ """
561
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
562
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
563
+ # elif task_type == "video":
564
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
565
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
566
+ image_tokens = " ".join(image_tokens)
567
+ question = image_tokens + "\n" + context
568
+ else:
569
+ question = context
570
+
571
+ # This is much safer for llama3, as we now have some object type in it
572
+ # print(self.conv_template)
573
+ if "llama_3" in self.conv_template or 'llada' in self.conv_template:
574
+ conv = copy.deepcopy(conv_templates[self.conv_template])
575
+ else:
576
+ conv = conv_templates[self.conv_template].copy()
577
+
578
+ if utils.is_json(question): # conversational question input
579
+ question = json.loads(question)
580
+ for idx, item in enumerate(question):
581
+ role = conv.roles[idx % 2]
582
+ message = item["value"]
583
+ conv.append_message(role, message)
584
+
585
+ assert len(conv.messages) % 2 == 1
586
+ conv.append_message(conv.roles[1], None)
587
+ prompt_question = conv.get_prompt()
588
+ question_input.append(prompt_question)
589
+ else: # only simple string for question
590
+ conv.append_message(conv.roles[0], question)
591
+ conv.append_message(conv.roles[1], None)
592
+ # breakpoint()
593
+ prompt_question = conv.get_prompt()
594
+ question_input.append(prompt_question)
595
+
596
+ # preconfigure gen_kwargs with defaults
597
+ if "max_new_tokens" not in gen_kwargs:
598
+ gen_kwargs["max_new_tokens"] = 256
599
+
600
+ if "temperature" not in gen_kwargs:
601
+ gen_kwargs["temperature"] = 0
602
+ if "do_sample" not in gen_kwargs:
603
+ gen_kwargs["do_sample"] = False
604
+ if "top_p" not in gen_kwargs:
605
+ gen_kwargs["top_p"] = None
606
+ if "num_beams" not in gen_kwargs:
607
+ gen_kwargs["num_beams"] = 1
608
+ schedule_kwargs = {}
609
+ for key in list(gen_kwargs.keys()):
610
+ if key.startswith('schedule__'):
611
+ value = gen_kwargs.pop(key)
612
+ schedule_kwargs[key.replace('schedule__','')] = value
613
+ if len(schedule_kwargs) > 0:
614
+ gen_kwargs['schedule_kwargs'] = schedule_kwargs
615
+
616
+ if 'block_length' not in gen_kwargs:
617
+ gen_kwargs['block_length'] = min(128,gen_kwargs["max_new_tokens"])
618
+ if 'step_per_block' not in gen_kwargs and 'step_ratio' not in gen_kwargs:
619
+ gen_kwargs['step_per_block'] = gen_kwargs['block_length']
620
+ gen_kwargs["temperature"] = 0
621
+
622
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
623
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
624
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
625
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
626
+
627
+ if task_type == "image":
628
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
629
+ elif task_type == "video":
630
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
631
+ keywords = [stop_str]
632
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
633
+ gen_kwargs["modalities"] = ["video"]
634
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
635
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
636
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
637
+ if gen_kwargs.pop("use_tvcd", False):
638
+ shuffle_seed = gen_kwargs.pop("tvcd_shuffle_seed", 42)
639
+ gen_kwargs["images_cd"] = [_shuffle_video_tensor(image_tensor[0], shuffle_seed)]
640
+
641
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
642
+ # TODO: attention to this major generation step...
643
+ # breakpoint()
644
+ if "image_aspect_ratio" in gen_kwargs.keys():
645
+ gen_kwargs.pop("image_aspect_ratio")
646
+ try:
647
+ with torch.inference_mode():
648
+ cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
649
+ # cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
650
+
651
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
652
+ text_outputs = [text_output.lstrip('!') for text_output in text_outputs]
653
+ except Exception as e:
654
+ raise e
655
+ # with open('/data1/jacklishufan/lmms-eval.pt', 'wb') as f:
656
+ # torch.save(self.model.state_dict(), f)
657
+ # print('saved')
658
+ # print(1/0)
659
+ t1 = time.time()
660
+ delta_t += t1-t0
661
+ num_generated +=1
662
+ print(f"Avg Latency (of {num_generated}): {delta_t/num_generated}")
663
+ if DEBUG_PRINT_OUTPUT:
664
+ print(f'\n--------Start of Sample {batched_doc_id[0]}---------')
665
+ print("Question: ",prompt_question)
666
+ print("Answer: ",text_outputs)
667
+ print("Answer: ",gen_kwargs)
668
+ print('--------End---------')
669
+
670
+ text_outputs = [response.strip() for response in text_outputs]
671
+ res.extend(text_outputs)
672
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
673
+ pbar.update(1)
674
+ # reorder this group of results back to original unsorted form
675
+ res = re_ords.get_original(res)
676
+
677
+ pbar.close()
678
+ return res
679
+
680
+ def generate_until_multi_round(self, requests: List[Instance]) -> List[str]:
681
+ res = []
682
+ raise NotImplementedError()
683
+
684
+ def _collate(x):
685
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
686
+ # - time estimates will always be over not underestimates, which is more useful for planning
687
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
688
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
689
+ # automatic adaptive batches much much easier to implement
690
+ # - any OOMs will happen right away rather than near the end
691
+ toks = self.tok_encode(x[0])
692
+ return -len(toks), x[0]
693
+
694
+ # we group requests by their generation_kwargs,
695
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
696
+ # in the same batch.
697
+ metadata = requests[0].metadata
698
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
699
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
700
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
701
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
702
+
703
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
704
+
705
+ for chunk in chunks:
706
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_to_text, batched_doc_id, batched_task, batched_split = zip(*chunk)
707
+ task = batched_task[0]
708
+ split = batched_split[0]
709
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
710
+ assert len(batched_visuals) == 1
711
+
712
+ # we assume all gen kwargs in the batch are the same
713
+ # this is safe to assume because the `grouper` object ensures it.
714
+ gen_kwargs = all_gen_kwargs[0]
715
+ if "until" in gen_kwargs:
716
+ gen_kwargs.pop("until")
717
+
718
+ # multi round inference: terminate when receiving signal from the doc_to_text
719
+ round_idx = 0
720
+ batched_round_res = []
721
+ batched_previous_round_info = None
722
+ while True:
723
+ question_input = []
724
+
725
+ if round_idx != 0: # get current round visual and context from doc_to_text function
726
+ batched_visuals, batched_contexts, batched_terminal_singal, batched_round_res, batched_previous_round_info = list(
727
+ zip(
728
+ *[
729
+ batched_doc_to_text[0](
730
+ self.task_dict[task][split][ids],
731
+ previous_output=[round_res[ids_idx] for round_res in batched_round_res],
732
+ round_idx=round_idx,
733
+ previous_round_info=batched_previous_round_info[ids_idx] if batched_previous_round_info is not None else None,
734
+ )
735
+ for ids_idx, ids in enumerate(batched_doc_id)
736
+ ]
737
+ )
738
+ )
739
+ # import ipdb; ipdb.set_trace()
740
+ batched_round_res = list(zip(*batched_round_res)) # [(r1_1, r1_2), (r2_1, r2_2), ...]
741
+ if batched_terminal_singal[0]: # terminal signal from doc_to_text function
742
+ break
743
+
744
+ for visual, context in zip(batched_visuals, batched_contexts):
745
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
746
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
747
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
748
+
749
+ if visual is None or visual == []: # for text-only tasks.
750
+ visual = None
751
+ task_type = "text"
752
+ placeholder_count = 0
753
+ image_tensor = None
754
+ else:
755
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
756
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
757
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
758
+
759
+ if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
760
+ assert type(visual) == list, "sample_frames must be specified for video task"
761
+ sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
762
+ visual = [visual[i] for i in sample_indices]
763
+ assert len(visual) == metadata["sample_frames"]
764
+
765
+ image_tensor = process_images(visual, self._image_processor, self._config)
766
+ if type(image_tensor) is list:
767
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
768
+ else:
769
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
770
+
771
+ task_type = "video"
772
+ placeholder_count = 1
773
+
774
+ elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
775
+ image_tensor = process_images(visual, self._image_processor, self._config)
776
+ if type(image_tensor) is list:
777
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
778
+ else:
779
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
780
+
781
+ task_type = "image"
782
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
783
+
784
+ elif type(visual[0]) == str: # For video task
785
+ image_tensor = []
786
+ try:
787
+ if self.video_decode_backend == "decord":
788
+ frames = self.load_video(visual, self.max_frames_num)
789
+ elif self.video_decode_backend == "pyav":
790
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
791
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
792
+ image_tensor.append(frames)
793
+ except Exception as e:
794
+ eval_logger.error(f"Error {e} in loading video")
795
+ image_tensor = None
796
+
797
+ task_type = "video"
798
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
799
+
800
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
801
+ """
802
+ Three senarios:
803
+ 1. No image, and there for, no image token should be added.
804
+ 2. image token is already specified in the context, so we don't need to add it.
805
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
806
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
807
+ """
808
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
809
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
810
+ # elif task_type == "video":
811
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
812
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
813
+ image_tokens = " ".join(image_tokens)
814
+ question = image_tokens + "\n" + context
815
+ else:
816
+ question = context
817
+
818
+ # This is much safer for llama3, as we now have some object type in it
819
+ if "llama_3" in self.conv_template:
820
+ conv = copy.deepcopy(conv_templates[self.conv_template])
821
+ else:
822
+ conv = conv_templates[self.conv_template].copy()
823
+
824
+ if utils.is_json(question): # conversational question input
825
+ question = json.loads(question)
826
+ for idx, item in enumerate(question):
827
+ role = conv.roles[idx % 2]
828
+ message = item["value"]
829
+ conv.append_message(role, message)
830
+
831
+ assert len(conv.messages) % 2 == 1
832
+ conv.append_message(conv.roles[1], None)
833
+ prompt_question = conv.get_prompt()
834
+ question_input.append(prompt_question)
835
+ else: # only simple string for question
836
+ conv.append_message(conv.roles[0], question)
837
+ conv.append_message(conv.roles[1], None)
838
+ prompt_question = conv.get_prompt()
839
+ question_input.append(prompt_question)
840
+
841
+ # preconfigure gen_kwargs with defaults
842
+ if "max_new_tokens" not in gen_kwargs:
843
+ gen_kwargs["max_new_tokens"] = 1024
844
+ if "temperature" not in gen_kwargs:
845
+ gen_kwargs["temperature"] = 0
846
+ if "do_sample" not in gen_kwargs:
847
+ gen_kwargs["do_sample"] = False
848
+ if "top_p" not in gen_kwargs:
849
+ gen_kwargs["top_p"] = None
850
+ if "num_beams" not in gen_kwargs:
851
+ gen_kwargs["num_beams"] = 1
852
+
853
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
854
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
855
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
856
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
857
+
858
+ if task_type == "image":
859
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
860
+ elif task_type == "video":
861
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
862
+ keywords = [stop_str]
863
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
864
+ gen_kwargs["modalities"] = ["video"]
865
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
866
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
867
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
868
+
869
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
870
+ # TODO: attention to this major generation step...
871
+ if "image_aspect_ratio" in gen_kwargs.keys():
872
+ gen_kwargs.pop("image_aspect_ratio")
873
+ try:
874
+ with torch.inference_mode():
875
+ cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
876
+ # cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
877
+
878
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
879
+ except Exception as e:
880
+ raise e
881
+
882
+ text_outputs = [response.strip() for response in text_outputs]
883
+ batched_round_res.append(text_outputs)
884
+
885
+ round_idx += 1
886
+
887
+ res.extend(list(zip(*batched_round_res)))
888
+ self.cache_hook.add_partial("generate_until_multi_round", (context, gen_kwargs), batched_round_res)
889
+ pbar.update(1)
890
+ # reorder this group of results back to original unsorted form
891
+ res = re_ords.get_original(res)
892
+
893
+ pbar.close()
894
+ return res
LaViDa/eval/lmms_eval/models/llava_onevision.py ADDED
@@ -0,0 +1,785 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import json
3
+ import logging
4
+ import math
5
+ import re
6
+ import warnings
7
+ from datetime import timedelta
8
+ from typing import List, Optional, Tuple, Union
9
+
10
+ import numpy as np
11
+ import PIL
12
+ import torch
13
+ import transformers
14
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
15
+ from accelerate.state import AcceleratorState
16
+ from decord import VideoReader, cpu
17
+ from packaging import version
18
+ from tqdm import tqdm
19
+ from transformers import AutoConfig
20
+
21
+ from lmms_eval import utils
22
+ from lmms_eval.api.instance import Instance
23
+ from lmms_eval.api.model import lmms
24
+ from lmms_eval.api.registry import register_model
25
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
26
+
27
+ # Suppress warnings
28
+ warnings.filterwarnings("ignore")
29
+
30
+ # Configure logging
31
+ eval_logger = logging.getLogger("lmms-eval")
32
+
33
+ # Enable TF32 for CUDA
34
+ torch.backends.cuda.matmul.allow_tf32 = True
35
+
36
+ # Import LLaVA modules
37
+ try:
38
+ from llava.constants import (
39
+ DEFAULT_IM_END_TOKEN,
40
+ DEFAULT_IM_START_TOKEN,
41
+ DEFAULT_IMAGE_TOKEN,
42
+ IGNORE_INDEX,
43
+ IMAGE_TOKEN_INDEX,
44
+ )
45
+ from llava.conversation import SeparatorStyle, conv_templates
46
+ from llava.mm_utils import (
47
+ KeywordsStoppingCriteria,
48
+ get_model_name_from_path,
49
+ process_images,
50
+ tokenizer_image_token,
51
+ )
52
+ from llava.model.builder import load_pretrained_model
53
+ except ImportError as e:
54
+ eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
55
+
56
+
57
+ # Determine best attention implementation
58
+ if version.parse(torch.__version__) >= version.parse("2.1.2"):
59
+ best_fit_attn_implementation = "sdpa"
60
+ else:
61
+ best_fit_attn_implementation = "eager"
62
+
63
+
64
+ @register_model("llava_onevision")
65
+ class Llava_OneVision(lmms):
66
+ """
67
+ Llava Model
68
+ """
69
+
70
+ def __init__(
71
+ self,
72
+ pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
73
+ truncation: Optional[bool] = True,
74
+ device: Optional[str] = "cuda:0",
75
+ batch_size: Optional[Union[int, str]] = 1,
76
+ model_name: Optional[str] = None,
77
+ attn_implementation: Optional[str] = best_fit_attn_implementation,
78
+ device_map: Optional[str] = "cuda:0",
79
+ conv_template: Optional[str] = "qwen_1_5",
80
+ use_cache: Optional[bool] = True,
81
+ truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
82
+ customized_config: Optional[str] = None, # ends in json
83
+ max_frames_num: Optional[int] = 32,
84
+ mm_spatial_pool_stride: Optional[int] = 2,
85
+ mm_spatial_pool_mode: Optional[str] = "bilinear",
86
+ token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
87
+ video_decode_backend: str = "decord",
88
+ **kwargs,
89
+ ) -> None:
90
+ super().__init__()
91
+ # Do not use kwargs for now
92
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
93
+
94
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
95
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
96
+ if accelerator.num_processes > 1:
97
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
98
+ self.device_map = f"cuda:{accelerator.local_process_index}"
99
+ elif accelerator.num_processes == 1 and device_map == "auto":
100
+ self._device = torch.device(device)
101
+ self.device_map = device_map
102
+ else:
103
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
104
+ self.device_map = f"cuda:{accelerator.local_process_index}"
105
+
106
+ llava_model_args = {
107
+ "multimodal": True,
108
+ }
109
+ if customized_config is not None:
110
+ llava_model_args["customized_config"] = customized_config
111
+ if attn_implementation is not None:
112
+ llava_model_args["attn_implementation"] = attn_implementation
113
+ if "use_flash_attention_2" in kwargs:
114
+ llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
115
+ model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
116
+
117
+ self.pretrained = pretrained
118
+ self.token_strategy = token_strategy
119
+ self.max_frames_num = max_frames_num
120
+ self.mm_spatial_pool_stride = mm_spatial_pool_stride
121
+ self.mm_spatial_pool_mode = mm_spatial_pool_mode
122
+ self.video_decode_backend = video_decode_backend
123
+
124
+ overwrite_config = {}
125
+ overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
126
+ overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
127
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
128
+
129
+ llava_model_args["overwrite_config"] = overwrite_config
130
+ try:
131
+ # Try to load the model with the multimodal argument
132
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
133
+ except TypeError:
134
+ # for older versions of LLaVA that don't have multimodal argument
135
+ llava_model_args.pop("multimodal", None)
136
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
137
+
138
+ self._config = self._model.config
139
+ self.model.eval()
140
+ self.truncation = truncation
141
+ self.batch_size_per_gpu = int(batch_size)
142
+ self.conv_template = conv_template
143
+ self.use_cache = use_cache
144
+ self.truncate_context = truncate_context
145
+ assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
146
+
147
+ if accelerator.num_processes > 1:
148
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
149
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
150
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
151
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
152
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
153
+ kwargs = {
154
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
155
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
156
+ }
157
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
158
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
159
+
160
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
161
+ self._model = accelerator.prepare(self.model)
162
+ else:
163
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
164
+ self.accelerator = accelerator
165
+ if self.accelerator.is_local_main_process:
166
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
167
+ self._rank = self.accelerator.local_process_index
168
+ self._world_size = self.accelerator.num_processes
169
+
170
+ elif accelerator.num_processes == 1 and device_map == "auto":
171
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
172
+ self._rank = 0
173
+ self._world_size = 1
174
+
175
+ else:
176
+ eval_logger.info(f"Using single device: {self._device}")
177
+ self.model.to(self._device)
178
+ self._rank = 0
179
+ self._world_size = 1
180
+
181
+ @property
182
+ def config(self):
183
+ # return the associated transformers.AutoConfig for the given pretrained model.
184
+ return self._config
185
+
186
+ @property
187
+ def tokenizer(self):
188
+ return self._tokenizer
189
+
190
+ @property
191
+ def model(self):
192
+ # returns the model, unwrapping it if using Accelerate
193
+ if hasattr(self, "accelerator"):
194
+ return self.accelerator.unwrap_model(self._model)
195
+ else:
196
+ return self._model
197
+
198
+ @property
199
+ def eot_token_id(self):
200
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
201
+ return self.tokenizer.eos_token_id
202
+
203
+ @property
204
+ def max_length(self):
205
+ return self._max_length
206
+
207
+ def pad_sequence(self, input_ids, batch_first, padding_value):
208
+ if self.tokenizer.padding_side == "left":
209
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
210
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
211
+ if self.tokenizer.padding_side == "left":
212
+ input_ids = torch.flip(input_ids, [1])
213
+ return input_ids
214
+
215
+ @property
216
+ def batch_size(self):
217
+ return self.batch_size_per_gpu
218
+
219
+ @property
220
+ def device(self):
221
+ return self._device
222
+
223
+ @property
224
+ def rank(self):
225
+ return self._rank
226
+
227
+ @property
228
+ def world_size(self):
229
+ return self._world_size
230
+
231
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
232
+ """ """
233
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
234
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
235
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
236
+ if left_truncate_len:
237
+ encoding = encoding[-left_truncate_len:]
238
+ return encoding
239
+
240
+ def tok_decode(self, tokens):
241
+ try:
242
+ return self.tokenizer.decode(tokens)
243
+ except:
244
+ return self.tokenizer.decode([tokens])
245
+
246
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
247
+ res = []
248
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
249
+
250
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
251
+
252
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
253
+ visual = doc_to_visual(self.task_dict[task][split][doc_id])
254
+
255
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
256
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
257
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
258
+
259
+ if visual is None or visual == []:
260
+ visual = None
261
+ task_type = "text"
262
+ image_tensor = None
263
+ else:
264
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__:
265
+ self._config.image_aspect_ratio = "pad"
266
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
267
+
268
+ if "task_type" in self.metadata and self.metadata["task_type"] == "video" and "sample_frames" in self.metadata:
269
+ assert type(visual) == list, "sample_frames must be specified for video task"
270
+ sample_indices = np.linspace(0, len(visual) - 1, self.metadata["sample_frames"], dtype=int)
271
+ visual = [visual[i] for i in sample_indices]
272
+ assert len(visual) == self.metadata["sample_frames"]
273
+
274
+ image_tensor = process_images(visual, self._image_processor, self._config)
275
+ if type(image_tensor) is list:
276
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
277
+ else:
278
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
279
+
280
+ task_type = "video"
281
+
282
+ # elif type(visual[0]) == PIL.Image.Image:
283
+ elif isinstance(visual[0], PIL.Image.Image):
284
+ image_tensor = process_images(visual, self._image_processor, self._config)
285
+ if type(image_tensor) is list:
286
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
287
+ else:
288
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
289
+
290
+ task_type = "image"
291
+
292
+ elif type(visual[0]) == str:
293
+ image_tensor = []
294
+ try:
295
+ if self.video_decode_backend == "decord":
296
+ frames = self.load_video(visual, self.max_frames_num)
297
+ elif self.video_decode_backend == "pyav":
298
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
299
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
300
+ image_tensor.append(frames)
301
+ except Exception as e:
302
+ eval_logger.error(f"Error {e} in loading video")
303
+ image_tensor = None
304
+
305
+ task_type = "video"
306
+
307
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in contexts:
308
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
309
+ if task_type == "video":
310
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
311
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
312
+ image_tokens = " ".join(image_tokens)
313
+ prompts_input = image_tokens + "\n" + contexts
314
+ else:
315
+ prompts_input = contexts
316
+
317
+ if "llama_3" in self.conv_template:
318
+ conv = copy.deepcopy(conv_templates[self.conv_template])
319
+ else:
320
+ conv = conv_templates[self.conv_template].copy()
321
+
322
+ conv.append_message(conv.roles[0], prompts_input)
323
+ conv.append_message(conv.roles[1], None)
324
+ prompt = conv.get_prompt()
325
+
326
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
327
+
328
+ if type(doc_to_target) == str:
329
+ continuation = doc_to_target
330
+ else:
331
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
332
+
333
+ conv.messages[-1][1] = continuation
334
+ full_prompt = conv.get_prompt()
335
+ full_input_ids = tokenizer_image_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
336
+
337
+ labels = full_input_ids.clone()
338
+ labels[0, : input_ids.shape[1]] = -100
339
+
340
+ kwargs = {}
341
+ if task_type == "image":
342
+ kwargs["image_sizes"] = [[v.size[0], v.size[1]] for v in visual] if isinstance(visual, list) else [[visual.size[0], visual.size[1]]]
343
+ elif task_type == "video":
344
+ kwargs["modalities"] = ["video"]
345
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
346
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
347
+
348
+ with torch.inference_mode():
349
+ outputs = self.model(input_ids=full_input_ids, labels=labels, images=image_tensor, use_cache=True, **kwargs)
350
+
351
+ loss = outputs["loss"]
352
+ logits = outputs["logits"]
353
+ greedy_tokens = logits.argmax(dim=-1)
354
+ cont_toks = full_input_ids[:, input_ids.shape[1] :]
355
+ greedy_tokens = greedy_tokens[:, input_ids.shape[1] : full_input_ids.shape[1]]
356
+ max_equal = (greedy_tokens == cont_toks).all()
357
+
358
+ res.append((float(loss.item()), bool(max_equal)))
359
+ pbar.update(1)
360
+
361
+ pbar.close()
362
+ return res
363
+
364
+ def flatten(self, input):
365
+ if not input or any(i is None for i in input):
366
+ return []
367
+ new_list = []
368
+ for i in input:
369
+ if i:
370
+ for j in i:
371
+ new_list.append(j)
372
+ return new_list
373
+
374
+ def load_video(self, video_path, max_frames_num):
375
+ if type(video_path) == str:
376
+ vr = VideoReader(video_path, ctx=cpu(0))
377
+ else:
378
+ vr = VideoReader(video_path[0], ctx=cpu(0))
379
+ total_frame_num = len(vr)
380
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
381
+ frame_idx = uniform_sampled_frames.tolist()
382
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
383
+ return spare_frames # (frames, height, width, channels)
384
+
385
+ def generate_until(self, requests: List[Instance]) -> List[str]:
386
+ res = []
387
+
388
+ def _collate(x):
389
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
390
+ # - time estimates will always be over not underestimates, which is more useful for planning
391
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
392
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
393
+ # automatic adaptive batches much much easier to implement
394
+ # - any OOMs will happen right away rather than near the end
395
+ toks = self.tok_encode(x[0])
396
+ return -len(toks), x[0]
397
+
398
+ # we group requests by their generation_kwargs,
399
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
400
+ # in the same batch.
401
+ metadata = requests[0].metadata
402
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
403
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
404
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
405
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
406
+
407
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
408
+
409
+ for chunk in chunks:
410
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
411
+ task = batched_task[0]
412
+ split = batched_split[0]
413
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
414
+ assert len(batched_visuals) == 1
415
+
416
+ # we assume all gen kwargs in the batch are the same
417
+ # this is safe to assume because the `grouper` object ensures it.
418
+ gen_kwargs = all_gen_kwargs[0]
419
+ if "until" in gen_kwargs:
420
+ gen_kwargs.pop("until")
421
+
422
+ question_input = []
423
+ # import ipdb; ipdb.set_trace()
424
+ for visual, context in zip(batched_visuals, batched_contexts):
425
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
426
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
427
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
428
+
429
+ if visual is None or visual == []: # for text-only tasks.
430
+ visual = None
431
+ task_type = "text"
432
+ placeholder_count = 0
433
+ image_tensor = None
434
+ else:
435
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
436
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
437
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
438
+
439
+ if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
440
+ assert type(visual) == list, "sample_frames must be specified for video task"
441
+ sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
442
+ visual = [visual[i] for i in sample_indices]
443
+ assert len(visual) == metadata["sample_frames"]
444
+
445
+ image_tensor = process_images(visual, self._image_processor, self._config)
446
+ if type(image_tensor) is list:
447
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
448
+ else:
449
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
450
+
451
+ task_type = "video"
452
+ placeholder_count = 1
453
+
454
+ elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
455
+ image_tensor = process_images(visual, self._image_processor, self._config)
456
+ if type(image_tensor) is list:
457
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
458
+ else:
459
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
460
+
461
+ task_type = "image"
462
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
463
+
464
+ elif type(visual[0]) == str: # For video task
465
+ image_tensor = []
466
+ try:
467
+ if self.video_decode_backend == "decord":
468
+ frames = self.load_video(visual, self.max_frames_num)
469
+ elif self.video_decode_backend == "pyav":
470
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
471
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
472
+ image_tensor.append(frames)
473
+ except Exception as e:
474
+ eval_logger.error(f"Error {e} in loading video")
475
+ image_tensor = None
476
+
477
+ task_type = "video"
478
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
479
+
480
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
481
+ """
482
+ Three senarios:
483
+ 1. No image, and there for, no image token should be added.
484
+ 2. image token is already specified in the context, so we don't need to add it.
485
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
486
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
487
+ """
488
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
489
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
490
+ # elif task_type == "video":
491
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
492
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
493
+ image_tokens = " ".join(image_tokens)
494
+ question = image_tokens + "\n" + context
495
+ else:
496
+ question = context
497
+
498
+ # This is much safer for llama3, as we now have some object type in it
499
+ if "llama_3" in self.conv_template:
500
+ conv = copy.deepcopy(conv_templates[self.conv_template])
501
+ else:
502
+ conv = conv_templates[self.conv_template].copy()
503
+
504
+ if utils.is_json(question): # conversational question input
505
+ question = json.loads(question)
506
+ for idx, item in enumerate(question):
507
+ role = conv.roles[idx % 2]
508
+ message = item["value"]
509
+ conv.append_message(role, message)
510
+
511
+ assert len(conv.messages) % 2 == 1
512
+ conv.append_message(conv.roles[1], None)
513
+ prompt_question = conv.get_prompt()
514
+ question_input.append(prompt_question)
515
+ else: # only simple string for question
516
+ conv.append_message(conv.roles[0], question)
517
+ conv.append_message(conv.roles[1], None)
518
+ prompt_question = conv.get_prompt()
519
+ question_input.append(prompt_question)
520
+
521
+ # preconfigure gen_kwargs with defaults
522
+ if "max_new_tokens" not in gen_kwargs:
523
+ gen_kwargs["max_new_tokens"] = 1024
524
+ if "temperature" not in gen_kwargs:
525
+ gen_kwargs["temperature"] = 0
526
+ if "do_sample" not in gen_kwargs:
527
+ gen_kwargs["do_sample"] = False
528
+ if "top_p" not in gen_kwargs:
529
+ gen_kwargs["top_p"] = None
530
+ if "num_beams" not in gen_kwargs:
531
+ gen_kwargs["num_beams"] = 1
532
+
533
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
534
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
535
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
536
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
537
+
538
+ if task_type == "image":
539
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
540
+ elif task_type == "video":
541
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
542
+ keywords = [stop_str]
543
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
544
+ gen_kwargs["modalities"] = ["video"]
545
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
546
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
547
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
548
+
549
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
550
+ # TODO: attention to this major generation step...
551
+ if "image_aspect_ratio" in gen_kwargs.keys():
552
+ gen_kwargs.pop("image_aspect_ratio")
553
+ try:
554
+ with torch.inference_mode():
555
+ cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
556
+ # cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
557
+
558
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
559
+ except Exception as e:
560
+ raise e
561
+
562
+ text_outputs = [response.strip() for response in text_outputs]
563
+ res.extend(text_outputs)
564
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
565
+ pbar.update(1)
566
+ # reorder this group of results back to original unsorted form
567
+ res = re_ords.get_original(res)
568
+
569
+ pbar.close()
570
+ return res
571
+
572
+ def generate_until_multi_round(self, requests: List[Instance]) -> List[str]:
573
+ res = []
574
+
575
+ def _collate(x):
576
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
577
+ # - time estimates will always be over not underestimates, which is more useful for planning
578
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
579
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
580
+ # automatic adaptive batches much much easier to implement
581
+ # - any OOMs will happen right away rather than near the end
582
+ toks = self.tok_encode(x[0])
583
+ return -len(toks), x[0]
584
+
585
+ # we group requests by their generation_kwargs,
586
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
587
+ # in the same batch.
588
+ metadata = requests[0].metadata
589
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
590
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
591
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
592
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
593
+
594
+ origin_image_aspect_ratio = getattr(self._config, "image_aspect_ratio", None)
595
+
596
+ for chunk in chunks:
597
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_to_text, batched_doc_id, batched_task, batched_split = zip(*chunk)
598
+ task = batched_task[0]
599
+ split = batched_split[0]
600
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
601
+ assert len(batched_visuals) == 1
602
+
603
+ # we assume all gen kwargs in the batch are the same
604
+ # this is safe to assume because the `grouper` object ensures it.
605
+ gen_kwargs = all_gen_kwargs[0]
606
+ if "until" in gen_kwargs:
607
+ gen_kwargs.pop("until")
608
+
609
+ # multi round inference: terminate when receiving signal from the doc_to_text
610
+ round_idx = 0
611
+ batched_round_res = []
612
+ batched_previous_round_info = None
613
+ while True:
614
+ question_input = []
615
+
616
+ if round_idx != 0: # get current round visual and context from doc_to_text function
617
+ batched_visuals, batched_contexts, batched_terminal_singal, batched_round_res, batched_previous_round_info = list(
618
+ zip(
619
+ *[
620
+ batched_doc_to_text[0](
621
+ self.task_dict[task][split][ids],
622
+ previous_output=[round_res[ids_idx] for round_res in batched_round_res],
623
+ round_idx=round_idx,
624
+ previous_round_info=batched_previous_round_info[ids_idx] if batched_previous_round_info is not None else None,
625
+ )
626
+ for ids_idx, ids in enumerate(batched_doc_id)
627
+ ]
628
+ )
629
+ )
630
+ # import ipdb; ipdb.set_trace()
631
+ batched_round_res = list(zip(*batched_round_res)) # [(r1_1, r1_2), (r2_1, r2_2), ...]
632
+ if batched_terminal_singal[0]: # terminal signal from doc_to_text function
633
+ break
634
+
635
+ for visual, context in zip(batched_visuals, batched_contexts):
636
+ if origin_image_aspect_ratio is not None and self._config.image_aspect_ratio != origin_image_aspect_ratio:
637
+ self._config.image_aspect_ratio = origin_image_aspect_ratio
638
+ eval_logger.info(f"Resetting image aspect ratio to {origin_image_aspect_ratio}")
639
+
640
+ if visual is None or visual == []: # for text-only tasks.
641
+ visual = None
642
+ task_type = "text"
643
+ placeholder_count = 0
644
+ image_tensor = None
645
+ else:
646
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
647
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
648
+ eval_logger.info(f"In Multi-Image setting, image aspect ratio: {self._config.image_aspect_ratio}")
649
+
650
+ if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task with multiple static image frames
651
+ assert type(visual) == list, "sample_frames must be specified for video task"
652
+ sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int)
653
+ visual = [visual[i] for i in sample_indices]
654
+ assert len(visual) == metadata["sample_frames"]
655
+
656
+ image_tensor = process_images(visual, self._image_processor, self._config)
657
+ if type(image_tensor) is list:
658
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
659
+ else:
660
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
661
+
662
+ task_type = "video"
663
+ placeholder_count = 1
664
+
665
+ elif type(visual[0]) == PIL.Image.Image: # For image, multi-image tasks
666
+ image_tensor = process_images(visual, self._image_processor, self._config)
667
+ if type(image_tensor) is list:
668
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
669
+ else:
670
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
671
+
672
+ task_type = "image"
673
+ placeholder_count = len(visual) if isinstance(visual, list) else 1
674
+
675
+ elif type(visual[0]) == str: # For video task
676
+ image_tensor = []
677
+ try:
678
+ if self.video_decode_backend == "decord":
679
+ frames = self.load_video(visual, self.max_frames_num)
680
+ elif self.video_decode_backend == "pyav":
681
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
682
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
683
+ image_tensor.append(frames)
684
+ except Exception as e:
685
+ eval_logger.error(f"Error {e} in loading video")
686
+ image_tensor = None
687
+
688
+ task_type = "video"
689
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
690
+
691
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
692
+ """
693
+ Three senarios:
694
+ 1. No image, and there for, no image token should be added.
695
+ 2. image token is already specified in the context, so we don't need to add it.
696
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
697
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
698
+ """
699
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
700
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
701
+ # elif task_type == "video":
702
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
703
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
704
+ image_tokens = " ".join(image_tokens)
705
+ question = image_tokens + "\n" + context
706
+ else:
707
+ question = context
708
+
709
+ # This is much safer for llama3, as we now have some object type in it
710
+ if "llama_3" in self.conv_template:
711
+ conv = copy.deepcopy(conv_templates[self.conv_template])
712
+ else:
713
+ conv = conv_templates[self.conv_template].copy()
714
+
715
+ if utils.is_json(question): # conversational question input
716
+ question = json.loads(question)
717
+ for idx, item in enumerate(question):
718
+ role = conv.roles[idx % 2]
719
+ message = item["value"]
720
+ conv.append_message(role, message)
721
+
722
+ assert len(conv.messages) % 2 == 1
723
+ conv.append_message(conv.roles[1], None)
724
+ prompt_question = conv.get_prompt()
725
+ question_input.append(prompt_question)
726
+ else: # only simple string for question
727
+ conv.append_message(conv.roles[0], question)
728
+ conv.append_message(conv.roles[1], None)
729
+ prompt_question = conv.get_prompt()
730
+ question_input.append(prompt_question)
731
+
732
+ # preconfigure gen_kwargs with defaults
733
+ if "max_new_tokens" not in gen_kwargs:
734
+ gen_kwargs["max_new_tokens"] = 1024
735
+ if "temperature" not in gen_kwargs:
736
+ gen_kwargs["temperature"] = 0
737
+ if "do_sample" not in gen_kwargs:
738
+ gen_kwargs["do_sample"] = False
739
+ if "top_p" not in gen_kwargs:
740
+ gen_kwargs["top_p"] = None
741
+ if "num_beams" not in gen_kwargs:
742
+ gen_kwargs["num_beams"] = 1
743
+
744
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
745
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
746
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
747
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
748
+
749
+ if task_type == "image":
750
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
751
+ elif task_type == "video":
752
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
753
+ keywords = [stop_str]
754
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
755
+ gen_kwargs["modalities"] = ["video"]
756
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
757
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
758
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
759
+
760
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
761
+ # TODO: attention to this major generation step...
762
+ if "image_aspect_ratio" in gen_kwargs.keys():
763
+ gen_kwargs.pop("image_aspect_ratio")
764
+ try:
765
+ with torch.inference_mode():
766
+ cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
767
+ # cont = self.model.generate(qwen_input_ids, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
768
+
769
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
770
+ except Exception as e:
771
+ raise e
772
+
773
+ text_outputs = [response.strip() for response in text_outputs]
774
+ batched_round_res.append(text_outputs)
775
+
776
+ round_idx += 1
777
+
778
+ res.extend(list(zip(*batched_round_res)))
779
+ self.cache_hook.add_partial("generate_until_multi_round", (context, gen_kwargs), batched_round_res)
780
+ pbar.update(1)
781
+ # reorder this group of results back to original unsorted form
782
+ res = re_ords.get_original(res)
783
+
784
+ pbar.close()
785
+ return res
LaViDa/eval/lmms_eval/models/llava_onevision_moviechat.py ADDED
@@ -0,0 +1,526 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import json
3
+ import logging
4
+ import math
5
+ import os
6
+ import re
7
+ import warnings
8
+ from datetime import timedelta
9
+ from typing import List, Optional, Tuple, Union
10
+
11
+ import av
12
+ import numpy as np
13
+ import PIL
14
+ import torch
15
+ import transformers
16
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
17
+ from accelerate.state import AcceleratorState
18
+ from decord import VideoReader, cpu
19
+ from moviepy.video.io.VideoFileClip import VideoFileClip
20
+ from packaging import version
21
+ from PIL import Image
22
+ from tqdm import tqdm
23
+ from transformers import AutoConfig
24
+
25
+ from lmms_eval import utils
26
+ from lmms_eval.api.instance import Instance
27
+ from lmms_eval.api.model import lmms
28
+ from lmms_eval.api.registry import register_model
29
+
30
+ # Suppress warnings
31
+ warnings.filterwarnings("ignore")
32
+
33
+ # Configure logging
34
+ eval_logger = logging.getLogger("lmms-eval")
35
+
36
+ # Enable TF32 for CUDA
37
+ torch.backends.cuda.matmul.allow_tf32 = True
38
+
39
+ # Import LLaVA modules
40
+ try:
41
+ from llava.constants import (
42
+ DEFAULT_IM_END_TOKEN,
43
+ DEFAULT_IM_START_TOKEN,
44
+ DEFAULT_IMAGE_TOKEN,
45
+ IGNORE_INDEX,
46
+ IMAGE_TOKEN_INDEX,
47
+ )
48
+ from llava.conversation import SeparatorStyle, conv_templates
49
+ from llava.mm_utils import (
50
+ KeywordsStoppingCriteria,
51
+ get_model_name_from_path,
52
+ process_images,
53
+ tokenizer_image_token,
54
+ )
55
+ from llava.model.builder import load_pretrained_model
56
+ except ImportError as e:
57
+ eval_logger.debug(f"LLaVA_NeXT is not installed. Please install llava from `https://github.com/rese1f/MovieChat.git` to use this model.\nError: {e}")
58
+
59
+
60
+ # Determine best attention implementation
61
+ if version.parse(torch.__version__) >= version.parse("2.1.2"):
62
+ best_fit_attn_implementation = "sdpa"
63
+ else:
64
+ best_fit_attn_implementation = "eager"
65
+
66
+
67
+ # llava_onevision_moviechat uses the same memory consolidation mechanism with the original MovieChat, but changes the base model from VideoLLamA to LLava-OneVision
68
+ @register_model("llava_onevision_moviechat")
69
+ class Llava_OneVision_MovieChat(lmms):
70
+ """
71
+ Llava Model
72
+ """
73
+
74
+ def __init__(
75
+ self,
76
+ pretrained: str = "lmms-lab/llava-onevision-qwen2-7b-ov",
77
+ truncation: Optional[bool] = True,
78
+ device: Optional[str] = "cuda:0",
79
+ batch_size: Optional[Union[int, str]] = 1,
80
+ model_name: str = "llava_qwen",
81
+ attn_implementation: Optional[str] = best_fit_attn_implementation,
82
+ device_map: Optional[str] = "cuda:0",
83
+ conv_template: Optional[str] = "qwen_1_5",
84
+ use_cache: Optional[bool] = True,
85
+ truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
86
+ customized_config: Optional[str] = None, # ends in json
87
+ short_memory_length: Optional[int] = 18,
88
+ long_memory_length: Optional[int] = 64,
89
+ sliding_window_length: Optional[int] = 8,
90
+ merge_frame_length: Optional[int] = 2,
91
+ tmp_folder: Optional[str] = "tmp/",
92
+ mm_spatial_pool_stride: Optional[int] = 2,
93
+ mm_spatial_pool_mode: Optional[str] = "bilinear",
94
+ token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
95
+ video_decode_backend: str = "decord",
96
+ **kwargs,
97
+ ) -> None:
98
+ super().__init__()
99
+ # Do not use kwargs for now
100
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
101
+
102
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
103
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
104
+ if accelerator.num_processes > 1:
105
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
106
+ self.device_map = f"cuda:{accelerator.local_process_index}"
107
+ elif accelerator.num_processes == 1 and device_map == "auto":
108
+ self._device = torch.device(device)
109
+ self.device_map = device_map
110
+ else:
111
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
112
+ self.device_map = f"cuda:{accelerator.local_process_index}"
113
+
114
+ llava_model_args = {
115
+ "multimodal": True,
116
+ }
117
+ if customized_config is not None:
118
+ llava_model_args["customized_config"] = customized_config
119
+ if attn_implementation is not None:
120
+ llava_model_args["attn_implementation"] = attn_implementation
121
+ if "use_flash_attention_2" in kwargs:
122
+ llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
123
+ model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
124
+
125
+ self.pretrained = pretrained
126
+ self.token_strategy = token_strategy
127
+ self.mm_spatial_pool_stride = mm_spatial_pool_stride
128
+ self.mm_spatial_pool_mode = mm_spatial_pool_mode
129
+ self.video_decode_backend = video_decode_backend
130
+
131
+ self.short_memory_length = short_memory_length
132
+ self.long_memory_length = long_memory_length
133
+ self.merge_frame_length = merge_frame_length
134
+ self.sliding_window_length = sliding_window_length
135
+ self.num_clips = (self.long_memory_length // self.merge_frame_length) * ((self.short_memory_length - self.merge_frame_length) // self.sliding_window_length)
136
+ self.tmp_folder = tmp_folder
137
+
138
+ overwrite_config = {}
139
+ overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
140
+ overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
141
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
142
+
143
+ if cfg_pretrained.architectures[0] == "LlavaLlamaForCausalLM": # Ugly code, only used in vicuna that needs ROPE
144
+ if "224" in cfg_pretrained.mm_vision_tower:
145
+ least_token_number = self.max_frames_num * (16 // self.mm_spatial_pool_stride) ** 2 + 1000
146
+ else:
147
+ least_token_number = self.max_frames_num * (24 // self.mm_spatial_pool_stride) ** 2 + 1000
148
+
149
+ scaling_factor = math.ceil(least_token_number / 4096)
150
+ if scaling_factor >= 2:
151
+ overwrite_config["rope_scaling"] = {"factor": float(scaling_factor), "type": "linear"}
152
+ overwrite_config["max_sequence_length"] = 4096 * scaling_factor
153
+ overwrite_config["tokenizer_model_max_length"] = 4096 * scaling_factor
154
+
155
+ llava_model_args["overwrite_config"] = overwrite_config
156
+ from LLaVA_NeXT.llava.model.builder import load_pretrained_model
157
+
158
+ try:
159
+ # Try to load the model with the multimodal argument
160
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
161
+ except TypeError:
162
+ # for older versions of LLaVA that don't have multimodal argument
163
+ llava_model_args.pop("multimodal", None)
164
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
165
+
166
+ self._config = self._model.config
167
+ self.model.eval()
168
+ self.truncation = truncation
169
+ self.batch_size_per_gpu = int(batch_size)
170
+ self.conv_template = conv_template
171
+ self.use_cache = use_cache
172
+ self.truncate_context = truncate_context
173
+ assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
174
+
175
+ if accelerator.num_processes > 1:
176
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
177
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
178
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
179
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
180
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
181
+ kwargs = {
182
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
183
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
184
+ }
185
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
186
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
187
+
188
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
189
+ self._model = accelerator.prepare(self.model)
190
+ else:
191
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
192
+ self.accelerator = accelerator
193
+ if self.accelerator.is_local_main_process:
194
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
195
+ self._rank = self.accelerator.local_process_index
196
+ self._world_size = self.accelerator.num_processes
197
+
198
+ elif accelerator.num_processes == 1 and device_map == "auto":
199
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
200
+ self._rank = 0
201
+ self._world_size = 1
202
+
203
+ else:
204
+ eval_logger.info(f"Using single device: {self._device}")
205
+ self.model.to(self._device)
206
+ self._rank = 0
207
+ self._world_size = 1
208
+
209
+ @property
210
+ def config(self):
211
+ # return the associated transformers.AutoConfig for the given pretrained model.
212
+ return self._config
213
+
214
+ @property
215
+ def tokenizer(self):
216
+ return self._tokenizer
217
+
218
+ @property
219
+ def model(self):
220
+ # returns the model, unwrapping it if using Accelerate
221
+ if hasattr(self, "accelerator"):
222
+ return self.accelerator.unwrap_model(self._model)
223
+ else:
224
+ return self._model
225
+
226
+ @property
227
+ def eot_token_id(self):
228
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
229
+ return self.tokenizer.eos_token_id
230
+
231
+ @property
232
+ def max_length(self):
233
+ return self._max_length
234
+
235
+ def pad_sequence(self, input_ids, batch_first, padding_value):
236
+ if self.tokenizer.padding_side == "left":
237
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
238
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
239
+ if self.tokenizer.padding_side == "left":
240
+ input_ids = torch.flip(input_ids, [1])
241
+ return input_ids
242
+
243
+ @property
244
+ def batch_size(self):
245
+ return self.batch_size_per_gpu
246
+
247
+ @property
248
+ def device(self):
249
+ return self._device
250
+
251
+ @property
252
+ def rank(self):
253
+ return self._rank
254
+
255
+ @property
256
+ def world_size(self):
257
+ return self._world_size
258
+
259
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
260
+ """ """
261
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
262
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
263
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
264
+ if left_truncate_len:
265
+ encoding = encoding[-left_truncate_len:]
266
+ return encoding
267
+
268
+ def tok_decode(self, tokens):
269
+ try:
270
+ return self.tokenizer.decode(tokens)
271
+ except:
272
+ return self.tokenizer.decode([tokens])
273
+
274
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
275
+ # TODO
276
+ raise NotImplementedError("MovieChat only supports generation.")
277
+
278
+ def flatten(self, input):
279
+ new_list = []
280
+ for i in input:
281
+ for j in i:
282
+ new_list.append(j)
283
+ return new_list
284
+
285
+ def load_video(self, video_path, max_frames_num):
286
+ if type(video_path) == str:
287
+ vr = VideoReader(video_path, ctx=cpu(0))
288
+ else:
289
+ vr = VideoReader(video_path[0], ctx=cpu(0))
290
+ total_frame_num = len(vr)
291
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
292
+ frame_idx = uniform_sampled_frames.tolist()
293
+ spare_frames = vr.get_batch(frame_idx).numpy()
294
+ return spare_frames # (frames, height, width, channels)
295
+
296
+ def extract_keyframes(self, video_path, keyframes):
297
+ container = av.open(video_path)
298
+ video_stream = container.streams.video[0]
299
+ fps = video_stream.average_rate
300
+ time_base = video_stream.time_base
301
+ frames = []
302
+
303
+ for keyframe in keyframes:
304
+ keyframe_time = float(keyframe)
305
+ frame_number = int(keyframe_time * fps)
306
+ container.seek(int(keyframe_time / time_base))
307
+ found = False
308
+ for packet in container.demux(video=0):
309
+ for frame in packet.decode():
310
+ if frame.index >= frame_number:
311
+ frames.append(frame)
312
+ found = True
313
+ break
314
+ if found:
315
+ break
316
+
317
+ if not found:
318
+ container.seek(-1, any_frame=False)
319
+ for packet in container.demux(video=0):
320
+ for frame in packet.decode():
321
+ pass
322
+ frames.append(frame)
323
+
324
+ video = [x.to_ndarray(format="rgb24") for x in frames]
325
+ video_frames = [Image.fromarray(x) for x in video]
326
+ return video_frames
327
+
328
+ def generate_until(self, requests: List[Instance]) -> List[str]:
329
+ res = []
330
+
331
+ def _collate(x):
332
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
333
+ # - time estimates will always be over not underestimates, which is more useful for planning
334
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
335
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
336
+ # automatic adaptive batches much much easier to implement
337
+ # - any OOMs will happen right away rather than near the end
338
+ toks = self.tok_encode(x[0])
339
+ return -len(toks), x[0]
340
+
341
+ # we group requests by their generation_kwargs,
342
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
343
+ # in the same batch.
344
+ metadata = requests[0].metadata
345
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
346
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
347
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
348
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
349
+ for chunk in chunks:
350
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
351
+ task = batched_task[0]
352
+ split = batched_split[0]
353
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
354
+ assert len(batched_visuals) == 1
355
+
356
+ # we assume all gen kwargs in the batch are the same
357
+ # this is safe to assume because the `grouper` object ensures it.
358
+ gen_kwargs = all_gen_kwargs[0]
359
+ if "until" in gen_kwargs:
360
+ gen_kwargs.pop("until")
361
+
362
+ question_input = []
363
+
364
+ for visual, context in zip(batched_visuals, batched_contexts):
365
+ if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default.
366
+ self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad")
367
+ eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
368
+ if type(visual[0]) == PIL.Image.Image and "task_type" not in metadata and "sample_frames" not in metadata: # For image task
369
+ raise NotImplementedError("MovieChat only supports video inputs.")
370
+
371
+ elif "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata:
372
+ raise NotImplementedError("MovieChat only supports video inputs.")
373
+
374
+ elif type(visual[0]) == str: # For video task
375
+ try:
376
+ self.short_memory_buffer = []
377
+ self.long_memory_buffer = []
378
+ # try:
379
+ os.makedirs(self.tmp_folder, exist_ok=True)
380
+
381
+ video = VideoFileClip(visual[0])
382
+ clip_duration = video.duration / self.num_clips
383
+
384
+ cur_frame = 0
385
+ for i in range(self.num_clips):
386
+ start_time = i * clip_duration
387
+ end_time = start_time + clip_duration
388
+ # uniformly sample self.sliding_window_length frames from the video from start_time to end_time
389
+ frames = list(video.subclip(start_time, end_time).iter_frames(fps=self.sliding_window_length / clip_duration))[: self.sliding_window_length]
390
+ frames = [Image.fromarray(frame).convert("RGB") for frame in frames]
391
+ preprocess_frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
392
+ encoded_window = self.model.encode_images(preprocess_frames) # [frames, 729,3584]
393
+
394
+ for frame in encoded_window:
395
+ if cur_frame < (self.short_memory_length - self.merge_frame_length):
396
+ if len(self.short_memory_buffer) == self.short_memory_length:
397
+ self.short_memory_buffer.pop(0)
398
+ self.short_memory_buffer.append(frame)
399
+ cur_frame += 1
400
+
401
+ if cur_frame == (self.short_memory_length - self.merge_frame_length):
402
+ cur_frame = 0
403
+
404
+ # merge short_memory_frames
405
+ similar_list = []
406
+ for frame_i in range(len(self.short_memory_buffer) - 1):
407
+ scores = self.short_memory_buffer[frame_i] @ self.short_memory_buffer[frame_i + 1].transpose(-1, -2)
408
+ frame_silimar = torch.mean(scores)
409
+ similar_list.append(frame_silimar)
410
+
411
+ while len(self.short_memory_buffer) > self.merge_frame_length:
412
+ max_value = max(similar_list)
413
+ max_index = similar_list.index(max_value)
414
+ new_frame_feature = (self.short_memory_buffer[max_index].cpu() + self.short_memory_buffer[max_index + 1].cpu()) / 2
415
+ self.short_memory_buffer[max_index] = new_frame_feature.cuda()
416
+ del self.short_memory_buffer[max_index + 1]
417
+ similar_list = []
418
+ for frame_i in range(len(self.short_memory_buffer) - 1):
419
+ scores = self.short_memory_buffer[frame_i] @ self.short_memory_buffer[frame_i + 1].transpose(-1, -2)
420
+ frame_silimar = torch.mean(scores)
421
+ similar_list.append(frame_silimar)
422
+
423
+ for frame in self.short_memory_buffer:
424
+ self.long_memory_buffer.append(frame)
425
+
426
+ image_features = torch.stack(self.long_memory_buffer)
427
+ except Exception as e:
428
+ print(e)
429
+ eval_logger.error(f"Error {e} in loading video")
430
+ image_features = None
431
+
432
+ task_type = "video"
433
+ placeholder_count = len(frames) if self.token_strategy == "multiple" else 1
434
+
435
+ if image_features is not None and DEFAULT_IMAGE_TOKEN not in context:
436
+ """
437
+ Three senarios:
438
+ 1. No image, and there for, no image token should be added.
439
+ 2. image token is already specified in the context, so we don't need to add it.
440
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
441
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
442
+ """
443
+ # if task_type == "image": # indeed in multi-image case, not the video in frames.
444
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
445
+ # elif task_type == "video":
446
+ # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
447
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count
448
+ image_tokens = " ".join(image_tokens)
449
+ question = image_tokens + "\n" + context
450
+ else:
451
+ question = context
452
+
453
+ # This is much safer for llama3, as we now have some object type in it
454
+ if "llama_3" in self.conv_template:
455
+ conv = copy.deepcopy(conv_templates[self.conv_template])
456
+ else:
457
+ conv = conv_templates[self.conv_template].copy()
458
+
459
+ if utils.is_json(question): # conversational question input
460
+ question = json.loads(question)
461
+ for idx, item in enumerate(question):
462
+ role = conv.roles[idx % 2]
463
+ message = item["value"]
464
+ conv.append_message(role, message)
465
+
466
+ assert len(conv.messages) % 2 == 1
467
+ conv.append_message(conv.roles[1], None)
468
+ prompt_question = conv.get_prompt()
469
+ question_input.append(prompt_question)
470
+ else: # only simple string for question
471
+ conv.append_message(conv.roles[0], question)
472
+ conv.append_message(conv.roles[1], None)
473
+ prompt_question = conv.get_prompt()
474
+ question_input.append(prompt_question)
475
+
476
+ # preconfigure gen_kwargs with defaults
477
+ if "max_new_tokens" not in gen_kwargs:
478
+ gen_kwargs["max_new_tokens"] = 1024
479
+ if "temperature" not in gen_kwargs:
480
+ gen_kwargs["temperature"] = 0
481
+ if "do_sample" not in gen_kwargs:
482
+ gen_kwargs["do_sample"] = False
483
+ if "top_p" not in gen_kwargs:
484
+ gen_kwargs["top_p"] = None
485
+ if "num_beams" not in gen_kwargs:
486
+ gen_kwargs["num_beams"] = 1
487
+
488
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
489
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
490
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
491
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
492
+
493
+ if task_type == "image":
494
+ gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))]
495
+ elif task_type == "video":
496
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
497
+ keywords = [stop_str]
498
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
499
+ gen_kwargs["modalities"] = ["video"]
500
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
501
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
502
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
503
+
504
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
505
+ # TODO: attention to this major generation step...
506
+ if "image_aspect_ratio" in gen_kwargs.keys():
507
+ gen_kwargs.pop("image_aspect_ratio")
508
+ try:
509
+ with torch.inference_mode():
510
+ gen_kwargs.pop("modalities")
511
+ cont = self.model.generate_moviechat(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, image_features=image_features, use_cache=self.use_cache, **gen_kwargs)
512
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
513
+ text_outputs = [response.strip() for response in text_outputs]
514
+ except Exception as e:
515
+ print(e)
516
+ text_outputs = "Can not infer the answer."
517
+
518
+ res.extend(text_outputs)
519
+ print(text_outputs)
520
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
521
+ pbar.update(1)
522
+ # reorder this group of results back to original unsorted form
523
+ res = re_ords.get_original(res)
524
+
525
+ pbar.close()
526
+ return res
LaViDa/eval/lmms_eval/models/llava_sglang.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+
3
+ import torch
4
+
5
+ torch.backends.cuda.matmul.allow_tf32 = True
6
+
7
+
8
+ import warnings
9
+ from datetime import timedelta
10
+ from typing import List, Optional, Tuple, Union
11
+
12
+ from tqdm import tqdm
13
+
14
+ from lmms_eval import utils
15
+ from lmms_eval.api.instance import Instance
16
+ from lmms_eval.api.model import lmms
17
+ from lmms_eval.api.registry import register_model
18
+
19
+ warnings.filterwarnings("ignore")
20
+ import tempfile
21
+ from concurrent.futures import ThreadPoolExecutor, as_completed
22
+
23
+ from loguru import logger as eval_logger
24
+
25
+ try:
26
+ import sglang as sgl
27
+ from sglang.lang.chat_template import get_chat_template
28
+ except ImportError:
29
+ eval_logger.debug("SGLang is not installed. If you want to use llava_sglang, please install it using pip install 'sglang[all]' ")
30
+
31
+ if torch.__version__ > "2.1.2":
32
+ best_fit_attn_implementation = "sdpa"
33
+ else:
34
+ best_fit_attn_implementation = "eager"
35
+
36
+
37
+ @register_model("llava_sglang")
38
+ class LlavaSglang(lmms):
39
+ """
40
+ Llava Sglang Model
41
+ """
42
+
43
+ def __init__(
44
+ self,
45
+ pretrained: str = "liuhaotian/llava-v1.5-7b",
46
+ tokenizer: str = "llava-hf/llava-1.5-7b-hf",
47
+ tp_size: int = 1,
48
+ parallel: Optional[Union[int, str]] = 64,
49
+ conv_template="vicuna_v1.1",
50
+ **kwargs,
51
+ ) -> None:
52
+ super().__init__()
53
+ self.pretrained = pretrained
54
+ self.tokenizer = tokenizer
55
+ self.tp_size = tp_size
56
+ self.conv_template = conv_template
57
+ # torch.multiprocessing.set_start_method("spawn")
58
+
59
+ # accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
60
+ # accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
61
+ # assert accelerator.num_processes == 1, "Llava-sglang does not support multi-processes yet (it does support tensor parallelism)."
62
+ self._rank = 0
63
+ self._world_size = 1
64
+ self.parallel = parallel
65
+
66
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
67
+ raise NotImplementedError("Llava-sglang does not support loglikelihood evaluation yet")
68
+
69
+ def generate_until(self, requests: List[Instance]) -> List[str]:
70
+ torch.multiprocessing.set_start_method("spawn", force=True)
71
+ runtime = sgl.Runtime(model_path=self.pretrained, tokenizer_path=self.tokenizer, tp_size=self.tp_size, port=random.randint(10000, 50000))
72
+ runtime.endpoint.chat_template = get_chat_template(self.conv_template)
73
+ sgl.set_default_backend(runtime)
74
+
75
+ @sgl.function
76
+ def image_qa(s, image_file, question):
77
+ s += sgl.user(sgl.image(image_file) + question)
78
+ s += sgl.assistant(sgl.gen("answer"))
79
+
80
+ res = []
81
+
82
+ def _collate(x):
83
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
84
+ # - time estimates will always be over not underestimates, which is more useful for planning
85
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
86
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
87
+ # automatic adaptive batches much much easier to implement
88
+ # - any OOMs will happen right away rather than near the end
89
+ toks = x[0].split(" ")
90
+ return -len(toks), x[0]
91
+
92
+ # we group requests by their generation_kwargs,
93
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
94
+ # in the same batch.
95
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
96
+ chunks = re_ords.get_batched(n=self.parallel, batch_fn=None)
97
+ num_iters = len(requests) // self.parallel if len(requests) % self.parallel == 0 else len(requests) // self.parallel + 1
98
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
99
+ for chunk in chunks:
100
+ contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
101
+ batched_visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)] # [B, N]
102
+ # we assume all gen kwargs in the batch are the same
103
+ # this is safe to assume because the `grouper` object ensures it.
104
+ gen_kwargs = all_gen_kwargs[0]
105
+ if "max_new_tokens" not in gen_kwargs:
106
+ gen_kwargs["max_new_tokens"] = 1024
107
+ if "temperature" not in gen_kwargs:
108
+ gen_kwargs["temperature"] = 0
109
+ if "top_p" not in gen_kwargs:
110
+ gen_kwargs["top_p"] = 1.0
111
+ if "num_beams" not in gen_kwargs:
112
+ gen_kwargs["num_beams"] = 1
113
+ assert gen_kwargs["num_beams"] == 1
114
+
115
+ def save_image_to_temp_file(image):
116
+ temp_file = tempfile.NamedTemporaryFile(suffix=".jpeg", delete=True)
117
+ image.save(temp_file.name)
118
+ return temp_file
119
+
120
+ def prepare_arguments_parallel(contexts, batched_visuals, max_workers=64):
121
+ arguments = [None] * len(contexts) # Initialize with placeholders
122
+ tmp_files = [None] * len(contexts) # Initialize with placeholders
123
+
124
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
125
+ # Associate each future with its index and content
126
+ future_to_info = {executor.submit(save_image_to_temp_file, pil_list[0]): (index, context, pil_list) for index, (context, pil_list) in enumerate(zip(contexts, batched_visuals))}
127
+
128
+ for future in as_completed(future_to_info):
129
+ index, context, pil_list = future_to_info[future]
130
+ if len(pil_list) > 1:
131
+ eval_logger.warning("Llava-sglang only supports one visual input per question. Using the first visual input.")
132
+ try:
133
+ temp_file = future.result()
134
+ arguments[index] = {
135
+ "image_file": temp_file.name,
136
+ "question": context,
137
+ }
138
+ tmp_files[index] = temp_file
139
+ except Exception as exc:
140
+ print(f"Generated an exception: {exc}")
141
+
142
+ # Filter out any None values in case of exceptions
143
+ arguments = [arg for arg in arguments if arg is not None]
144
+ tmp_files = [tmp_file for tmp_file in tmp_files if tmp_file is not None]
145
+
146
+ return arguments, tmp_files
147
+
148
+ arguments, tmp_files = prepare_arguments_parallel(contexts, batched_visuals, self.parallel)
149
+ states = image_qa.run_batch(arguments, temperature=gen_kwargs["temperature"], max_new_tokens=gen_kwargs["max_new_tokens"], top_p=gen_kwargs["top_p"], num_threads=self.parallel, progress_bar=False)
150
+
151
+ text_outputs = [state["answer"].strip() for state in states]
152
+ # clean up the temporary files
153
+ for tmp_file in tmp_files:
154
+ tmp_file.close()
155
+ res.extend(text_outputs)
156
+ pbar.update(1)
157
+ # reorder this group of results back to original unsorted form
158
+ res = re_ords.get_original(res)
159
+
160
+ pbar.close()
161
+ runtime.shutdown()
162
+ return res
163
+
164
+ def generate_until_multi_round(self, requests) -> List[str]:
165
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVA-SGLang")
LaViDa/eval/lmms_eval/models/llava_vid.py ADDED
@@ -0,0 +1,533 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import glob
2
+ import math
3
+ import os
4
+ from datetime import timedelta
5
+ from typing import List, Optional, Tuple, Union
6
+
7
+ import numpy as np
8
+ import torch
9
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
10
+ from accelerate.state import AcceleratorState
11
+ from decord import VideoReader, cpu
12
+ from llava.constants import (
13
+ DEFAULT_IM_END_TOKEN,
14
+ DEFAULT_IM_START_TOKEN,
15
+ DEFAULT_IMAGE_TOKEN,
16
+ IGNORE_INDEX,
17
+ IMAGE_TOKEN_INDEX,
18
+ )
19
+ from llava.conversation import SeparatorStyle, conv_templates
20
+ from llava.mm_utils import (
21
+ KeywordsStoppingCriteria,
22
+ get_model_name_from_path,
23
+ tokenizer_image_token,
24
+ )
25
+ from llava.model.builder import load_pretrained_model
26
+ from llava.model.language_model.llava_llama import LlavaConfig
27
+ from llava.model.language_model.llava_qwen import LlavaQwenConfig
28
+
29
+ # eval_logger = logging.getLogger("lmms-eval")
30
+ # import sys;sys.path.append("llava-video")
31
+ from loguru import logger as eval_logger
32
+ from PIL import Image
33
+ from tqdm import tqdm
34
+ from transformers import AutoConfig, AutoModelForCausalLM
35
+
36
+ from lmms_eval.api.instance import Instance
37
+ from lmms_eval.api.model import lmms
38
+ from lmms_eval.api.registry import register_model
39
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
40
+
41
+ # try:
42
+ # from llavavid.model.builder import load_pretrained_model
43
+ # from llavavid.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
44
+ # from llavavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
45
+ # from llavavid.conversation import conv_templates, SeparatorStyle
46
+ # from llavavid.mm_utils import tokenizer_image_token_qwen_merge, preprocess_qwen, preprocess_llama3
47
+ # except ImportError:
48
+ # import llava
49
+ # import pdb;pdb.set_trace()
50
+ # if "llava-video-old" in llava.__file__:
51
+ # from llava.model.language_model.llava_llama import LlavaConfig
52
+ # from llava.model.language_model.llava_qwen import LlavaQwenConfig
53
+ # from llava.model.builder import load_pretrained_model
54
+ # from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
55
+ # from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
56
+ # from llava.conversation import conv_templates, SeparatorStyle
57
+
58
+ # AutoConfig.register("llava_llama", LlavaConfig)
59
+ # AutoConfig.register("llava_qwen", LlavaQwenConfig)
60
+ # else:
61
+ # eval_logger.debug("LLaVA-Video is not installed. Please install LLaVA-Video to use this model.")
62
+
63
+ # from llavavid.model.language_model.llava_qwen import LlavaQwenConfig
64
+ # from llavavid.model.language_model.llava_llama import LlavaConfig
65
+
66
+ # AutoConfig.register("llava_qwen", LlavaQwenConfig)
67
+ # AutoConfig.register("llava_llama", LlavaConfig)
68
+
69
+
70
+ AutoConfig.register("llava_llama", LlavaConfig)
71
+ AutoConfig.register("llava_qwen", LlavaQwenConfig)
72
+
73
+
74
+ @register_model("llava_vid")
75
+ class LlavaVid(lmms):
76
+ """
77
+ LlavaVid Model
78
+ """
79
+
80
+ def __init__(
81
+ self,
82
+ pretrained: str = "liuhaotian/llava-v1.5-7b",
83
+ truncation: Optional[bool] = True,
84
+ torch_dtype: Optional[Union[str, torch.dtype]] = "float16",
85
+ device: Optional[str] = "cuda:0",
86
+ batch_size: Optional[Union[int, str]] = 1,
87
+ attn_implementation=(
88
+ "sdpa" if torch.__version__ >= "2.1.2" else "eager"
89
+ ), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
90
+ device_map="cuda:0",
91
+ conv_template="vicuna_v1",
92
+ use_cache=True,
93
+ truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
94
+ max_frames_num: int = 20,
95
+ video_fps: int = 1,
96
+ mm_resampler_type: str = "spatial_pool",
97
+ mm_spatial_pool_stride: int = 2,
98
+ mm_spatial_pool_out_channels: int = 1024,
99
+ mm_spatial_pool_mode: str = "average",
100
+ mm_resampler_location: str = "before",
101
+ mm_newline_position: str = "grid",
102
+ overwrite: bool = True,
103
+ video_decode_backend: str = "decord",
104
+ delay_load: bool = False,
105
+ tie_weights: bool = True,
106
+ force_sample: bool = False,
107
+ add_time_instruction: bool = False,
108
+ add_faster_video: bool = False,
109
+ faster_token_stride: int = 10,
110
+ **kwargs,
111
+ ) -> None:
112
+ super().__init__()
113
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
114
+
115
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
116
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
117
+ if accelerator.num_processes > 1:
118
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
119
+ self.device_map = f"cuda:{accelerator.local_process_index}"
120
+ elif accelerator.num_processes == 1 and (device_map == "auto" or device_map == "balanced_low_0"):
121
+ self._device = torch.device(device)
122
+ self.device_map = device_map
123
+ else:
124
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
125
+ self.device_map = f"cuda:{accelerator.local_process_index}"
126
+
127
+ self.pretrained = pretrained
128
+ self.model_name = get_model_name_from_path(pretrained)
129
+ self.video_decode_backend = video_decode_backend
130
+ # self._config = AutoConfig.from_pretrained(self.pretrained)
131
+ self.overwrite = overwrite
132
+ self.mm_resampler_type = mm_resampler_type
133
+ self.mm_spatial_pool_stride = int(mm_spatial_pool_stride)
134
+ self.mm_spatial_pool_out_channels = int(mm_spatial_pool_out_channels)
135
+ self.mm_spatial_pool_mode = mm_spatial_pool_mode
136
+ self.max_frames_num = int(max_frames_num)
137
+ self.fps = int(video_fps)
138
+ self.mm_resampler_location = mm_resampler_location
139
+ self.delay_load = delay_load
140
+ self.force_sample = force_sample
141
+ self.add_time_instruction = add_time_instruction
142
+ print("force sample:", self.force_sample)
143
+ # self.add_faster_video = add_faster_video
144
+ # self.faster_token_stride = faster_token_stride
145
+ self.torch_dtype = torch_dtype
146
+ if self.overwrite == True:
147
+ overwrite_config = {}
148
+ # overwrite_config["mm_resampler_type"] = self.mm_resampler_type
149
+ overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
150
+ overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
151
+ overwrite_config["mm_pooling_position"] = self.mm_resampler_location
152
+ overwrite_config["mm_newline_position"] = mm_newline_position
153
+ overwrite_config["delay_load"] = self.delay_load
154
+ # overwrite_config["attn_implementation"] = attn_implementation
155
+
156
+ if "vicuna" in self.pretrained.lower() or "yi" in self.pretrained.lower():
157
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
158
+
159
+ if "224" in cfg_pretrained.mm_vision_tower:
160
+ # suppose the length of text tokens is around 1000, from bo's report
161
+ least_token_number = self.max_frames_num * (16 // self.mm_spatial_pool_stride) ** 2 + 1000
162
+ else:
163
+ least_token_number = self.max_frames_num * (24 // self.mm_spatial_pool_stride) ** 2 + 1000
164
+
165
+ scaling_factor = math.ceil(least_token_number / 4096)
166
+ if scaling_factor >= 2:
167
+ eval_logger.info(f"Scaling factor: {scaling_factor}")
168
+ # print(float(scaling_factor))
169
+ overwrite_config["rope_scaling"] = {"factor": float(scaling_factor), "type": "linear"}
170
+ overwrite_config["max_sequence_length"] = 4096 * scaling_factor
171
+ overwrite_config["tokenizer_model_max_length"] = 4096 * scaling_factor
172
+
173
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(
174
+ pretrained, None, self.model_name, device_map=self.device_map, torch_dtype=self.torch_dtype, overwrite_config=overwrite_config, attn_implementation=attn_implementation
175
+ )
176
+ else:
177
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, self.model_name, device_map=self.device_map, torch_dtype=self.torch_dtype, attn_implementation=attn_implementation)
178
+
179
+ self._config = self._model.config
180
+
181
+ # import pdb;pdb.set_trace()
182
+
183
+ if self._tokenizer.pad_token_id is None:
184
+ if "qwen" in self._tokenizer.name_or_path.lower():
185
+ print("Setting pad token to bos token for qwen model.")
186
+ self._tokenizer.pad_token_id = 151643
187
+
188
+ self.model.eval()
189
+ if tie_weights:
190
+ self.model.tie_weights()
191
+ self.truncation = truncation
192
+ self.batch_size_per_gpu = int(batch_size)
193
+ self.conv_template = conv_template
194
+ self.use_cache = use_cache
195
+ self.truncate_context = truncate_context
196
+ # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
197
+ if accelerator.num_processes > 1:
198
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
199
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
200
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
201
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
202
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
203
+ kwargs = {
204
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
205
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
206
+ }
207
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
208
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
209
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
210
+ self._model = accelerator.prepare(self.model)
211
+ else:
212
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
213
+ self.accelerator = accelerator
214
+ if self.accelerator.is_local_main_process:
215
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
216
+ self._rank = self.accelerator.local_process_index
217
+ self._world_size = self.accelerator.num_processes
218
+ elif accelerator.num_processes == 1 and device_map == "auto":
219
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
220
+ self._rank = 0
221
+ self._world_size = 1
222
+ else:
223
+ eval_logger.info(f"Using single device: {self._device}")
224
+ self.model.to(self._device)
225
+ self._rank = 0
226
+ self._world_size = 1
227
+
228
+ @property
229
+ def config(self):
230
+ # return the associated transformers.AutoConfig for the given pretrained model.
231
+ return self._config
232
+
233
+ @property
234
+ def tokenizer(self):
235
+ return self._tokenizer
236
+
237
+ @property
238
+ def model(self):
239
+ # returns the model, unwrapping it if using Accelerate
240
+ if hasattr(self, "accelerator"):
241
+ return self.accelerator.unwrap_model(self._model)
242
+ else:
243
+ return self._model
244
+
245
+ @property
246
+ def eot_token_id(self):
247
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
248
+ return self.tokenizer.eos_token_id
249
+
250
+ @property
251
+ def max_length(self):
252
+ return self._max_length
253
+
254
+ def pad_sequence(self, input_ids, batch_first, padding_value):
255
+ if self.tokenizer.padding_side == "left":
256
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
257
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
258
+ if self.tokenizer.padding_side == "left":
259
+ input_ids = torch.flip(input_ids, [1])
260
+ return input_ids
261
+
262
+ @property
263
+ def batch_size(self):
264
+ return self.batch_size_per_gpu
265
+
266
+ @property
267
+ def device(self):
268
+ return self._device
269
+
270
+ @property
271
+ def rank(self):
272
+ return self._rank
273
+
274
+ @property
275
+ def world_size(self):
276
+ return self._world_size
277
+
278
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
279
+ """ """
280
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
281
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
282
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
283
+ if left_truncate_len:
284
+ encoding = encoding[-left_truncate_len:]
285
+ return encoding
286
+
287
+ def load_image(self, image_path):
288
+ frame_files = [os.path.join(image_path, f) for f in os.listdir(image_path) if os.path.isfile(os.path.join(image_path, f))]
289
+ frame_files.sort() # Ensure the frames are sorted if they are named sequentially
290
+
291
+ # TODO: Hard CODE: Determine the indices for uniformly sampling 10 frames
292
+ num_frames_to_sample = 10
293
+
294
+ total_frames = len(frame_files)
295
+
296
+ sampled_indices = np.linspace(0, total_frames - 1, num_frames_to_sample, dtype=int)
297
+
298
+ # Read and store the sampled frames
299
+ video = []
300
+ for idx in sampled_indices:
301
+ frame_path = frame_files[idx]
302
+ try:
303
+ with Image.open(frame_path) as img:
304
+ # Convert the PIL image to a numpy array if needed
305
+ # frame = np.array(img.convert('RGB'))
306
+ frame = img.convert("RGB")
307
+ video.append(frame)
308
+ except IOError:
309
+ print(f"Failed to read frame at path: {frame_path}")
310
+ return video
311
+
312
+ def load_video(self, video_path, max_frames_num, fps, force_sample=False):
313
+ if max_frames_num == 0:
314
+ return np.zeros((1, 336, 336, 3))
315
+ vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
316
+ total_frame_num = len(vr)
317
+ video_time = total_frame_num / vr.get_avg_fps()
318
+ fps = round(vr.get_avg_fps() / fps)
319
+ frame_idx = [i for i in range(0, len(vr), fps)]
320
+ frame_time = [i / fps for i in frame_idx]
321
+ if len(frame_idx) > max_frames_num or force_sample:
322
+ sample_fps = max_frames_num
323
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, sample_fps, dtype=int)
324
+ frame_idx = uniform_sampled_frames.tolist()
325
+ frame_time = [i / vr.get_avg_fps() for i in frame_idx]
326
+ frame_time = ",".join([f"{i:.2f}s" for i in frame_time])
327
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
328
+ # import pdb;pdb.set_trace()
329
+
330
+ return spare_frames, frame_time, video_time
331
+
332
+ def tok_decode(self, tokens):
333
+ return self.tokenizer.decode(tokens)
334
+
335
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
336
+ res = []
337
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
338
+
339
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
340
+ # encode, pad, and truncate contexts for this batch
341
+ if type(doc_to_target) == str:
342
+ continuation = doc_to_target
343
+ else:
344
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
345
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
346
+ visuals = self.flatten(visuals)
347
+ videos = []
348
+ for visual in visuals:
349
+ video, frame_time, video_time = self.load_video(visual, self.max_frames_num, self.fps, force_sample=self.force_sample)
350
+ video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].cuda()
351
+ if self.torch_dtype == "bfloat16":
352
+ video = video.bfloat16()
353
+ else:
354
+ video = video.half()
355
+ videos.append(video)
356
+
357
+ qs = contexts
358
+ if self.model.config.mm_use_im_start_end:
359
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
360
+ else:
361
+ qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
362
+
363
+ conv = conv_templates[self.conv_template].copy()
364
+ conv.append_message(conv.roles[0], qs)
365
+ conv.append_message(conv.roles[1], None)
366
+ prompt = conv.get_prompt()
367
+
368
+ contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
369
+
370
+ conv = conv_templates[self.conv_template].copy()
371
+ conv.append_message(conv.roles[0], qs)
372
+ conv.append_message(conv.roles[1], continuation)
373
+ prompt = conv.get_prompt()
374
+
375
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
376
+ attention_masks = input_ids.ne(self.tokenizer.pad_token_id).long().cuda()
377
+
378
+ labels = input_ids.clone()
379
+ # Context part no need to calculate for loss
380
+ labels[0, : contxt_id.shape[1]] = -100
381
+
382
+ with torch.inference_mode():
383
+ outputs = self.model(input_ids=input_ids, labels=labels, images=videos, modalities="video")
384
+
385
+ loss = outputs["loss"]
386
+ # loss = torch.exp(loss)
387
+ logits = outputs["logits"]
388
+ greedy_tokens = logits.argmax(dim=-1)
389
+ cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
390
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
391
+ max_equal = (greedy_tokens == cont_toks).all()
392
+ res.append((float(loss.item()), bool(max_equal)))
393
+ pbar.update(1)
394
+ pbar.close()
395
+ return res
396
+
397
+ def flatten(self, input):
398
+ new_list = []
399
+ for i in input:
400
+ for j in i:
401
+ new_list.append(j)
402
+ return new_list
403
+
404
+ def generate_until(self, requests) -> List[str]:
405
+ res = []
406
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
407
+
408
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
409
+ # if self.task_dict[task][split][doc_id]["duration"] != "short":
410
+ # # if doc_id != 112:
411
+ # # import pdb;pdb.set_trace()
412
+ # res.append("A")
413
+ # pbar.update(1)
414
+ # continue
415
+ # encode, pad, and truncate contexts for this batch
416
+ # import pdb;pdb.set_trace()
417
+ visuals = doc_to_visual(self.task_dict[task][split][doc_id])
418
+ # visuals = [visuals]
419
+ # visuals = self.flatten(visuals)
420
+ if os.path.isdir(visuals[0]):
421
+ visuals = glob.glob(visuals[0] + "/*")
422
+ videos = []
423
+ try:
424
+ # for visual in visuals:
425
+ if len(visuals) == 1:
426
+ if self.video_decode_backend == "decord":
427
+ video, frame_time, video_time = self.load_video(visuals[0], self.max_frames_num, self.fps, force_sample=self.force_sample)
428
+ elif self.video_decode_backend == "pyav":
429
+ video, frame_time, video_time = read_video_pyav(visuals[0], self.max_frames_num, self.fps, force_sample=self.force_sample)
430
+ elif self.video_decode_backend == "image":
431
+ video = self.load_image(visuals[0])
432
+ else:
433
+ if task == "seedbench":
434
+ video = visuals
435
+ frame_time = "1.00s"
436
+ video_time = 1
437
+ elif "mvbench" in task:
438
+ # video = visuals
439
+ # Reference: https://github.com/jayleicn/TVQA/blob/dfb0e5fe4582efca574dfddfeafd1008db3b33ef/data/README.md?plain=1#L50C34-L50C60
440
+ fps = 3
441
+ video_time = len(visuals) / fps
442
+ sampled_indices = np.linspace(0, len(visuals) - 1, self.max_frames_num, dtype=int)
443
+ frame_idx = sampled_indices.tolist()
444
+ frame_time = [i / fps for i in frame_idx]
445
+ frame_time = ",".join([f"{i:.2f}s" for i in frame_time])
446
+ # video = [visuals[i] for i in frame_idx]
447
+ video = np.stack([np.array(Image.open(visuals[i])) for i in frame_idx], axis=0)
448
+
449
+ video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].cuda()
450
+ if self.torch_dtype == "bfloat16":
451
+ video = video.bfloat16()
452
+ else:
453
+ video = video.half()
454
+ videos.append(video)
455
+ except Exception as e:
456
+ # import pdb;pdb.set_trace()
457
+ eval_logger.info(f"{e}")
458
+ eval_logger.info(f"Video {visuals} can not load, check the source")
459
+ video_path = "\n".join(visuals)
460
+ res.append(f"Video {video_path} can not load, check the source")
461
+ pbar.update(1)
462
+ continue
463
+
464
+ qs = contexts
465
+ # import pdb;pdb.set_trace()
466
+ if self.add_time_instruction:
467
+ time_instruciton = f"The video lasts for {video_time:.2f} seconds, and {len(video)} frames are uniformly sampled from it. These frames are located at {frame_time}.Please answer the following questions related to this video."
468
+ qs = f"{time_instruciton}\n{qs}"
469
+ if self.model.config.mm_use_im_start_end:
470
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
471
+ else:
472
+ qs = DEFAULT_IMAGE_TOKEN * len(videos) + "\n" + qs
473
+
474
+ # This is much safer for llama3, as we now have some object type in it
475
+ if "llama_3" in self.conv_template:
476
+ conv = copy.deepcopy(conv_templates[self.conv_template])
477
+ else:
478
+ conv = conv_templates[self.conv_template].copy()
479
+
480
+ conv.append_message(conv.roles[0], qs)
481
+ conv.append_message(conv.roles[1], None)
482
+ prompt = conv.get_prompt()
483
+
484
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda()
485
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
486
+ if "llama_3" in self.conv_template:
487
+ pad_token_ids = 0 # lmms-lab/llama3-llava-8b is trained on this pad token id. You may need to customize this for other models.
488
+ attention_masks = input_ids.ne(pad_token_ids).long().cuda()
489
+
490
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
491
+ keywords = [stop_str]
492
+
493
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
494
+
495
+ cur_prompt = qs
496
+
497
+ if "max_new_tokens" not in gen_kwargs:
498
+ gen_kwargs["max_new_tokens"] = 1024
499
+ if "temperature" not in gen_kwargs:
500
+ gen_kwargs["temperature"] = 0
501
+ if "top_p" not in gen_kwargs:
502
+ gen_kwargs["top_p"] = None
503
+ if "num_beams" not in gen_kwargs:
504
+ gen_kwargs["num_beams"] = 1
505
+
506
+ # import pdb;pdb.set_trace()
507
+ with torch.inference_mode():
508
+ output_ids = self.model.generate(
509
+ inputs=input_ids,
510
+ images=videos,
511
+ attention_mask=attention_masks,
512
+ modalities="video",
513
+ use_cache=self.use_cache,
514
+ stopping_criteria=[stopping_criteria],
515
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
516
+ temperature=gen_kwargs["temperature"],
517
+ top_p=gen_kwargs["top_p"],
518
+ num_beams=gen_kwargs["num_beams"],
519
+ max_new_tokens=gen_kwargs["max_new_tokens"],
520
+ )
521
+ # output_ids_2 = self.model.generate(inputs=input_ids, images=videos, attention_mask=attention_masks, modalities="video", do_sample=False, max_new_tokens=50,stopping_criteria=[stopping_criteria])
522
+ # output_ids = self.model.generate(inputs=input_ids, images=videos, attention_mask=attention_masks, modalities="video", do_sample=True, temperature=0.2, max_new_tokens=50,use_cache=True)
523
+
524
+ outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
525
+ eval_logger.debug(f"Question: {cur_prompt}")
526
+ eval_logger.debug(f"Answer: {outputs}")
527
+ # import pdb;pdb.set_trace()
528
+ res.append(outputs)
529
+ pbar.update(1)
530
+ return res
531
+
532
+ def generate_until_multi_round(self, requests) -> List[str]:
533
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVAVid")
LaViDa/eval/lmms_eval/models/longva.py ADDED
@@ -0,0 +1,476 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+
3
+ import torch
4
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
5
+ from accelerate.state import AcceleratorState
6
+ from transformers import AutoConfig
7
+
8
+ torch.backends.cuda.matmul.allow_tf32 = True
9
+
10
+ import copy
11
+ import logging
12
+ import warnings
13
+ from datetime import timedelta
14
+ from typing import List, Optional, Tuple, Union
15
+
16
+ import numpy as np
17
+ import PIL
18
+ from decord import VideoReader, cpu
19
+ from packaging import version
20
+ from tqdm import tqdm
21
+
22
+ warnings.filterwarnings("ignore")
23
+
24
+ eval_logger = logging.getLogger("lmms-eval")
25
+
26
+ from lmms_eval import utils
27
+ from lmms_eval.api.instance import Instance
28
+ from lmms_eval.api.model import lmms
29
+ from lmms_eval.api.registry import register_model
30
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
31
+
32
+ try:
33
+ from longva.constants import (
34
+ DEFAULT_IM_END_TOKEN,
35
+ DEFAULT_IM_START_TOKEN,
36
+ DEFAULT_IMAGE_TOKEN,
37
+ IGNORE_INDEX,
38
+ IMAGE_TOKEN_INDEX,
39
+ )
40
+ from longva.conversation import SeparatorStyle, conv_templates
41
+ from longva.mm_utils import (
42
+ KeywordsStoppingCriteria,
43
+ get_model_name_from_path,
44
+ process_images,
45
+ tokenizer_image_token,
46
+ )
47
+ from longva.model.builder import load_pretrained_model
48
+
49
+ except Exception as e:
50
+ eval_logger.debug("longva is not installed. Please install longva to use this model.\nError: %s" % e)
51
+
52
+ # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
53
+ # if is_flash_attn_2_available:
54
+ # best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
55
+
56
+ if version.parse(torch.__version__) >= version.parse("2.1.2"):
57
+ best_fit_attn_implementation = "sdpa"
58
+ else:
59
+ best_fit_attn_implementation = "eager"
60
+
61
+
62
+ @register_model("longva")
63
+ class LongVA(lmms):
64
+ def __init__(
65
+ self,
66
+ pretrained: str = "lmms-lab/LongVA-7B",
67
+ truncation: Optional[bool] = True,
68
+ device: Optional[str] = "cuda:0",
69
+ batch_size: Optional[Union[int, str]] = 1,
70
+ model_name: Optional[str] = None,
71
+ attn_implementation: Optional[str] = best_fit_attn_implementation,
72
+ device_map: Optional[str] = "cuda:0",
73
+ conv_template: Optional[str] = "vicuna_v1",
74
+ use_cache: Optional[bool] = True,
75
+ truncate_context: Optional[bool] = False, # whether to truncate the context in generation, set it False for LLaVA-1.6
76
+ customized_config: Optional[str] = None, # ends in json
77
+ max_frames_num: Optional[int] = 32,
78
+ mm_spatial_pool_stride: Optional[int] = 2,
79
+ mm_spatial_pool_mode: Optional[str] = "average",
80
+ token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple <image> tokens for each frame
81
+ video_decode_backend: str = "pyav",
82
+ **kwargs,
83
+ ) -> None:
84
+ super().__init__()
85
+ # Do not use kwargs for now
86
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
87
+
88
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
89
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
90
+ self.accelerator = accelerator
91
+ if accelerator.num_processes > 1:
92
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
93
+ self.device_map = f"cuda:{accelerator.local_process_index}"
94
+ elif accelerator.num_processes == 1 and device_map == "auto":
95
+ self._device = torch.device(device)
96
+ self.device_map = device_map
97
+ else:
98
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
99
+ self.device_map = f"cuda:{accelerator.local_process_index}"
100
+
101
+ llava_model_args = {
102
+ "multimodal": True,
103
+ }
104
+ if customized_config is not None:
105
+ llava_model_args["customized_config"] = customized_config
106
+ if attn_implementation is not None:
107
+ llava_model_args["attn_implementation"] = attn_implementation
108
+ if "use_flash_attention_2" in kwargs:
109
+ llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
110
+ model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
111
+
112
+ self.pretrained = pretrained
113
+ self.token_strategy = token_strategy
114
+ self.max_frames_num = max_frames_num
115
+ self.mm_spatial_pool_stride = mm_spatial_pool_stride
116
+ self.mm_spatial_pool_mode = mm_spatial_pool_mode
117
+ self.video_decode_backend = video_decode_backend
118
+
119
+ overwrite_config = {}
120
+ overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
121
+ overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
122
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
123
+
124
+ llava_model_args["overwrite_config"] = overwrite_config
125
+ try:
126
+ # Try to load the model with the multimodal argument
127
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
128
+ except TypeError:
129
+ # for older versions of LLaVA that don't have multimodal argument
130
+ llava_model_args.pop("multimodal", None)
131
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
132
+
133
+ self._config = self._model.config
134
+ self.model.eval()
135
+ self.model.tie_weights()
136
+ self.truncation = truncation
137
+ self.batch_size_per_gpu = int(batch_size)
138
+ self.conv_template = conv_template
139
+ self.use_cache = use_cache
140
+ self.truncate_context = truncate_context
141
+ assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
142
+
143
+ if accelerator.num_processes > 1:
144
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
145
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
146
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
147
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
148
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
149
+ kwargs = {
150
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
151
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
152
+ }
153
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
154
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
155
+
156
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
157
+ self._model = accelerator.prepare(self.model)
158
+ else:
159
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
160
+ self.accelerator = accelerator
161
+ if self.accelerator.is_local_main_process:
162
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
163
+ self._rank = self.accelerator.local_process_index
164
+ self._world_size = self.accelerator.num_processes
165
+
166
+ elif accelerator.num_processes == 1 and device_map == "auto":
167
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
168
+ self._rank = 0
169
+ self._world_size = 1
170
+
171
+ else:
172
+ eval_logger.info(f"Using single device: {self._device}")
173
+ self.model.to(self._device)
174
+ self._rank = 0
175
+ self._world_size = 1
176
+
177
+ @property
178
+ def config(self):
179
+ # return the associated transformers.AutoConfig for the given pretrained model.
180
+ return self._config
181
+
182
+ @property
183
+ def tokenizer(self):
184
+ return self._tokenizer
185
+
186
+ @property
187
+ def model(self):
188
+ # returns the model, unwrapping it if using Accelerate
189
+ if hasattr(self, "accelerator"):
190
+ return self.accelerator.unwrap_model(self._model)
191
+ else:
192
+ return self._model
193
+
194
+ @property
195
+ def eot_token_id(self):
196
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
197
+ return self.tokenizer.eos_token_id
198
+
199
+ @property
200
+ def max_length(self):
201
+ return self._max_length
202
+
203
+ def pad_sequence(self, input_ids, batch_first, padding_value):
204
+ if self.tokenizer.padding_side == "left":
205
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
206
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
207
+ if self.tokenizer.padding_side == "left":
208
+ input_ids = torch.flip(input_ids, [1])
209
+ return input_ids
210
+
211
+ @property
212
+ def batch_size(self):
213
+ return self.batch_size_per_gpu
214
+
215
+ @property
216
+ def device(self):
217
+ return self._device
218
+
219
+ @property
220
+ def rank(self):
221
+ return self._rank
222
+
223
+ @property
224
+ def world_size(self):
225
+ return self._world_size
226
+
227
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
228
+ """ """
229
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
230
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
231
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
232
+ if left_truncate_len:
233
+ encoding = encoding[-left_truncate_len:]
234
+ return encoding
235
+
236
+ def tok_decode(self, tokens):
237
+ try:
238
+ return self.tokenizer.decode(tokens)
239
+ except:
240
+ return self.tokenizer.decode([tokens])
241
+
242
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
243
+ # TODO
244
+ res = []
245
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
246
+
247
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
248
+ # encode, pad, and truncate contexts for this batch
249
+ if type(doc_to_target) == str:
250
+ continuation = doc_to_target
251
+ else:
252
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
253
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
254
+ visuals = self.flatten(visuals)
255
+ image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
256
+ if visuals:
257
+ image = process_images(visuals, self._image_processor, self._config)
258
+ if type(image) is list:
259
+ image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
260
+ else:
261
+ image = image.to(dtype=torch.float16, device=self.device)
262
+ else:
263
+ image = None
264
+
265
+ prompts_input = contexts[0] if isinstance(contexts, list) else contexts
266
+
267
+ if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
268
+ """
269
+ Three senarios:
270
+ 1. No image, and there for, no image token should be added.
271
+ 2. image token is already specified in the context, so we don't need to add it.
272
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
273
+ """
274
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
275
+ image_tokens = " ".join(image_tokens)
276
+ prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts)
277
+
278
+ # This is much safer for llama3, as we now have some object type in it
279
+ if "llama_3" in self.conv_template:
280
+ conv = copy.deepcopy(conv_templates[self.conv_template])
281
+ else:
282
+ conv = conv_templates[self.conv_template].copy()
283
+
284
+ conv.append_message(conv.roles[0], prompts_input)
285
+ conv.append_message(conv.roles[1], None)
286
+ prompt = conv.get_prompt()
287
+ pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
288
+ contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
289
+ # Add the answer of the second role
290
+ conv.messages[1][1] = continuation
291
+
292
+ prompt = conv.get_prompt()
293
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
294
+ labels = input_ids.clone()
295
+ # Context part no need to calculate for loss
296
+ labels[0, : contxt_id.shape[1]] = -100
297
+ with torch.inference_mode():
298
+ outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
299
+ loss = outputs["loss"]
300
+ # loss = torch.exp(loss)
301
+ logits = outputs["logits"]
302
+ greedy_tokens = logits.argmax(dim=-1)
303
+ cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
304
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
305
+ max_equal = (greedy_tokens == cont_toks).all()
306
+ res.append((float(loss.item()), bool(max_equal)))
307
+ pbar.update(1)
308
+
309
+ pbar.close()
310
+ return res
311
+
312
+ def flatten(self, input):
313
+ new_list = []
314
+ for i in input:
315
+ for j in i:
316
+ new_list.append(j)
317
+ return new_list
318
+
319
+ def load_video(self, video_path, max_frames_num):
320
+ if type(video_path) == str:
321
+ vr = VideoReader(video_path, ctx=cpu(0))
322
+ else:
323
+ vr = VideoReader(video_path[0], ctx=cpu(0))
324
+ total_frame_num = len(vr)
325
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
326
+ frame_idx = uniform_sampled_frames.tolist()
327
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
328
+ return spare_frames # (frames, height, width, channels)
329
+
330
+ def generate_until(self, requests: List[Instance]) -> List[str]:
331
+ res = []
332
+
333
+ def _collate(x):
334
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
335
+ # - time estimates will always be over not underestimates, which is more useful for planning
336
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
337
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
338
+ # automatic adaptive batches much much easier to implement
339
+ # - any OOMs will happen right away rather than near the end
340
+ toks = self.tok_encode(x[0])
341
+ return -len(toks), x[0]
342
+
343
+ # we group requests by their generation_kwargs,
344
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
345
+ # in the same batch.
346
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
347
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
348
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
349
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
350
+ for chunk in chunks:
351
+ batched_contexts, all_gen_kwargs, batched_doc_to_visual, batched_doc_id, batched_task, batched_split = zip(*chunk)
352
+ task = batched_task[0]
353
+ split = batched_split[0]
354
+ batched_visuals = [batched_doc_to_visual[0](self.task_dict[task][split][ids]) for ids in batched_doc_id] # [B, N]
355
+ flattened_visuals = self.flatten(batched_visuals) # [B*N]
356
+ assert len(batched_visuals) == 1
357
+
358
+ # we assume all gen kwargs in the batch are the same
359
+ # this is safe to assume because the `grouper` object ensures it.
360
+ gen_kwargs = all_gen_kwargs[0]
361
+ if "until" in gen_kwargs:
362
+ gen_kwargs.pop("until")
363
+
364
+ question_input = []
365
+
366
+ for visual, context in zip(batched_visuals, batched_contexts):
367
+ if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
368
+ # here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
369
+ self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
370
+ eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
371
+
372
+ # encode, pad, and truncate contexts for this batch
373
+ if type(visual[0]) == PIL.Image.Image: # For image task
374
+ image_tensor = process_images(visual, self._image_processor, self._config)
375
+ if type(image_tensor) is list:
376
+ image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
377
+ else:
378
+ image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
379
+
380
+ task_type = "image"
381
+
382
+ elif type(visual[0]) == str: # For video task
383
+ image_tensor = []
384
+ try:
385
+ if self.video_decode_backend == "decord":
386
+ frames = self.load_video(visual, self.max_frames_num)
387
+ elif self.video_decode_backend == "pyav":
388
+ frames = read_video_pyav(visual[0], num_frm=self.max_frames_num)
389
+ frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda()
390
+ image_tensor.append(frames)
391
+ except Exception as e:
392
+ eval_logger.error(f"Error {e} in loading video")
393
+ image_tensor = None
394
+
395
+ task_type = "video"
396
+
397
+ if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
398
+ """
399
+ Three senarios:
400
+ 1. No image, and there for, no image token should be added.
401
+ 2. image token is already specified in the context, so we don't need to add it.
402
+ 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
403
+ 4. For video tasks, we could add a <image> token or multiple <image> tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better
404
+ """
405
+ if task_type == "image":
406
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
407
+ elif task_type == "video":
408
+ image_tokens = [DEFAULT_IMAGE_TOKEN] * len(frames) if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN]
409
+
410
+ image_tokens = " ".join(image_tokens)
411
+ question = image_tokens + "\n" + context
412
+ else:
413
+ question = context
414
+
415
+ # This is much safer for llama3, as we now have some object type in it
416
+ if "llama_3" in self.conv_template:
417
+ conv = copy.deepcopy(conv_templates[self.conv_template])
418
+ else:
419
+ conv = conv_templates[self.conv_template].copy()
420
+ conv.append_message(conv.roles[0], question)
421
+ conv.append_message(conv.roles[1], None)
422
+ prompt_question = conv.get_prompt()
423
+ question_input.append(prompt_question)
424
+
425
+ # preconfigure gen_kwargs with defaults
426
+ if "max_new_tokens" not in gen_kwargs:
427
+ gen_kwargs["max_new_tokens"] = 1024
428
+ if "temperature" not in gen_kwargs:
429
+ gen_kwargs["temperature"] = 0
430
+ if "do_sample" not in gen_kwargs:
431
+ gen_kwargs["do_sample"] = False
432
+ if "top_p" not in gen_kwargs:
433
+ gen_kwargs["top_p"] = None
434
+ if "num_beams" not in gen_kwargs:
435
+ gen_kwargs["num_beams"] = 1
436
+
437
+ input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
438
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
439
+ input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
440
+ attention_masks = input_ids.ne(pad_token_ids).to(self.device)
441
+
442
+ if task_type == "image":
443
+ gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
444
+ elif task_type == "video":
445
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
446
+ keywords = [stop_str]
447
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
448
+ gen_kwargs["modalities"] = ["video"]
449
+ gen_kwargs["stopping_criteria"] = [stopping_criteria]
450
+ self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride
451
+ self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode
452
+
453
+ # These steps are not in LLaVA's original code, but are necessary for generation to work
454
+ # TODO: attention to this major generation step...
455
+ if "image_aspect_ratio" in gen_kwargs.keys():
456
+ gen_kwargs.pop("image_aspect_ratio")
457
+ try:
458
+ with torch.inference_mode():
459
+ cont = self.model.generate(input_ids, attention_mask=attention_masks, pad_token_id=pad_token_ids, images=image_tensor, use_cache=self.use_cache, **gen_kwargs)
460
+
461
+ text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
462
+ except Exception as e:
463
+ raise e
464
+
465
+ text_outputs = [response.strip() for response in text_outputs]
466
+ res.extend(text_outputs)
467
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
468
+ pbar.update(1)
469
+ # reorder this group of results back to original unsorted form
470
+ res = re_ords.get_original(res)
471
+
472
+ pbar.close()
473
+ return res
474
+
475
+ def generate_until_multi_round(self, requests) -> List[str]:
476
+ raise NotImplementedError("TODO: Implement multi-round generation for LongVA")
LaViDa/eval/lmms_eval/models/mantis.py ADDED
@@ -0,0 +1,312 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ torch.backends.cuda.matmul.allow_tf32 = True
4
+
5
+
6
+ import copy
7
+ import warnings
8
+ from datetime import timedelta
9
+ from typing import List, Optional, Tuple, Union
10
+
11
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
12
+ from accelerate.state import AcceleratorState
13
+ from loguru import logger as eval_logger
14
+ from packaging import version
15
+ from tqdm import tqdm
16
+
17
+ from lmms_eval import utils
18
+ from lmms_eval.api.instance import Instance
19
+ from lmms_eval.api.model import lmms
20
+ from lmms_eval.api.registry import register_model
21
+ from lmms_eval.utils import stop_sequences_criteria
22
+
23
+ warnings.filterwarnings("ignore")
24
+
25
+ try:
26
+ from mantis.models.conversation import conv_mllava_v1 as default_conv
27
+ from mantis.models.conversation import conv_templates
28
+ from mantis.models.mfuyu import MFuyuForCausalLM, MFuyuProcessor
29
+ from mantis.models.mllava import LlavaForConditionalGeneration, MLlavaProcessor
30
+
31
+ except Exception as e:
32
+ eval_logger.debug("Mantis is not installed. Please install Mantis to use this model.\nError: %s" % e)
33
+
34
+ try:
35
+ from transformers import AutoModelForVision2Seq, AutoProcessor
36
+ except Exception as e:
37
+ eval_logger.debug("Upgrade transformers to use Mantis's idefics model.\nError: %s" % e)
38
+
39
+ # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
40
+ # if is_flash_attn_2_available:
41
+ # best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
42
+
43
+ try:
44
+ import flash_attn
45
+
46
+ best_fit_attn_implementation = "flash_attention_2"
47
+ except ImportError:
48
+ best_fit_attn_implementation = "eager"
49
+
50
+ DEFAULT_IMAGE_TOKEN = "<image>"
51
+
52
+
53
+ @register_model("mantis")
54
+ class Mantis(lmms):
55
+ """
56
+ Mantis Model
57
+ This implementation is adpated from the Llava model from llava.py and the Idefics model from idefics.py
58
+ """
59
+
60
+ def __init__(
61
+ self,
62
+ pretrained: str = "TIGER-Lab/Mantis-8B-siglip-llama3",
63
+ truncation: Optional[bool] = True,
64
+ device: Optional[str] = "cuda:0",
65
+ dtype: Optional[Union[str, torch.dtype]] = "float16",
66
+ batch_size: Optional[Union[int, str]] = 1,
67
+ attn_implementation=best_fit_attn_implementation,
68
+ device_map="cuda:0",
69
+ use_cache=True,
70
+ truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6
71
+ **kwargs,
72
+ ) -> None:
73
+ super().__init__()
74
+ # Do not use kwargs for now
75
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
76
+
77
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
78
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
79
+ if accelerator.num_processes > 1:
80
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
81
+ self.device_map = f"cuda:{accelerator.local_process_index}"
82
+ elif accelerator.num_processes == 1 and device_map == "auto":
83
+ self._device = torch.device(device)
84
+ self.device_map = device_map
85
+ else:
86
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
87
+ self.device_map = f"cuda:{accelerator.local_process_index}"
88
+
89
+ self._is_idefics = "idefics" in pretrained.lower()
90
+ if isinstance(dtype, str) and dtype != "auto":
91
+ dtype = getattr(torch, dtype)
92
+
93
+ # Here we load the "non-idefics" Mantis model.
94
+ if not self._is_idefics:
95
+ if "fuyu" in pretrained.lower():
96
+ self._processor = MFuyuProcessor.from_pretrained(pretrained)
97
+ self._model = MFuyuForCausalLM.from_pretrained(pretrained, device_map=self.device_map, attn_implementation=attn_implementation, torch_dtype=dtype)
98
+ else:
99
+ self._processor = MLlavaProcessor.from_pretrained(pretrained)
100
+ self._model = LlavaForConditionalGeneration.from_pretrained(pretrained, device_map=self.device_map, attn_implementation=attn_implementation, torch_dtype=dtype)
101
+
102
+ else:
103
+ self._processor = AutoProcessor.from_pretrained(pretrained)
104
+ self._model = AutoModelForVision2Seq.from_pretrained(pretrained, device_map=self.device_map, torch_dtype=dtype)
105
+ eval_logger.info(f"Using {type(self._model)} to instantiate the Mantis model.")
106
+
107
+ self._tokenizer = self._processor.tokenizer
108
+
109
+ self._config = self._model.config
110
+ self.model.eval()
111
+ self.model.tie_weights()
112
+ self.truncation = truncation
113
+ self.batch_size_per_gpu = int(batch_size)
114
+ self.use_cache = use_cache
115
+ self.truncate_context = truncate_context
116
+
117
+ if accelerator.num_processes > 1:
118
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
119
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
120
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
121
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
122
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
123
+ kwargs = {
124
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
125
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
126
+ }
127
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
128
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
129
+
130
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
131
+ self._model = accelerator.prepare(self.model)
132
+ else:
133
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
134
+ self.accelerator = accelerator
135
+ if self.accelerator.is_local_main_process:
136
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
137
+ self._rank = self.accelerator.local_process_index
138
+ self._world_size = self.accelerator.num_processes
139
+ elif accelerator.num_processes == 1 and device_map == "auto":
140
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
141
+ self._rank = 0
142
+ self._world_size = 1
143
+ else:
144
+ eval_logger.info(f"Using single device: {self._device}")
145
+ self.model.to(self._device)
146
+ self._rank = 0
147
+ self._world_size = 1
148
+
149
+ @property
150
+ def config(self):
151
+ # return the associated transformers.AutoConfig for the given pretrained model.
152
+ return self._config
153
+
154
+ @property
155
+ def tokenizer(self):
156
+ return self._tokenizer
157
+
158
+ @property
159
+ def model(self):
160
+ # returns the model, unwrapping it if using Accelerate
161
+ if hasattr(self, "accelerator"):
162
+ return self.accelerator.unwrap_model(self._model)
163
+ else:
164
+ return self._model
165
+
166
+ @property
167
+ def eot_token_id(self):
168
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
169
+ return self.tokenizer.eos_token_id
170
+
171
+ @property
172
+ def max_length(self):
173
+ return self._max_length
174
+
175
+ def pad_sequence(self, input_ids, batch_first, padding_value):
176
+ if self.tokenizer.padding_side == "left":
177
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
178
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
179
+ if self.tokenizer.padding_side == "left":
180
+ input_ids = torch.flip(input_ids, [1])
181
+ return input_ids
182
+
183
+ @property
184
+ def batch_size(self):
185
+ return self.batch_size_per_gpu
186
+
187
+ @property
188
+ def device(self):
189
+ return self._device
190
+
191
+ @property
192
+ def rank(self):
193
+ return self._rank
194
+
195
+ @property
196
+ def world_size(self):
197
+ return self._world_size
198
+
199
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
200
+ """ """
201
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
202
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
203
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
204
+ if left_truncate_len:
205
+ encoding = encoding[-left_truncate_len:]
206
+ return encoding
207
+
208
+ def tok_decode(self, tokens):
209
+ try:
210
+ return self.tokenizer.decode(tokens)
211
+ except:
212
+ return self.tokenizer.decode([tokens])
213
+
214
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
215
+ raise NotImplementedError
216
+
217
+ def flatten(self, input):
218
+ new_list = []
219
+ for i in input:
220
+ for j in i:
221
+ new_list.append(j)
222
+ return new_list
223
+
224
+ def generate_until(self, requests: List[Instance]) -> List[str]:
225
+ res = []
226
+
227
+ def _collate(x):
228
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
229
+ # - time estimates will always be over not underestimates, which is more useful for planning
230
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
231
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
232
+ # automatic adaptive batches much much easier to implement
233
+ # - any OOMs will happen right away rather than near the end
234
+ toks = self.tok_encode(x[0])
235
+ return -len(toks), x[0]
236
+
237
+ # we group requests by their generation_kwargs,
238
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
239
+ # in the same batch.
240
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
241
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
242
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
243
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
244
+ for chunk in chunks:
245
+ contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
246
+ visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
247
+
248
+ # we assume all gen kwargs in the batch are the same
249
+ # this is safe to assume because the `grouper` object ensures it.
250
+ gen_kwargs = all_gen_kwargs[0]
251
+
252
+ until = gen_kwargs.pop("until", None)
253
+ image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio", None)
254
+
255
+ if "max_new_tokens" not in gen_kwargs:
256
+ gen_kwargs["max_new_tokens"] = 1024
257
+ if "temperature" not in gen_kwargs:
258
+ gen_kwargs["temperature"] = 0
259
+
260
+ # prompts_input = contexts[0]
261
+
262
+ prompts = []
263
+ for visual, context in zip(visuals, contexts):
264
+ if self._is_idefics:
265
+ # Follow the idefics implementation:
266
+ content = []
267
+ if DEFAULT_IMAGE_TOKEN not in context:
268
+ for _ in visual:
269
+ content.append({"type": "image"})
270
+ content.append({"type": "text", "text": context})
271
+ message = [{"role": "user", "content": content}]
272
+ prompt = self._processor.apply_chat_template(message, add_generation_prompt=True)
273
+ prompts.append(prompt)
274
+ else:
275
+ # We follow the Mantis code base: https://github.com/TIGER-AI-Lab/Mantis/blob/main/mantis/models/mllava/utils.py#L33 to make sure they are consistent
276
+ # Users don't need to define chat template as it is done here
277
+ if "llama-3" in self._model.language_model.name_or_path.lower():
278
+ conv = conv_templates["llama_3"]
279
+ terminators = [self._processor.tokenizer.eos_token_id, self._processor.tokenizer.convert_tokens_to_ids("<|eot_id|>")]
280
+ else:
281
+ conv = default_conv
282
+ terminators = None
283
+
284
+ gen_kwargs["eos_token_id"] = terminators
285
+
286
+ conv = conv.copy()
287
+ conv.append_message(conv.roles[0], context)
288
+ conv.append_message(conv.roles[1], "")
289
+ prompt = conv.get_prompt()
290
+ prompts.append(prompt)
291
+ inputs = self._processor(images=visuals, text=prompts, return_tensors="pt", truncation=True)
292
+ if "image_patches" in inputs.keys():
293
+ inputs["image_patches"] = inputs["image_patches"][0] # FIXME: Fuyu model would return a list instead of a pytorch tensor. This weird behavior needs fixing.
294
+ inputs = {k: v.to(self.device) for k, v in inputs.items()}
295
+
296
+ output_ids = self.model.generate(**inputs, **gen_kwargs)
297
+ for output_id, input_id in zip(output_ids, inputs["input_ids"]):
298
+ generated_id = output_id[len(input_id) :]
299
+ generated_text = self.tokenizer.decode(generated_id, skip_special_tokens=True)
300
+
301
+ res.append(generated_text)
302
+
303
+ # self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
304
+ pbar.update(1)
305
+ # reorder this group of results back to original unsorted form
306
+ res = re_ords.get_original(res)
307
+
308
+ pbar.close()
309
+ return res
310
+
311
+ def generate_until_multi_round(self, requests) -> List[str]:
312
+ raise NotImplementedError("TODO: Implement multi-round generation for Mantis")
LaViDa/eval/lmms_eval/models/minimonkey.py ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ from accelerate import Accelerator, DistributedType
6
+ from accelerate.state import AcceleratorState
7
+ from tqdm import tqdm
8
+ from transformers import AutoModel, AutoTokenizer
9
+
10
+ from lmms_eval import utils
11
+ from lmms_eval.api.instance import Instance
12
+ from lmms_eval.api.model import lmms
13
+ from lmms_eval.api.registry import register_model
14
+
15
+ warnings.filterwarnings("ignore")
16
+
17
+ from loguru import logger as eval_logger
18
+
19
+
20
+ @register_model("minimonkey")
21
+ class MiniMonkey(lmms):
22
+ """
23
+ MiniMonkey Model
24
+ """
25
+
26
+ def __init__(
27
+ self,
28
+ pretrained: str = "mx262/MiniMonkey",
29
+ device: Optional[str] = "cuda",
30
+ dtype: Optional[Union[str, torch.dtype]] = torch.bfloat16,
31
+ batch_size: Optional[Union[int, str]] = 1,
32
+ trust_remote_code: Optional[bool] = True,
33
+ **kwargs,
34
+ ) -> None:
35
+ super().__init__()
36
+ # Do not use kwargs for now
37
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
38
+
39
+ accelerator = Accelerator()
40
+ if accelerator.num_processes > 1:
41
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
42
+ else:
43
+ self._device = device
44
+ self.dtype = dtype
45
+ self._model = AutoModel.from_pretrained(pretrained, trust_remote_code=trust_remote_code, torch_dtype=dtype, device_map=self._device)
46
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code)
47
+ self._config = self._model.config
48
+ self.model.eval()
49
+ self.model.tie_weights()
50
+ self.batch_size_per_gpu = int(batch_size)
51
+ if accelerator.num_processes > 1:
52
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
53
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
54
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
55
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
56
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
57
+ kwargs = {
58
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
59
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
60
+ }
61
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
62
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
63
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
64
+ self._model = accelerator.prepare(self.model)
65
+ else:
66
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
67
+ self.accelerator = accelerator
68
+ if self.accelerator.is_local_main_process:
69
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
70
+ self._rank = self.accelerator.local_process_index
71
+ self._world_size = self.accelerator.num_processes
72
+ else:
73
+ # self.model.to(self._device)
74
+ self._rank = 0
75
+ self._world_size = 1
76
+
77
+ @property
78
+ def config(self):
79
+ # return the associated transformers.AutoConfig for the given pretrained model.
80
+ return self._config
81
+
82
+ @property
83
+ def tokenizer(self):
84
+ return self._tokenizer
85
+
86
+ @property
87
+ def model(self):
88
+ # returns the model, unwrapping it if using Accelerate
89
+ if hasattr(self, "accelerator"):
90
+ return self.accelerator.unwrap_model(self._model)
91
+ else:
92
+ return self._model
93
+
94
+ @property
95
+ def eot_token_id(self):
96
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
97
+ return self.tokenizer.eos_token_id
98
+
99
+ @property
100
+ def max_length(self):
101
+ return self._max_length
102
+
103
+ @property
104
+ def batch_size(self):
105
+ return self.batch_size_per_gpu
106
+
107
+ @property
108
+ def device(self):
109
+ return self._device
110
+
111
+ @property
112
+ def rank(self):
113
+ return self._rank
114
+
115
+ @property
116
+ def world_size(self):
117
+ return self._world_size
118
+
119
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
120
+ """ """
121
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
122
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
123
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
124
+ if left_truncate_len:
125
+ encoding = encoding[-left_truncate_len:]
126
+ return encoding
127
+
128
+ def tok_decode(self, tokens):
129
+ return self.tokenizer.decode(tokens)
130
+
131
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
132
+ # TODO
133
+ assert False, "We have not implemented this function for MiniMonkey yet"
134
+
135
+ def flatten(self, input):
136
+ new_list = []
137
+ for i in input:
138
+ for j in i:
139
+ new_list.append(j)
140
+ return new_list
141
+
142
+ def generate_until(self, requests: List[Instance]) -> List[str]:
143
+ res = []
144
+
145
+ def _collate(x):
146
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
147
+ # - time estimates will always be over not underestimates, which is more useful for planning
148
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
149
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
150
+ # automatic adaptive batches much much easier to implement
151
+ # - any OOMs will happen right away rather than near the end
152
+ toks = self.tok_encode(x[0])
153
+ return -len(toks), x[0]
154
+
155
+ # we group requests by their generation_kwargs,
156
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
157
+ # in the same batch.
158
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
159
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
160
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
161
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
162
+ for chunk in chunks:
163
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
164
+ task = task[0]
165
+ split = split[0]
166
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
167
+ visuals = self.flatten(visuals)
168
+ # we assume all gen kwargs in the batch are the same
169
+ # this is safe to assume because the `grouper` object ensures it.
170
+ gen_kwargs = all_gen_kwargs[0]
171
+
172
+ # Set default values for until and max_new_tokens
173
+ until = [self.tok_decode(self.eot_token_id)]
174
+
175
+ # Update values from gen_kwargs if present
176
+ if "until" in gen_kwargs:
177
+ until = gen_kwargs.pop("until")
178
+ if isinstance(until, str):
179
+ until = [until]
180
+ elif not isinstance(until, list):
181
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
182
+ assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
183
+ assert len(visuals) == 1, "MiniMonkey interface does not support bn_image > 1 for now"
184
+ context = contexts[0]
185
+ if "<image>" in context:
186
+ context = context.replace("<image>", "")
187
+
188
+ if "max_new_tokens" not in gen_kwargs:
189
+ gen_kwargs["max_new_tokens"] = 1024
190
+ if "temperature" not in gen_kwargs:
191
+ gen_kwargs["temperature"] = 0
192
+ if "top_p" not in gen_kwargs:
193
+ gen_kwargs["top_p"] = None
194
+ if "num_beams" not in gen_kwargs:
195
+ gen_kwargs["num_beams"] = 1
196
+
197
+ image, prompt = visuals[0], context
198
+ try:
199
+ pixel_values, target_aspect_ratio = load_image(image, min_num=4, max_num=12)
200
+ pixel_values2 = load_image2(image, min_num=3, max_num=7, target_aspect_ratio=target_aspect_ratio)
201
+ pixel_values = torch.cat([pixel_values2[:-1], pixel_values[:-1], pixel_values2[-1:]], 0).to(self._device).to(self.dtype)
202
+
203
+ response, history = self.model.chat(self.tokenizer, pixel_values, target_aspect_ratio, prompt, gen_kwargs, history=None, return_history=True)
204
+
205
+ context = [{"role": "user", "content": prompt}, {"role": "assistant", "content": response}]
206
+ except Exception as e:
207
+ eval_logger.error(f"Error {e} in generating")
208
+ cont = ""
209
+ res.append(response)
210
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response)
211
+ pbar.update(1)
212
+ # reorder this group of results back to original unsorted form
213
+ res = re_ords.get_original(res)
214
+
215
+ pbar.close()
216
+ return res
217
+
218
+ def generate_until_multi_round(self, requests) -> List[str]:
219
+ raise NotImplementedError("TODO: Implement multi-round generation")
220
+
221
+
222
+ import numpy as np
223
+ import torchvision.transforms as T
224
+ from PIL import Image
225
+ from torchvision.transforms.functional import InterpolationMode
226
+
227
+ IMAGENET_MEAN = (0.485, 0.456, 0.406)
228
+ IMAGENET_STD = (0.229, 0.224, 0.225)
229
+
230
+
231
+ def build_transform(input_size):
232
+ MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
233
+ transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
234
+ return transform
235
+
236
+
237
+ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
238
+ best_ratio_diff = float("inf")
239
+ best_ratio = (1, 1)
240
+ area = width * height
241
+ for ratio in target_ratios:
242
+ target_aspect_ratio = ratio[0] / ratio[1]
243
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
244
+ if ratio_diff < best_ratio_diff:
245
+ best_ratio_diff = ratio_diff
246
+ best_ratio = ratio
247
+ elif ratio_diff == best_ratio_diff:
248
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
249
+ best_ratio = ratio
250
+ return best_ratio
251
+
252
+
253
+ def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
254
+ orig_width, orig_height = image.size
255
+ aspect_ratio = orig_width / orig_height
256
+
257
+ # calculate the existing image aspect ratio
258
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
259
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
260
+
261
+ # find the closest aspect ratio to the target
262
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
263
+
264
+ # calculate the target width and height
265
+ target_width = image_size * target_aspect_ratio[0]
266
+ target_height = image_size * target_aspect_ratio[1]
267
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
268
+
269
+ # resize the image
270
+ resized_img = image.resize((target_width, target_height))
271
+ processed_images = []
272
+ for i in range(blocks):
273
+ box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
274
+ # split the image
275
+ split_img = resized_img.crop(box)
276
+ processed_images.append(split_img)
277
+ assert len(processed_images) == blocks
278
+ if use_thumbnail and len(processed_images) != 1:
279
+ thumbnail_img = image.resize((image_size, image_size))
280
+ processed_images.append(thumbnail_img)
281
+ return processed_images, target_aspect_ratio
282
+
283
+
284
+ def dynamic_preprocess2(image, min_num=1, max_num=12, prior_aspect_ratio=None, image_size=448, use_thumbnail=False):
285
+ orig_width, orig_height = image.size
286
+ aspect_ratio = orig_width / orig_height
287
+
288
+ # calculate the existing image aspect ratio
289
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
290
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
291
+ new_target_ratios = []
292
+ for i in target_ratios:
293
+ if prior_aspect_ratio[0] % i[0] or prior_aspect_ratio[1] % i[1]:
294
+ new_target_ratios.append(i)
295
+ else:
296
+ continue
297
+ # find the closest aspect ratio to the target
298
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, new_target_ratios, orig_width, orig_height, image_size)
299
+ # calculate the target width and height
300
+ target_width = image_size * target_aspect_ratio[0]
301
+ target_height = image_size * target_aspect_ratio[1]
302
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
303
+
304
+ # resize the image
305
+ resized_img = image.resize((target_width, target_height))
306
+ processed_images = []
307
+ for i in range(blocks):
308
+ box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
309
+ # split the image
310
+ split_img = resized_img.crop(box)
311
+ processed_images.append(split_img)
312
+ assert len(processed_images) == blocks
313
+ if use_thumbnail and len(processed_images) != 1:
314
+ thumbnail_img = image.resize((image_size, image_size))
315
+ processed_images.append(thumbnail_img)
316
+ return processed_images
317
+
318
+
319
+ def load_image(image, input_size=448, min_num=1, max_num=12):
320
+ image = image.convert("RGB")
321
+ transform = build_transform(input_size=input_size)
322
+ images, target_aspect_ratio = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, min_num=min_num, max_num=max_num)
323
+ pixel_values = [transform(image) for image in images]
324
+ pixel_values = torch.stack(pixel_values)
325
+ return pixel_values, target_aspect_ratio
326
+
327
+
328
+ def load_image2(image, input_size=448, min_num=1, max_num=12, target_aspect_ratio=None):
329
+ image = image.convert("RGB")
330
+ transform = build_transform(input_size=input_size)
331
+ images = dynamic_preprocess2(image, image_size=input_size, use_thumbnail=True, min_num=min_num, max_num=max_num, prior_aspect_ratio=target_aspect_ratio)
332
+ pixel_values = [transform(image) for image in images]
333
+ pixel_values = torch.stack(pixel_values)
334
+ return pixel_values
LaViDa/eval/lmms_eval/models/model_utils/__init__.py ADDED
File without changes
LaViDa/eval/lmms_eval/models/model_utils/audio_processing.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ from librosa import resample
3
+
4
+
5
+ def downsample_audio(audio_array: np.ndarray, original_sr: int, target_sr: int) -> np.ndarray:
6
+ audio_resample_array = resample(audio_array, orig_sr=original_sr, target_sr=target_sr)
7
+ return audio_resample_array
LaViDa/eval/lmms_eval/models/model_utils/load_video.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ from io import BytesIO
3
+ from random import sample
4
+ from typing import Optional, Tuple, Union
5
+
6
+ import av
7
+ import numpy as np
8
+ from av.codec.context import CodecContext
9
+ from decord import VideoReader, cpu
10
+ from PIL import Image
11
+
12
+
13
+ def load_video_decord(video_path, max_frames_num):
14
+ if type(video_path) == str:
15
+ vr = VideoReader(video_path, ctx=cpu(0))
16
+ else:
17
+ vr = VideoReader(video_path[0], ctx=cpu(0))
18
+ total_frame_num = len(vr)
19
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
20
+ frame_idx = uniform_sampled_frames.tolist()
21
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
22
+ return spare_frames # (frames, height, width, channels)
23
+
24
+
25
+ # This one is faster
26
+ def record_video_length_stream(container, indices):
27
+ frames = []
28
+ start_index = indices[0]
29
+ end_index = indices[-1]
30
+ for i, frame in enumerate(container.decode(video=0)):
31
+ if i > end_index:
32
+ break
33
+ if i >= start_index and i in indices:
34
+ frames.append(frame)
35
+ return frames
36
+
37
+
38
+ # This one works for all types of video
39
+ def record_video_length_packet(container):
40
+ frames = []
41
+ # https://github.com/PyAV-Org/PyAV/issues/1269
42
+ # https://www.cnblogs.com/beyond-tester/p/17641872.html
43
+ # context = CodecContext.create("libvpx-vp9", "r")
44
+ for packet in container.demux(video=0):
45
+ for frame in packet.decode():
46
+ frames.append(frame)
47
+ return frames
48
+
49
+
50
+ def load_video_stream(container, num_frm: int = 8, fps: float = None, force_include_last_frame=False):
51
+ # container = av.open(video_path)
52
+ total_frames = container.streams.video[0].frames
53
+ frame_rate = container.streams.video[0].average_rate
54
+ if fps is not None:
55
+ video_length = total_frames / frame_rate
56
+ num_frm = min(num_frm, int(video_length * fps))
57
+ sampled_frm = min(total_frames, num_frm)
58
+ indices = np.linspace(0, total_frames - 1, sampled_frm, dtype=int)
59
+ if force_include_last_frame:
60
+ last_frame = total_frames - 1
61
+ if last_frame not in indices:
62
+ indices = np.linspace(0, total_frames - 2, sampled_frm - 1, dtype=int)
63
+ indices = np.append(indices, last_frame)
64
+
65
+ return record_video_length_stream(container, indices)
66
+
67
+
68
+ def load_video_packet(container, num_frm: int = 8, fps: float = None):
69
+ frames = record_video_length_packet(container)
70
+ total_frames = len(frames)
71
+ frame_rate = container.streams.video[0].average_rate
72
+ if fps is not None:
73
+ video_length = total_frames / frame_rate
74
+ num_frm = min(num_frm, int(video_length * fps))
75
+ sampled_frm = min(total_frames, num_frm)
76
+ indices = np.linspace(0, total_frames - 1, sampled_frm, dtype=int)
77
+
78
+ # Append the last frame index if not already included
79
+ if total_frames - 1 not in indices:
80
+ indices = np.append(indices, total_frames - 1)
81
+
82
+ return [frames[i] for i in indices]
83
+
84
+
85
+ def read_video_pyav(video_path: str, *, num_frm: int = 8, fps: float = None, format="rgb24", force_include_last_frame=False) -> np.ndarray:
86
+ """
87
+ Read video using the PyAV library.
88
+
89
+ Args:
90
+ video_path (str): The path to the video file.
91
+ num_frm (int, optional): The maximum number of frames to extract. Defaults to 8.
92
+ fps (float, optional): The frames per second for extraction. If `None`, the maximum number of frames will be extracted. Defaults to None.
93
+ format (str, optional): The format of the extracted frames. Defaults to "rgb24".
94
+
95
+ Returns:
96
+ np.ndarray: A numpy array containing the extracted frames in RGB format.
97
+ """
98
+
99
+ container = av.open(video_path)
100
+
101
+ if "webm" not in video_path and "mkv" not in video_path:
102
+ # For mp4, we try loading with stream first
103
+ try:
104
+ frames = load_video_stream(container, num_frm, fps, force_include_last_frame=force_include_last_frame)
105
+ except:
106
+ frames = record_video_length_packet(container)
107
+ else:
108
+ frames = record_video_length_packet(container)
109
+
110
+ return np.stack([x.to_ndarray(format=format) for x in frames])
111
+
112
+
113
+ def read_video_pyav_pil(video_path: str, *, num_frm: int = 8, fps: float = None, format="rgb24", max_image_size: Optional[Union[Tuple[int, int], int]] = None, resize_strategy: str = "resize", force_include_last_frame=False):
114
+ frames = read_video_pyav(video_path, num_frm=num_frm, fps=fps, format=format, force_include_last_frame=force_include_last_frame)
115
+ pil_frames = []
116
+ for frame in frames:
117
+ img = Image.fromarray(frame)
118
+ if max_image_size:
119
+ if resize_strategy == "resize":
120
+ if isinstance(max_image_size, int):
121
+ max_image_size = (max_image_size, max_image_size)
122
+ img = img.resize(max_image_size)
123
+ elif resize_strategy == "thumbnail":
124
+ img.thumbnail(max_image_size)
125
+ else:
126
+ raise ValueError(f"Unknown resize strategy: {resize_strategy}")
127
+ pil_frames.append(img)
128
+ return pil_frames
129
+ # return [Image.fromarray(frame) for frame in frames]
130
+
131
+
132
+ def read_video_pyav_base64(video_path: str, *, num_frm: int = 8, fps: Optional[float] = None, format="rgb24", img_format="PNG", max_image_size: Optional[Union[Tuple[int, int], int]] = None, resize_strategy: str = "resize"):
133
+ frames = read_video_pyav(video_path, num_frm=num_frm, fps=fps, format=format)
134
+ base64_frames = []
135
+ for frame in frames:
136
+ img = Image.fromarray(frame)
137
+ if max_image_size:
138
+ if resize_strategy == "resize":
139
+ if isinstance(max_image_size, int):
140
+ max_image_size = (max_image_size, max_image_size)
141
+ img = img.resize(max_image_size)
142
+ elif resize_strategy == "thumbnail":
143
+ img.thumbnail(max_image_size)
144
+ else:
145
+ raise ValueError(f"Unknown resize strategy: {resize_strategy}")
146
+ output_buffer = BytesIO()
147
+ img.save(output_buffer, format=img_format)
148
+ byte_data = output_buffer.getvalue()
149
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
150
+ base64_frames.append(base64_str)
151
+ return base64_frames
LaViDa/eval/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py ADDED
@@ -0,0 +1,369 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ """Generation support."""
7
+ import warnings
8
+
9
+ warnings.simplefilter("ignore", category=DeprecationWarning)
10
+ warnings.filterwarnings("ignore")
11
+
12
+ from typing import Iterable, List, Tuple, Union
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from loguru import logger
18
+ from transformers import PreTrainedTokenizer
19
+ from transformers.generation import LogitsProcessor
20
+
21
+ # Types.
22
+ HistoryType = List[Tuple[str, str]]
23
+ TokensType = List[int]
24
+ BatchTokensType = List[List[int]]
25
+
26
+
27
+ def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType:
28
+ for tokens in batch:
29
+ context_length = len(tokens)
30
+ if context_length < seq_length:
31
+ tokens.extend([pad_id] * (seq_length - context_length))
32
+ return batch
33
+
34
+
35
+ def get_ltor_masks_and_position_ids(
36
+ data,
37
+ eod_token,
38
+ reset_position_ids,
39
+ reset_attention_mask,
40
+ eod_mask_loss,
41
+ ):
42
+ """Build masks and position id for left to right model."""
43
+
44
+ # Extract batch size and sequence length.
45
+ micro_batch_size, seq_length = data.size()
46
+
47
+ # Attention mask (lower triangular).
48
+ if reset_attention_mask:
49
+ att_mask_batch = micro_batch_size
50
+ else:
51
+ att_mask_batch = 1
52
+ attention_mask = torch.tril(torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)).view(att_mask_batch, 1, seq_length, seq_length)
53
+
54
+ # Loss mask.
55
+ loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device)
56
+ if eod_mask_loss:
57
+ loss_mask[data == eod_token] = 0.0
58
+
59
+ # Position ids.
60
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
61
+ position_ids = position_ids.unsqueeze(0).expand_as(data)
62
+ # We need to clone as the ids will be modifed based on batch index.
63
+ if reset_position_ids:
64
+ position_ids = position_ids.clone()
65
+
66
+ if reset_position_ids or reset_attention_mask:
67
+ # Loop through the batches:
68
+ for b in range(micro_batch_size):
69
+ # Find indecies where EOD token is.
70
+ eod_index = position_ids[b, data[b] == eod_token]
71
+ # Detach indecies from positions if going to modify positions.
72
+ if reset_position_ids:
73
+ eod_index = eod_index.clone()
74
+
75
+ # Loop through EOD indecies:
76
+ prev_index = 0
77
+ for j in range(eod_index.size()[0]):
78
+ i = eod_index[j]
79
+ # Mask attention loss.
80
+ if reset_attention_mask:
81
+ attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0
82
+ # Reset positions.
83
+ if reset_position_ids:
84
+ position_ids[b, (i + 1) :] -= i + 1 - prev_index
85
+ prev_index = i + 1
86
+
87
+ # Convert attention mask to binary:
88
+ attention_mask = attention_mask < 0.5
89
+
90
+ return attention_mask, loss_mask, position_ids
91
+
92
+
93
+ def get_batch(context_tokens: torch.LongTensor, eod_id: int):
94
+ """Generate batch from context tokens."""
95
+ # Move to GPU.
96
+ tokens = context_tokens.contiguous().to(context_tokens.device)
97
+ # Get the attention mask and postition ids.
98
+ attention_mask, _, position_ids = get_ltor_masks_and_position_ids(
99
+ tokens,
100
+ eod_id,
101
+ reset_position_ids=False,
102
+ reset_attention_mask=False,
103
+ eod_mask_loss=False,
104
+ )
105
+ return tokens, attention_mask, position_ids
106
+
107
+
108
+ def get_stop_words_ids(chat_format, tokenizer):
109
+ if chat_format == "raw":
110
+ stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]]
111
+ elif chat_format == "chatml":
112
+ stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]]
113
+ else:
114
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
115
+ return stop_words_ids
116
+
117
+
118
+ def make_context(
119
+ tokenizer: PreTrainedTokenizer,
120
+ query: str,
121
+ history: List[Tuple[str, str]] = None,
122
+ system: str = "",
123
+ max_window_size: int = 6144,
124
+ chat_format: str = "chatml",
125
+ ):
126
+ if history is None:
127
+ history = []
128
+
129
+ if chat_format == "chatml":
130
+ im_start, im_end = "<|im_start|>", "<|im_end|>"
131
+ im_start_tokens = [tokenizer.im_start_id]
132
+ im_end_tokens = [tokenizer.im_end_id]
133
+ nl_tokens = tokenizer.encode("\n")
134
+
135
+ def _tokenize_str(role, content):
136
+ return f"{role}\n{content}", tokenizer.encode(role, allowed_special=set(tokenizer.IMAGE_ST)) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST))
137
+
138
+ system_text, system_tokens_part = _tokenize_str("system", system)
139
+ system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
140
+
141
+ raw_text = ""
142
+ context_tokens = []
143
+
144
+ for turn_query, turn_response in reversed(history):
145
+ query_text, query_tokens_part = _tokenize_str("user", turn_query)
146
+ query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
147
+ if turn_response is not None:
148
+ response_text, response_tokens_part = _tokenize_str("assistant", turn_response)
149
+ response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
150
+
151
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
152
+ prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
153
+ else:
154
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens
155
+ prev_chat = f"\n{im_start}{query_text}{im_end}\n"
156
+
157
+ current_context_size = len(system_tokens) + len(next_context_tokens) + len(context_tokens)
158
+ if current_context_size < max_window_size:
159
+ context_tokens = next_context_tokens + context_tokens
160
+ raw_text = prev_chat + raw_text
161
+ else:
162
+ break
163
+
164
+ context_tokens = system_tokens + context_tokens
165
+ raw_text = f"{im_start}{system_text}{im_end}" + raw_text
166
+ context_tokens += nl_tokens + im_start_tokens + _tokenize_str("user", query)[1] + im_end_tokens + nl_tokens + im_start_tokens + tokenizer.encode("assistant") + nl_tokens
167
+ raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
168
+
169
+ elif chat_format == "raw":
170
+ raw_text = query
171
+ context_tokens = tokenizer.encode(raw_text)
172
+ else:
173
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
174
+
175
+ return raw_text, context_tokens
176
+
177
+
178
+ def _decode_default(
179
+ tokens: List[int],
180
+ *,
181
+ stop_words: List[str],
182
+ eod_words: List[str],
183
+ tokenizer: PreTrainedTokenizer,
184
+ raw_text_len: int,
185
+ verbose: bool = False,
186
+ return_end_reason: bool = False,
187
+ errors: str = "replace",
188
+ ):
189
+ trim_decode_tokens = tokenizer.decode(tokens, errors=errors)[raw_text_len:]
190
+ if verbose:
191
+ print("\nRaw Generate: ", trim_decode_tokens)
192
+
193
+ end_reason = f"Gen length {len(tokens)}"
194
+ for stop_word in stop_words:
195
+ trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
196
+ for eod_word in eod_words:
197
+ if eod_word in trim_decode_tokens:
198
+ end_reason = f"Gen {eod_word!r}"
199
+ trim_decode_tokens = trim_decode_tokens.split(eod_word)[0]
200
+ trim_decode_tokens = trim_decode_tokens.strip()
201
+ if verbose:
202
+ print("\nEnd Reason:", end_reason)
203
+ print("\nGenerate: ", trim_decode_tokens)
204
+
205
+ if return_end_reason:
206
+ return trim_decode_tokens, end_reason
207
+ else:
208
+ return trim_decode_tokens
209
+
210
+
211
+ def _decode_chatml(
212
+ tokens: List[int], *, stop_words: List[str], eod_token_ids: List[int], tokenizer: PreTrainedTokenizer, raw_text_len: int, context_length: int, verbose: bool = False, return_end_reason: bool = False, errors: str = "replace"
213
+ ):
214
+ end_reason = f"Gen length {len(tokens)}"
215
+ eod_token_idx = context_length
216
+ for eod_token_idx in range(context_length, len(tokens)):
217
+ if tokens[eod_token_idx] in eod_token_ids:
218
+ end_reason = f"Gen {tokenizer.decode([tokens[eod_token_idx]])!r}"
219
+ break
220
+
221
+ trim_decode_tokens = tokenizer.decode(tokens[:eod_token_idx], errors=errors)[raw_text_len:]
222
+ if verbose:
223
+ print("\nRaw Generate w/o EOD:", tokenizer.decode(tokens, errors=errors)[raw_text_len:])
224
+ print("\nRaw Generate:", trim_decode_tokens)
225
+ print("\nEnd Reason:", end_reason)
226
+ for stop_word in stop_words:
227
+ trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
228
+ trim_decode_tokens = trim_decode_tokens.strip()
229
+ if verbose:
230
+ print("\nGenerate:", trim_decode_tokens)
231
+
232
+ if return_end_reason:
233
+ return trim_decode_tokens, end_reason
234
+ else:
235
+ return trim_decode_tokens
236
+
237
+
238
+ def decode_tokens(
239
+ tokens: Union[torch.LongTensor, TokensType],
240
+ tokenizer: PreTrainedTokenizer,
241
+ raw_text_len: int,
242
+ context_length: int,
243
+ chat_format: str,
244
+ verbose: bool = False,
245
+ return_end_reason: bool = False,
246
+ errors: str = "replace",
247
+ ) -> str:
248
+ if torch.is_tensor(tokens):
249
+ tokens = tokens.cpu().numpy().tolist()
250
+
251
+ if chat_format == "chatml":
252
+ return _decode_chatml(
253
+ tokens,
254
+ stop_words=[],
255
+ eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id],
256
+ tokenizer=tokenizer,
257
+ raw_text_len=raw_text_len,
258
+ context_length=context_length,
259
+ verbose=verbose,
260
+ return_end_reason=return_end_reason,
261
+ errors=errors,
262
+ )
263
+ elif chat_format == "raw":
264
+ return _decode_default(
265
+ tokens,
266
+ stop_words=["<|endoftext|>"],
267
+ eod_words=["<|endoftext|>"],
268
+ tokenizer=tokenizer,
269
+ raw_text_len=raw_text_len,
270
+ verbose=verbose,
271
+ return_end_reason=return_end_reason,
272
+ errors=errors,
273
+ )
274
+ else:
275
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
276
+
277
+
278
+ class StopWordsLogitsProcessor(LogitsProcessor):
279
+ """
280
+ :class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration.
281
+ Args:
282
+ stop_words_ids (:obj:`List[List[int]]`):
283
+ List of list of token ids of stop ids. In order to get the tokens of the words
284
+ that should not appear in the generated text, use :obj:`tokenizer(bad_word,
285
+ add_prefix_space=True).input_ids`.
286
+ eos_token_id (:obj:`int`):
287
+ The id of the `end-of-sequence` token.
288
+ """
289
+
290
+ def __init__(self, stop_words_ids: Iterable[Iterable[int]], eos_token_id: int):
291
+ if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0:
292
+ raise ValueError(f"`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}.")
293
+ if any(not isinstance(bad_word_ids, list) for bad_word_ids in stop_words_ids):
294
+ raise ValueError(f"`stop_words_ids` has to be a list of lists, but is {stop_words_ids}.")
295
+ if any(any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in stop_word_ids) for stop_word_ids in stop_words_ids):
296
+ raise ValueError(f"Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}.")
297
+
298
+ self.stop_words_ids = list(filter(lambda bad_token_seq: bad_token_seq != [eos_token_id], stop_words_ids))
299
+ self.eos_token_id = eos_token_id
300
+ for stop_token_seq in self.stop_words_ids:
301
+ assert len(stop_token_seq) > 0, "Stop words token sequences {} cannot have an empty list".format(stop_words_ids)
302
+
303
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
304
+ stopped_samples = self._calc_stopped_samples(input_ids)
305
+ for i, should_stop in enumerate(stopped_samples):
306
+ if should_stop:
307
+ scores[i, self.eos_token_id] = float(2**15)
308
+ return scores
309
+
310
+ def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool:
311
+ if len(tokens) == 0:
312
+ # if bad word tokens is just one token always ban it
313
+ return True
314
+ elif len(tokens) > len(prev_tokens):
315
+ # if bad word tokens are longer then prev input_ids they can't be equal
316
+ return False
317
+ elif prev_tokens[-len(tokens) :].tolist() == tokens:
318
+ # if tokens match
319
+ return True
320
+ else:
321
+ return False
322
+
323
+ def _calc_stopped_samples(self, prev_input_ids: Iterable[int]) -> Iterable[int]:
324
+ stopped_samples = []
325
+ for prev_input_ids_slice in prev_input_ids:
326
+ match = False
327
+ for stop_token_seq in self.stop_words_ids:
328
+ if self._tokens_match(prev_input_ids_slice, stop_token_seq):
329
+ # if tokens do not match continue
330
+ match = True
331
+ break
332
+ stopped_samples.append(match)
333
+
334
+ return stopped_samples
335
+
336
+
337
+ def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float("Inf")):
338
+ """This function has been mostly taken from huggingface conversational
339
+ ai code at
340
+ https://medium.com/huggingface/how-to-build-a-state-of-the-art-
341
+ conversational-ai-with-transfer-learning-2d818ac26313"""
342
+
343
+ if top_k > 0:
344
+ # Remove all tokens with a probability less than the
345
+ # last token of the top-k
346
+ indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
347
+ logits[indices_to_remove] = filter_value
348
+
349
+ if top_p > 0.0:
350
+ # Cconvert to 1D
351
+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
352
+ cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
353
+
354
+ # Remove tokens with cumulative probability above the threshold
355
+ sorted_indices_to_remove = cumulative_probs > top_p
356
+ # Shift the indices to the right to keep also the first token
357
+ # above the threshold
358
+ sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
359
+ sorted_indices_to_remove[..., 0] = 0
360
+ for i in range(sorted_indices.size(0)):
361
+ indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]]
362
+ logits[i][indices_to_remove] = filter_value
363
+
364
+ return logits
365
+
366
+
367
+ def switch(val1, val2, boolean):
368
+ boolean = boolean.type_as(val1)
369
+ return (1 - boolean) * val1 + boolean * val2
LaViDa/eval/lmms_eval/models/mplug_owl_video.py ADDED
@@ -0,0 +1,199 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from datetime import timedelta
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
6
+ from accelerate.state import AcceleratorState
7
+ from loguru import logger
8
+ from tqdm import tqdm
9
+ from transformers import AutoTokenizer
10
+
11
+ from lmms_eval.api.instance import Instance
12
+ from lmms_eval.api.model import lmms
13
+ from lmms_eval.api.registry import register_model
14
+ from lmms_eval.models.mplug_owl_video.modeling_mplug_owl import (
15
+ MplugOwlForConditionalGeneration,
16
+ )
17
+ from lmms_eval.models.mplug_owl_video.processing_mplug_owl import (
18
+ MplugOwlImageProcessor,
19
+ MplugOwlProcessor,
20
+ )
21
+
22
+ eval_logger = logger
23
+
24
+
25
+ @register_model("mplug_owl_video")
26
+ class mplug_Owl(lmms):
27
+ def __init__(
28
+ self,
29
+ pretrained: str = "MAGAer13/mplug-owl-llama-7b-video",
30
+ device: Optional[str] = "cuda:0",
31
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
32
+ batch_size: Optional[Union[int, str]] = 1,
33
+ device_map="cuda:0",
34
+ num_frames: Union[str, int] = 4,
35
+ **kwargs,
36
+ ) -> None:
37
+ """
38
+ Install instructions:
39
+ 1. Install lmms-eval
40
+ cd lmms-eval
41
+ pip install -e .;
42
+ 2. Install other packages with restricted versions
43
+ pip install av sentencepiece protobuf==3.20 transformers==4.28.1 einops;
44
+ """
45
+ super().__init__()
46
+
47
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
48
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
49
+ if accelerator.num_processes > 1:
50
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
51
+ self.device_map = f"cuda:{accelerator.local_process_index}"
52
+ elif accelerator.num_processes == 1 and device_map == "auto":
53
+ self._device = torch.device(device)
54
+ self.device_map = device_map
55
+ else:
56
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
57
+ self.device_map = f"cuda:{accelerator.local_process_index}"
58
+
59
+ # import pdb; pdb.set_trace()
60
+ # This is very slow. Their issue, not mine
61
+ # Also, keep transformers in version 4.28.1
62
+ # They put a Config object inside a config object, this is not acceptable
63
+ # for transformers == 4.39.1, object type not serializable
64
+ # Protobuf needs to be in 3.20.x otherwise error
65
+ # ヽ(`Д´)ノ
66
+ self._model = MplugOwlForConditionalGeneration.from_pretrained(
67
+ pretrained,
68
+ torch_dtype=torch.bfloat16,
69
+ )
70
+ self.image_processor = MplugOwlImageProcessor.from_pretrained(pretrained)
71
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
72
+ self.processor = MplugOwlProcessor(self.image_processor, self.tokenizer)
73
+ self.model.eval()
74
+ self.batch_size_per_gpu = batch_size
75
+ self.num_frames = num_frames
76
+
77
+ self.model.to(self.device)
78
+
79
+ if accelerator.num_processes > 1:
80
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
81
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
82
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
83
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
84
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
85
+ kwargs = {
86
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
87
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
88
+ }
89
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
90
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
91
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
92
+ self._model = accelerator.prepare(self.model)
93
+ else:
94
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
95
+ self.accelerator = accelerator
96
+ if self.accelerator.is_local_main_process:
97
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
98
+ self._rank = self.accelerator.local_process_index
99
+ self._world_size = self.accelerator.num_processes
100
+ else:
101
+ eval_logger.info(f"Using single device: {self._device}")
102
+ self.model.to(self._device)
103
+ self._rank = 0
104
+ self._world_size = 1
105
+
106
+ @property
107
+ def config(self):
108
+ # return the associated transformers.AutoConfig for the given pretrained model.
109
+ return self._config
110
+
111
+ @property
112
+ def tokenizer(self):
113
+ return self._tokenizer
114
+
115
+ @property
116
+ def model(self):
117
+ # returns the model, unwrapping it if using Accelerate
118
+ if hasattr(self, "accelerator"):
119
+ return self.accelerator.unwrap_model(self._model)
120
+ else:
121
+ return self._model
122
+
123
+ @property
124
+ def eot_token_id(self):
125
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
126
+ return self.tokenizer.eos_token_id
127
+
128
+ @property
129
+ def max_length(self):
130
+ return self._max_length
131
+
132
+ @property
133
+ def batch_size(self):
134
+ return self.batch_size_per_gpu
135
+
136
+ @property
137
+ def device(self):
138
+ return self._device
139
+
140
+ @property
141
+ def rank(self):
142
+ return self._rank
143
+
144
+ @property
145
+ def world_size(self):
146
+ return self._world_size
147
+
148
+ def flatten(self, input):
149
+ new_list = []
150
+ for i in input:
151
+ for j in i:
152
+ new_list.append(j)
153
+ return new_list
154
+
155
+ def format_prompt(self, question):
156
+ prompts = [f" <|video|> Question : {question} Answer : "]
157
+ return prompts
158
+
159
+ def generate_until(self, requests) -> List[str]:
160
+ res = []
161
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
162
+
163
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
164
+ # encode, pad, and truncate contexts for this batch
165
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
166
+ visuals = self.flatten(visuals)
167
+ inputs = self.processor(text=self.format_prompt(contexts), videos=visuals, num_frames=self.num_frames, return_tensors="pt")
168
+ pixel_values_videos = inputs["video_pixel_values"]
169
+ if pixel_values_videos.shape[2] != self.num_frames:
170
+ empty_frames = torch.zeros((1, pixel_values_videos.shape[1], self.num_frames - pixel_values_videos.shape[2], *pixel_values_videos.shape[3:]), dtype=pixel_values_videos.dtype)
171
+ pixel_values_videos = torch.cat([pixel_values_videos, empty_frames], dim=2)
172
+ inputs["video_pixel_values"] = pixel_values_videos
173
+ inputs = {k: v.bfloat16() if v.dtype == torch.float else v for k, v in inputs.items()}
174
+ inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
175
+
176
+ if "max_new_tokens" in gen_kwargs:
177
+ gen_kwargs["max_length"] = gen_kwargs["max_new_tokens"]
178
+ if "max_new_tokens" not in gen_kwargs:
179
+ gen_kwargs["max_length"] = 128
180
+ if "do_sample" not in gen_kwargs:
181
+ gen_kwargs["do_sample"] = False
182
+ if "top_k" not in gen_kwargs:
183
+ gen_kwargs["top_k"] = 1
184
+
185
+ generate_kwargs = {"do_sample": gen_kwargs["do_sample"], "top_k": gen_kwargs["top_k"], "max_length": gen_kwargs["max_length"]}
186
+
187
+ with torch.no_grad():
188
+ outputs = self.model.generate(**inputs, **generate_kwargs)
189
+ sentence = self.tokenizer.decode(outputs.tolist()[0], skip_special_tokens=True)
190
+ pbar.update(1)
191
+ res.append(sentence)
192
+ pbar.close()
193
+ return res
194
+
195
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
196
+ return super().loglikelihood(requests)
197
+
198
+ def generate_until_multi_round(self, requests) -> List[str]:
199
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/mplug_owl_video/__init__.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from typing import TYPE_CHECKING
15
+
16
+ from transformers.utils import (
17
+ OptionalDependencyNotAvailable,
18
+ _LazyModule,
19
+ is_tokenizers_available,
20
+ is_torch_available,
21
+ )
22
+
23
+ _import_structure = {
24
+ "configuration_mplug_owl": ["MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP", "MplugOwlConfig"],
25
+ "processing_mplug_owl": ["MplugOwlImageProcessor", "MplugOwlProcessor"],
26
+ "tokenization_mplug_owl": ["MplugOwlTokenizer"],
27
+ }
28
+
29
+ try:
30
+ if not is_tokenizers_available():
31
+ raise OptionalDependencyNotAvailable()
32
+ except OptionalDependencyNotAvailable:
33
+ pass
34
+
35
+
36
+ try:
37
+ if not is_torch_available():
38
+ raise OptionalDependencyNotAvailable()
39
+ except OptionalDependencyNotAvailable:
40
+ pass
41
+ else:
42
+ _import_structure["modeling_mplug_owl"] = [
43
+ "MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST",
44
+ "MplugOwlForConditionalGeneration",
45
+ "MplugOwlModel",
46
+ ]
47
+
48
+
49
+ if TYPE_CHECKING:
50
+ from .configuration_mplug_owl import (
51
+ MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP,
52
+ MplugOwlConfig,
53
+ )
54
+ from .tokenization_mplug_owl import MplugOwlTokenizer
55
+
56
+ try:
57
+ if not is_tokenizers_available():
58
+ raise OptionalDependencyNotAvailable()
59
+ except OptionalDependencyNotAvailable:
60
+ pass
61
+
62
+ try:
63
+ if not is_torch_available():
64
+ raise OptionalDependencyNotAvailable()
65
+ except OptionalDependencyNotAvailable:
66
+ pass
67
+ else:
68
+ from .modeling_mplug_owl import (
69
+ MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST,
70
+ MplugOwlForConditionalGeneration,
71
+ MplugOwlModel,
72
+ MplugOwlPreTrainedModel,
73
+ )
74
+
75
+
76
+ else:
77
+ import sys
78
+
79
+ sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
80
+
81
+ from .configuration_mplug_owl import *
82
+ from .modeling_mplug_owl import *
83
+ from .processing_mplug_owl import *
84
+ from .tokenization_mplug_owl import *
LaViDa/eval/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 x-plug and The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ MplugOwl model configuration """
16
+ import copy
17
+ import os
18
+ from typing import Union
19
+
20
+ from loguru import logger
21
+ from transformers.configuration_utils import PretrainedConfig
22
+ from transformers.models.auto import CONFIG_MAPPING
23
+ from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
24
+
25
+ MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
26
+ "MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/config.json",
27
+ # See all MplugOwl models at https://huggingface.co/models?filter=mplug_owl
28
+ }
29
+
30
+
31
+ class MplugOwlVisionConfig(PretrainedConfig):
32
+ r"""
33
+ This is the configuration class to store the configuration of a [`MplugOwlVisionModel`]. It is used to instantiate a
34
+ mPLUG-Owl vision encoder according to the specified arguments, defining the model architecture. Instantiating a
35
+ configuration defaults will yield a similar configuration to that of the mPLUG-Owl
36
+ [x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b) architecture.
37
+
38
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
39
+ documentation from [`PretrainedConfig`] for more information.
40
+
41
+ Args:
42
+ hidden_size (`int`, *optional*, defaults to 768):
43
+ Dimensionality of the encoder layers and the pooler layer.
44
+ intermediate_size (`int`, *optional*, defaults to 3072):
45
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
46
+ num_hidden_layers (`int`, *optional*, defaults to 12):
47
+ Number of hidden layers in the Transformer encoder.
48
+ num_attention_heads (`int`, *optional*, defaults to 12):
49
+ Number of attention heads for each attention layer in the Transformer encoder.
50
+ image_size (`int`, *optional*, defaults to 224):
51
+ The size (resolution) of each image.
52
+ patch_size (`int`, *optional*, defaults to 32):
53
+ The size (resolution) of each patch.
54
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
55
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
56
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
57
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
58
+ The epsilon used by the layer normalization layers.
59
+ attention_dropout (`float`, *optional*, defaults to 0.0):
60
+ The dropout ratio for the attention probabilities.
61
+ initializer_range (`float`, *optional*, defaults to 0.02):
62
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
63
+ initializer_factor (`float`, *optional*, defaults to 1):
64
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
65
+ testing).
66
+
67
+
68
+ ```"""
69
+
70
+ model_type = "mplug_owl_vision_model"
71
+
72
+ def __init__(
73
+ self,
74
+ hidden_size=1024,
75
+ intermediate_size=4096,
76
+ projection_dim=768,
77
+ num_hidden_layers=24,
78
+ num_attention_heads=16,
79
+ num_channels=3,
80
+ image_size=224,
81
+ patch_size=14,
82
+ hidden_act="quick_gelu",
83
+ layer_norm_eps=1e-6,
84
+ attention_dropout=0.0,
85
+ initializer_range=0.02,
86
+ initializer_factor=1.0,
87
+ use_flash_attn=False,
88
+ **kwargs,
89
+ ):
90
+ super().__init__(**kwargs)
91
+ self.hidden_size = hidden_size
92
+ self.intermediate_size = intermediate_size
93
+ self.projection_dim = projection_dim
94
+ self.num_hidden_layers = num_hidden_layers
95
+ self.num_attention_heads = num_attention_heads
96
+ self.num_channels = num_channels
97
+ self.patch_size = patch_size
98
+ self.image_size = image_size
99
+ self.initializer_range = initializer_range
100
+ self.initializer_factor = initializer_factor
101
+ self.attention_dropout = attention_dropout
102
+ self.layer_norm_eps = layer_norm_eps
103
+ self.hidden_act = hidden_act
104
+ self.use_flash_attn = use_flash_attn
105
+
106
+ @classmethod
107
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
108
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
109
+
110
+ # get the vision config dict if we are loading from MplugOwlConfig
111
+ if config_dict.get("model_type") == "mplug-owl":
112
+ config_dict = config_dict["vision_config"]
113
+
114
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
115
+ logger.warning(f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")
116
+
117
+ return cls.from_dict(config_dict, **kwargs)
118
+
119
+
120
+ class MplugOwlVisualAbstractorConfig(PretrainedConfig):
121
+ model_type = "mplug_owl_visual_abstract"
122
+
123
+ def __init__(
124
+ self,
125
+ hidden_size=1024, #
126
+ num_hidden_layers=6, #
127
+ num_attention_heads=16, #
128
+ intermediate_size=4096, #
129
+ attention_probs_dropout_prob=0.1, #
130
+ initializer_range=0.02,
131
+ layer_norm_eps=1e-6, #
132
+ encoder_hidden_size=1024, #
133
+ **kwargs,
134
+ ):
135
+ super().__init__(**kwargs)
136
+ self.hidden_size = hidden_size
137
+ self.num_hidden_layers = num_hidden_layers
138
+ self.num_attention_heads = num_attention_heads
139
+ self.intermediate_size = intermediate_size
140
+ self.attention_probs_dropout_prob = attention_probs_dropout_prob
141
+ self.initializer_range = initializer_range
142
+ self.layer_norm_eps = layer_norm_eps
143
+ self.encoder_hidden_size = encoder_hidden_size
144
+
145
+ @classmethod
146
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
147
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
148
+
149
+ # get the visual_abstractor config dict if we are loading from MplugOwlConfig
150
+ if config_dict.get("model_type") == "mplug-owl":
151
+ config_dict = config_dict["abstractor_config"]
152
+
153
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
154
+ logger.warning(f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")
155
+
156
+ return cls.from_dict(config_dict, **kwargs)
157
+
158
+
159
+ class MplugOwlConfig(PretrainedConfig):
160
+ r"""
161
+ [`MplugOwlConfig`] is the configuration class to store the configuration of a [`MplugOwlForConditionalGeneration`]. It is
162
+ used to instantiate a mPLUG-Owl model according to the specified arguments, defining the vision model, Q-Former model
163
+ and language model configs. Instantiating a configuration with the defaults will yield a similar configuration to
164
+ that of the mPLUG-Owl [x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b) architecture.
165
+
166
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
167
+ documentation from [`PretrainedConfig`] for more information.
168
+
169
+ Args:
170
+ vision_config (`dict`, *optional*):
171
+ Dictionary of configuration options used to initialize [`MplugOwlVisionConfig`].
172
+ visual_abstractor_config (`dict`, *optional*):
173
+ Dictionary of configuration options used to initialize [`MplugOwlVisualAbstractorConfig`].
174
+ text_config (`dict`, *optional*):
175
+ Dictionary of configuration options used to initialize any [`PretrainedConfig`].
176
+ num_query_tokens (`int`, *optional*, defaults to 32):
177
+ The number of query tokens passed through the Transformer.
178
+
179
+ kwargs (*optional*):
180
+ Dictionary of keyword arguments.
181
+
182
+ Example:
183
+
184
+ ```python
185
+ >>> from transformers import (
186
+ ... MplugOwlVisionConfig,
187
+ ... MplugOwlVisualAbstractorConfig,
188
+ ... OPTConfig,
189
+ ... MplugOwlConfig,
190
+ ... MplugOwlForConditionalGeneration,
191
+ ... )
192
+
193
+ >>> # Initializing a MplugOwlConfig with x-plug/x_plug-llama-7b style configuration
194
+ >>> configuration = MplugOwlConfig()
195
+
196
+ >>> # Initializing a MplugOwlForConditionalGeneration (with random weights) from the x-plug/x_plug-llama-7b style configuration
197
+ >>> model = MplugOwlForConditionalGeneration(configuration)
198
+
199
+ >>> # Accessing the model configuration
200
+ >>> configuration = model.config
201
+
202
+ >>> # We can also initialize a MplugOwlConfig from a MplugOwlVisionConfig, MplugOwlVisualAbstractorConfig and any PretrainedConfig
203
+
204
+ >>> # Initializing mPLUG-Owl vision, mPLUG-Owl Q-Former and language model configurations
205
+ >>> vision_config = MplugOwlVisionConfig()
206
+ >>> visual_abstractor_config = MplugOwlVisualAbstractorConfig()
207
+ >>> text_config = OPTConfig()
208
+
209
+ >>> config = MplugOwlConfig.from_text_vision_configs(vision_config, visual_abstractor_config, text_config)
210
+ ```"""
211
+
212
+ model_type = "mplug-owl"
213
+ is_composition = True
214
+
215
+ def __init__(self, vision_config=None, visual_abstractor_config=None, text_config=None, num_query_tokens=64, **kwargs):
216
+ super().__init__(**kwargs)
217
+ if vision_config is None:
218
+ vision_config = MplugOwlVisionConfig().to_dict()
219
+ logger.info("vision_config is None.")
220
+
221
+ if visual_abstractor_config is None:
222
+ visual_abstractor_config = {}
223
+ logger.info("abstractor_config is None. ")
224
+
225
+ if text_config is None:
226
+ # we use LLAMA 7b by default
227
+ from ..llama.configuration_llama import LlamaConfig
228
+
229
+ text_config = LlamaConfig(pad_token_id=2).to_dict()
230
+ logger.info("text_config is None.")
231
+
232
+ self.vision_config = MplugOwlVisionConfig(**vision_config)
233
+ self.visual_abstractor_config = MplugOwlVisualAbstractorConfig(**visual_abstractor_config)
234
+ # self.visual_abstractor_config.layer_norm_eps = 1e-6
235
+ text_model_type = text_config["model_type"] if "model_type" in text_config else "llama"
236
+ self.text_config = CONFIG_MAPPING[text_model_type](**text_config)
237
+
238
+ self.tie_word_embeddings = self.text_config.tie_word_embeddings
239
+ self.is_encoder_decoder = self.text_config.is_encoder_decoder
240
+
241
+ self.num_query_tokens = num_query_tokens
242
+ # self.visual_abstractor_config.encoder_hidden_size = self.vision_config.hidden_size
243
+ self.use_decoder_only_language_model = self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
244
+ self.initializer_factor = 1.0
245
+ self.initializer_range = 0.02
246
+
247
+ for attr in dir(self.text_config):
248
+ if not hasattr(self, attr):
249
+ setattr(self, attr, getattr(self.text_config, attr))
250
+
251
+ @classmethod
252
+ def from_vision_visual_abstractor_text_configs(
253
+ cls,
254
+ vision_config: MplugOwlVisionConfig,
255
+ visual_abstractor_config: MplugOwlVisualAbstractorConfig,
256
+ text_config: PretrainedConfig,
257
+ **kwargs,
258
+ ):
259
+ r"""
260
+ Instantiate a [`MplugOwlConfig`] (or a derived class) from a mPLUG-Owl vision model, Q-Former and language model
261
+ configurations.
262
+
263
+ Returns:
264
+ [`MplugOwlConfig`]: An instance of a configuration object
265
+ """
266
+
267
+ return cls(
268
+ vision_config=vision_config.to_dict(),
269
+ visual_abstractor_config=visual_abstractor_config.to_dict(),
270
+ text_config=text_config.to_dict(),
271
+ **kwargs,
272
+ )
273
+
274
+ def to_dict(self):
275
+ """
276
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
277
+
278
+ Returns:
279
+ `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
280
+ """
281
+ output = copy.deepcopy(self.__dict__)
282
+ output["vision_config"] = self.vision_config.to_dict()
283
+ output["visual_abstractor_config"] = self.visual_abstractor_config.to_dict()
284
+ output["text_config"] = self.text_config.to_dict()
285
+ output["model_type"] = self.__class__.model_type
286
+ return output
LaViDa/eval/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py ADDED
@@ -0,0 +1,1850 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 x-plug The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ PyTorch MplugOwl model. """
16
+
17
+ import math
18
+ from typing import Any, Optional, Tuple, Union
19
+
20
+ from loguru import logger
21
+
22
+ try:
23
+ from flash_attn.flash_attn_interface import flash_attn_unpadded_func
24
+
25
+ flash_attn_func = flash_attn_unpadded_func
26
+ except:
27
+ flash_attn_func = None
28
+ logger.warning("Error importing flash_attn in mplug_owl. Please install flash-attn first.")
29
+
30
+ import math
31
+ from dataclasses import dataclass
32
+ from typing import Any, Optional, Tuple, Union
33
+
34
+ import einops
35
+ import torch
36
+ import torch.utils.checkpoint
37
+ from torch import nn
38
+ from transformers.modeling_outputs import (
39
+ BaseModelOutput,
40
+ BaseModelOutputWithPastAndCrossAttentions,
41
+ BaseModelOutputWithPooling,
42
+ )
43
+ from transformers.modeling_utils import PreTrainedModel
44
+ from transformers.models.auto import AutoModelForCausalLM
45
+ from transformers.pytorch_utils import (
46
+ find_pruneable_heads_and_indices,
47
+ prune_linear_layer,
48
+ )
49
+ from transformers.utils import (
50
+ ModelOutput,
51
+ add_start_docstrings,
52
+ add_start_docstrings_to_model_forward,
53
+ replace_return_docstrings,
54
+ )
55
+
56
+ from .configuration_mplug_owl import (
57
+ MplugOwlConfig,
58
+ MplugOwlVisionConfig,
59
+ MplugOwlVisualAbstractorConfig,
60
+ )
61
+
62
+ _CHECKPOINT_FOR_DOC = "MAGAer13/mplug-owl-llama-7b"
63
+ _CONFIG_FOR_DOC = "MplugOwlConfig"
64
+
65
+
66
+ MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST = [
67
+ "MAGAer13/mplug-owl-llama-7b",
68
+ # See all MplugOwl models at https://huggingface.co/models?filter=mplug_owl
69
+ ]
70
+
71
+
72
+ @dataclass
73
+ class MplugOwlForConditionalGenerationModelOutput(ModelOutput):
74
+ """
75
+ Class defining the outputs of [`MPlugOwlForConditionalGeneration`].
76
+
77
+ Args:
78
+ loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
79
+ Language modeling loss from the language model.
80
+ logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
81
+ Prediction scores of the language modeling head of the language model.
82
+ vision_outputs (`BaseModelOutputWithPooling`):
83
+ Outputs of the vision encoder.
84
+
85
+ language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
86
+ Outputs of the language model.
87
+ """
88
+
89
+ loss: Optional[Tuple[torch.FloatTensor]] = None
90
+ logits: Optional[Tuple[torch.FloatTensor]] = None
91
+ vision_outputs: Optional[torch.FloatTensor] = None
92
+ language_model_outputs: Optional[Tuple[torch.FloatTensor]] = None
93
+
94
+ def to_tuple(self) -> Tuple[Any]:
95
+ return tuple(self[k] if k not in ["vision_outputs", "language_model_outputs"] else getattr(self, k).to_tuple() for k in self.keys())
96
+
97
+
98
+ def get_ltor_masks_and_position_ids_from_embeddings(data):
99
+ """Build masks and position id for left to right model."""
100
+
101
+ # Extract batch size and sequence length.
102
+ micro_batch_size, seq_length = data.size()[:2]
103
+
104
+ # Attention mask (lower triangular).
105
+ att_mask_batch = 1
106
+ attention_mask = torch.tril(torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)).view(att_mask_batch, 1, seq_length, seq_length)
107
+
108
+ # Loss mask.
109
+ loss_mask = torch.ones(data.size()[:2], dtype=torch.float, device=data.device)
110
+
111
+ # Position ids.
112
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
113
+ position_ids = position_ids.unsqueeze(0).expand_as(data[..., 0])
114
+
115
+ # Convert attention mask to binary:
116
+ attention_mask = attention_mask < 0.5
117
+
118
+ return attention_mask, loss_mask, position_ids
119
+
120
+
121
+ class MplugOwlVisionEmbeddings(nn.Module):
122
+ def __init__(self, config: MplugOwlVisionConfig):
123
+ super().__init__()
124
+ self.config = config
125
+ self.hidden_size = config.hidden_size
126
+ self.image_size = config.image_size
127
+ self.patch_size = config.patch_size
128
+
129
+ self.cls_token = nn.Parameter(torch.randn(1, 1, self.hidden_size))
130
+
131
+ self.patch_embed = nn.Conv2d(
132
+ in_channels=3,
133
+ out_channels=self.hidden_size,
134
+ kernel_size=self.patch_size,
135
+ stride=self.patch_size,
136
+ bias=False,
137
+ )
138
+
139
+ self.num_patches = (self.image_size // self.patch_size) ** 2
140
+
141
+ self.position_embedding = nn.Parameter(torch.randn(1, self.num_patches + 1, self.hidden_size))
142
+
143
+ self.pre_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
144
+
145
+ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
146
+ # [B, C, T, H, W] or [B, C, H, W]
147
+ batch_size = pixel_values.size(0)
148
+ T = pixel_values.size(2) if pixel_values.dim() > 4 else 1
149
+ if T > 1:
150
+ pixel_values = einops.rearrange(pixel_values, "b c t h w -> (b t) c h w")
151
+ image_embeds = self.patch_embed(pixel_values)
152
+ image_embeds = image_embeds.flatten(2).transpose(1, 2)
153
+
154
+ class_embeds = self.cls_token.expand(batch_size * T, 1, -1).to(image_embeds.dtype)
155
+ embeddings = torch.cat([class_embeds, image_embeds], dim=1)
156
+ embeddings = embeddings + self.position_embedding[:, : embeddings.size(1)].to(image_embeds.dtype)
157
+ embeddings = self.pre_layernorm(embeddings)
158
+ embeddings = einops.rearrange(embeddings, "(b t) n d -> b t n d", b=batch_size)
159
+ return embeddings
160
+
161
+
162
+ class LayerNormFp32(nn.LayerNorm):
163
+ """Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back)."""
164
+
165
+ def __init__(self, *args, **kwargs):
166
+ super().__init__(*args, **kwargs)
167
+
168
+ def forward(self, x: torch.Tensor):
169
+ output = torch.nn.functional.layer_norm(
170
+ x.float(),
171
+ self.normalized_shape,
172
+ self.weight.float() if self.weight is not None else None,
173
+ self.bias.float() if self.bias is not None else None,
174
+ self.eps,
175
+ )
176
+ return output.type_as(x)
177
+
178
+
179
+ class QuickGELU(nn.Module):
180
+ def forward(self, x: torch.Tensor):
181
+ return x * torch.sigmoid(1.702 * x)
182
+
183
+
184
+ class MplugOwlVisionLocalTemporal(nn.Module):
185
+ def __init__(self, config):
186
+ super(MplugOwlVisionLocalTemporal, self).__init__()
187
+
188
+ self.image_size = config.image_size
189
+ self.patch_size = config.patch_size
190
+ self.num_patches = 1 + (self.image_size // self.patch_size) ** 2
191
+ self.hidden_size = config.hidden_size
192
+ d_bottleneck = self.hidden_size // 2
193
+
194
+ self.ln = LayerNormFp32(self.hidden_size)
195
+ self.down_proj = nn.Conv3d(self.hidden_size, d_bottleneck, kernel_size=1, stride=1, padding=0)
196
+ self.conv = nn.Conv3d(d_bottleneck, d_bottleneck, kernel_size=(3, 1, 1), stride=1, padding=(1, 0, 0), groups=d_bottleneck)
197
+ self.up_proj = nn.Conv3d(d_bottleneck, self.hidden_size, kernel_size=1, stride=1, padding=0)
198
+
199
+ nn.init.constant_(self.up_proj.weight, 0)
200
+ nn.init.constant_(self.up_proj.bias, 0)
201
+
202
+ self.activation_func = QuickGELU()
203
+
204
+ def forward(self, x):
205
+ # [b, t, s, c]
206
+ T = x.size(1)
207
+ H = int((self.num_patches - 1) ** 0.5)
208
+ cls_token, x = x[:, :, 0:1], x[:, :, 1:]
209
+ x = self.ln(x)
210
+ x = einops.rearrange(x, "b t (h w) c -> b c t h w", h=H)
211
+ x = self.down_proj(x)
212
+ if self.conv.weight.dtype == torch.bfloat16:
213
+ x = torch.nn.functional.conv3d(x.half(), self.conv.weight.half(), bias=self.conv.bias.half(), stride=1, padding=(1, 0, 0), groups=self.conv.weight.shape[0]).to(cls_token.dtype)
214
+ else:
215
+ x = self.conv(x)
216
+ x = self.activation_func(x)
217
+ x = self.up_proj(x)
218
+ x = einops.rearrange(x, "b c t h w -> b t (h w) c")
219
+ x = torch.cat([cls_token, x], dim=2)
220
+ return x
221
+
222
+
223
+ class MplugOwlVisionAttention(nn.Module):
224
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
225
+
226
+ def __init__(self, config):
227
+ super().__init__()
228
+ self.config = config
229
+ self.hidden_size = config.hidden_size
230
+ self.num_heads = config.num_attention_heads
231
+ self.head_dim = self.hidden_size // self.num_heads
232
+ if self.head_dim * self.num_heads != self.hidden_size:
233
+ raise ValueError(f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" f" {self.num_heads}).")
234
+ self.scale = self.head_dim**-0.5
235
+ self.dropout = nn.Dropout(config.attention_dropout)
236
+
237
+ self.query_key_value = nn.Linear(self.hidden_size, 3 * self.hidden_size)
238
+ self.dense = nn.Linear(self.hidden_size, self.hidden_size)
239
+
240
+ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
241
+ return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
242
+
243
+ def forward(
244
+ self,
245
+ hidden_states: torch.Tensor,
246
+ head_mask: Optional[torch.Tensor] = None,
247
+ output_attentions: Optional[bool] = False,
248
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
249
+ """Input shape: Batch x Time x Channel"""
250
+
251
+ bsz, seq_len, embed_dim = hidden_states.size()
252
+
253
+ mixed_qkv = self.query_key_value(hidden_states)
254
+
255
+ mixed_qkv = mixed_qkv.reshape(bsz, seq_len, self.num_heads, 3, embed_dim // self.num_heads).permute(3, 0, 2, 1, 4) # [3, b, np, sq, hn]
256
+ query_states, key_states, value_states = (
257
+ mixed_qkv[0],
258
+ mixed_qkv[1],
259
+ mixed_qkv[2],
260
+ )
261
+ # if self.config.use_flash_attn and flash_attn_func is not None:
262
+ if False:
263
+ # [b*sq, np, hn]
264
+ query_states = query_states.permute(0, 2, 1, 3).contiguous()
265
+ query_states = query_states.view(query_states.size(0) * query_states.size(1), query_states.size(2), -1)
266
+
267
+ key_states = key_states.permute(0, 2, 1, 3).contiguous()
268
+ key_states = key_states.view(key_states.size(0) * key_states.size(1), key_states.size(2), -1)
269
+
270
+ value_states = value_states.permute(0, 2, 1, 3).contiguous()
271
+ value_states = value_states.view(value_states.size(0) * value_states.size(1), value_states.size(2), -1)
272
+
273
+ cu_seqlens = torch.arange(0, (bsz + 1) * seq_len, step=seq_len, dtype=torch.int32, device=query_states.device)
274
+
275
+ context_layer = flash_attn_func(
276
+ query_states,
277
+ key_states,
278
+ value_states,
279
+ cu_seqlens,
280
+ cu_seqlens,
281
+ seq_len,
282
+ seq_len,
283
+ self.dropout if self.training else 0.0,
284
+ softmax_scale=self.scale,
285
+ causal=False,
286
+ return_attn_probs=False,
287
+ )
288
+ # [b*sq, np, hn] => [b, sq, np, hn]
289
+ context_layer = context_layer.view(bsz, seq_len, context_layer.size(1), context_layer.size(2))
290
+ else:
291
+ # Take the dot product between "query" and "key" to get the raw attention scores.
292
+ attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2))
293
+
294
+ attention_scores = attention_scores * self.scale
295
+
296
+ # Normalize the attention scores to probabilities.
297
+ attention_probs = torch.softmax(attention_scores, dim=-1)
298
+
299
+ # This is actually dropping out entire tokens to attend to, which might
300
+ # seem a bit unusual, but is taken from the original Transformer paper.
301
+ attention_probs = self.dropout(attention_probs)
302
+
303
+ # Mask heads if we want to
304
+ if head_mask is not None:
305
+ attention_probs = attention_probs * head_mask
306
+
307
+ context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3)
308
+
309
+ new_context_layer_shape = context_layer.size()[:-2] + (self.hidden_size,)
310
+ context_layer = context_layer.reshape(new_context_layer_shape)
311
+
312
+ output = self.dense(context_layer)
313
+
314
+ outputs = (output, attention_probs) if output_attentions else (output, None)
315
+
316
+ return outputs
317
+
318
+
319
+ class MplugOwlMLP(nn.Module):
320
+ def __init__(self, config):
321
+ super().__init__()
322
+ self.config = config
323
+ self.activation_fn = QuickGELU()
324
+ self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
325
+ self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
326
+
327
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
328
+ hidden_states = self.fc1(hidden_states)
329
+ hidden_states = self.activation_fn(hidden_states)
330
+ hidden_states = self.fc2(hidden_states)
331
+ return hidden_states
332
+
333
+
334
+ class MplugOwlVisionEncoderLayer(nn.Module):
335
+ def __init__(self, config: MplugOwlVisionConfig):
336
+ super().__init__()
337
+ self.hidden_size = config.hidden_size
338
+ self.temporal = MplugOwlVisionLocalTemporal(config)
339
+ self.self_attn = MplugOwlVisionAttention(config)
340
+ self.input_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
341
+ self.mlp = MplugOwlMLP(config)
342
+ self.post_attention_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
343
+
344
+ def forward(
345
+ self,
346
+ hidden_states: torch.Tensor,
347
+ attention_mask: torch.Tensor,
348
+ output_attentions: Optional[bool] = False,
349
+ ) -> Tuple[torch.FloatTensor]:
350
+ """
351
+ Args:
352
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, time, seq_len, embed_dim)`
353
+ attention_mask (`torch.FloatTensor`): attention mask of size
354
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
355
+ `(config.encoder_attention_heads,)`.
356
+ output_attentions (`bool`, *optional*):
357
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
358
+ returned tensors for more detail.
359
+ """
360
+ B, T = hidden_states.size(0), hidden_states.size(1)
361
+ if T > 1:
362
+ hidden_states = hidden_states + self.temporal(hidden_states)
363
+ hidden_states = einops.rearrange(hidden_states, "b t n d -> (b t) n d")
364
+
365
+ residual = hidden_states
366
+
367
+ hidden_states = self.input_layernorm(hidden_states)
368
+ hidden_states, attn_weights = self.self_attn(
369
+ hidden_states=hidden_states,
370
+ head_mask=attention_mask,
371
+ output_attentions=output_attentions,
372
+ )
373
+ hidden_states = hidden_states + residual
374
+ residual = hidden_states
375
+ hidden_states = self.post_attention_layernorm(hidden_states)
376
+ hidden_states = self.mlp(hidden_states)
377
+
378
+ hidden_states = hidden_states + residual
379
+ hidden_states = einops.rearrange(hidden_states, "(b t) n d -> b t n d", b=B)
380
+
381
+ outputs = (hidden_states,)
382
+
383
+ if output_attentions:
384
+ outputs += (attn_weights,)
385
+
386
+ return outputs
387
+
388
+
389
+ class MplugOwlPreTrainedModel(PreTrainedModel):
390
+ """
391
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
392
+ models.
393
+ """
394
+
395
+ config_class = MplugOwlConfig
396
+ base_model_prefix = "mplug_owl"
397
+ supports_gradient_checkpointing = True
398
+ _keys_to_ignore_on_load_missing = [
399
+ r"position_ids",
400
+ r"language_model.encoder.embed_tokens.weight",
401
+ r"language_model.decoder.embed_tokens.weight",
402
+ r"language_model.lm_head.weight",
403
+ ]
404
+ _no_split_modules = [
405
+ "MplugOwlVisionEncoderLayer",
406
+ "LlamaDecoderLayer",
407
+ "MplugOwlVisualAbstractorLayer",
408
+ "LlamaForCausalLM",
409
+ "Parameter",
410
+ ]
411
+ _keep_in_fp32_modules = ["wo"]
412
+
413
+ def _init_weights(self, module):
414
+ """Initialize the weights"""
415
+ factor = self.config.initializer_range
416
+ if isinstance(module, nn.Conv2d) or isinstance(module, nn.Embedding) or isinstance(module, nn.Linear):
417
+ module.weight.data.normal_(mean=0.0, std=factor)
418
+ if hasattr(module, "bias") and module.bias is not None:
419
+ module.bias.data.zero_()
420
+
421
+ if isinstance(module, MplugOwlVisionEmbeddings):
422
+ if hasattr(self.config, "vision_config"):
423
+ factor = self.config.vision_config.initializer_range
424
+ nn.init.trunc_normal_(module.position_embedding, mean=0.0, std=factor)
425
+ nn.init.trunc_normal_(module.cls_token, mean=0.0, std=factor)
426
+
427
+ elif isinstance(module, nn.LayerNorm):
428
+ module.bias.data.zero_()
429
+ module.weight.data.fill_(1.0)
430
+ elif isinstance(module, nn.Linear) and module.bias is not None:
431
+ module.bias.data.zero_()
432
+ elif isinstance(module, nn.Parameter):
433
+ raise ValueError
434
+ nn.init.trunc_normal_(module.data, mean=0.0, std=factor)
435
+
436
+ def _set_gradient_checkpointing(self, module, value=False):
437
+ if isinstance(module, MplugOwlVisionEncoder):
438
+ module.gradient_checkpointing = value
439
+
440
+
441
+ MPLUG_OWL_START_DOCSTRING = r"""
442
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
443
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
444
+ etc.)
445
+
446
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
447
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
448
+ and behavior.
449
+
450
+ Parameters:
451
+ config ([`MplugOwlConfig`]): Model configuration class with all the parameters of the model.
452
+ Initializing with a config file does not load the weights associated with the model, only the
453
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
454
+ """
455
+
456
+ MPLUG_OWL_VISION_INPUTS_DOCSTRING = r"""
457
+ Args:
458
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
459
+ Pixel values. Pixel values can be obtained using [`MplugOwlProcessor`]. See [`MplugOwlProcessor.__call__`] for
460
+ details.
461
+ output_attentions (`bool`, *optional*):
462
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
463
+ tensors for more detail.
464
+ output_hidden_states (`bool`, *optional*):
465
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
466
+ more detail.
467
+ return_dict (`bool`, *optional*):
468
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
469
+ """
470
+
471
+ MPLUG_OWL_TEXT_INPUTS_DOCSTRING = r"""
472
+ Args:
473
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
474
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
475
+ it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
476
+ [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
477
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
478
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
479
+ - 1 for tokens that are **not masked**,
480
+ - 0 for tokens that are **masked**.
481
+ [What are attention masks?](../glossary#attention-mask)
482
+ decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
483
+ Indices of decoder input sequence tokens in the vocabulary.
484
+
485
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
486
+ [`PreTrainedTokenizer.__call__`] for details.
487
+
488
+ [What are decoder input IDs?](../glossary#decoder-input-ids)
489
+
490
+ T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
491
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
492
+
493
+ To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
494
+ Training](./t5#training).
495
+ decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
496
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
497
+ be used by default.
498
+ output_attentions (`bool`, *optional*):
499
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
500
+ tensors for more detail.
501
+ output_hidden_states (`bool`, *optional*):
502
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
503
+ more detail.
504
+ return_dict (`bool`, *optional*):
505
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
506
+ """
507
+
508
+ MPLUG_OWL_INPUTS_DOCSTRING = r"""
509
+ Args:
510
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
511
+ Pixel values. Pixel values can be obtained using [`MplugOwlProcessor`]. See [`MplugOwlProcessor.__call__`] for
512
+ details.
513
+
514
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
515
+ Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be
516
+ provided to serve as text prompt, which the language model can continue.
517
+
518
+ Indices can be obtained using [`MplugOwlProcessor`]. See [`MplugOwlProcessor.__call__`] for details.
519
+
520
+ [What are input IDs?](../glossary#input-ids)
521
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
522
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
523
+
524
+ - 1 for tokens that are **not masked**,
525
+ - 0 for tokens that are **masked**.
526
+
527
+ [What are attention masks?](../glossary#attention-mask)
528
+
529
+ decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
530
+ Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an
531
+ encoder-decoder language model (like T5) is used.
532
+
533
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
534
+ [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids)
535
+
536
+ decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
537
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
538
+ be used by default.
539
+
540
+ Only relevant in case an encoder-decoder language model (like T5) is used.
541
+
542
+ output_attentions (`bool`, *optional*):
543
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
544
+ tensors for more detail.
545
+ output_hidden_states (`bool`, *optional*):
546
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
547
+ more detail.
548
+ return_dict (`bool`, *optional*):
549
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
550
+ """
551
+
552
+
553
+ class MplugOwlVisionEncoder(nn.Module):
554
+ """
555
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
556
+ [`MplugOwlVisionEncoderLayer`].
557
+
558
+ Args:
559
+ config (`MplugOwlVisionConfig`):
560
+ The corresponding vision configuration for the `MplugOwlEncoder`.
561
+ """
562
+
563
+ def __init__(self, config: MplugOwlVisionConfig):
564
+ super().__init__()
565
+ self.config = config
566
+ self.layers = nn.ModuleList([MplugOwlVisionEncoderLayer(config) for _ in range(config.num_hidden_layers)])
567
+ self.gradient_checkpointing = False
568
+
569
+ def forward(
570
+ self,
571
+ inputs_embeds,
572
+ attention_mask: Optional[torch.Tensor] = None,
573
+ output_attentions: Optional[bool] = None,
574
+ output_hidden_states: Optional[bool] = None,
575
+ return_dict: Optional[bool] = None,
576
+ ) -> Union[Tuple, BaseModelOutput]:
577
+ r"""
578
+ Args:
579
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
580
+ Embedded representation of the inputs. Should be float, not int tokens.
581
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
582
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
583
+
584
+ - 1 for tokens that are **not masked**,
585
+ - 0 for tokens that are **masked**.
586
+
587
+ [What are attention masks?](../glossary#attention-mask)
588
+ output_attentions (`bool`, *optional*):
589
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
590
+ returned tensors for more detail.
591
+ output_hidden_states (`bool`, *optional*):
592
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
593
+ for more detail.
594
+ return_dict (`bool`, *optional*):
595
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
596
+ """
597
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
598
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
599
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
600
+
601
+ encoder_states = () if output_hidden_states else None
602
+ all_attentions = () if output_attentions else None
603
+
604
+ hidden_states = inputs_embeds
605
+ for idx, encoder_layer in enumerate(self.layers):
606
+ if output_hidden_states:
607
+ encoder_states = encoder_states + (hidden_states,)
608
+ if self.gradient_checkpointing and self.training:
609
+
610
+ def create_custom_forward(module):
611
+ def custom_forward(*inputs):
612
+ return module(*inputs, output_attentions)
613
+
614
+ return custom_forward
615
+
616
+ layer_outputs = torch.utils.checkpoint.checkpoint(
617
+ create_custom_forward(encoder_layer),
618
+ hidden_states,
619
+ attention_mask,
620
+ )
621
+ else:
622
+ layer_outputs = encoder_layer(
623
+ hidden_states,
624
+ attention_mask,
625
+ output_attentions=output_attentions,
626
+ )
627
+
628
+ hidden_states = layer_outputs[0]
629
+
630
+ if output_attentions:
631
+ all_attentions = all_attentions + (layer_outputs[1],)
632
+
633
+ if output_hidden_states:
634
+ encoder_states = encoder_states + (hidden_states,)
635
+
636
+ if not return_dict:
637
+ return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
638
+ return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions)
639
+
640
+
641
+ class MplugOwlVisionModel(MplugOwlPreTrainedModel):
642
+ main_input_name = "pixel_values"
643
+ config_class = MplugOwlVisionConfig
644
+
645
+ def __init__(self, config: MplugOwlVisionConfig):
646
+ super().__init__(config)
647
+ self.config = config
648
+ self.hidden_size = config.hidden_size
649
+
650
+ self.embeddings = MplugOwlVisionEmbeddings(config)
651
+ self.encoder = MplugOwlVisionEncoder(config)
652
+ self.post_layernorm = LayerNormFp32(self.hidden_size, eps=config.layer_norm_eps)
653
+
654
+ self.post_init()
655
+
656
+ @add_start_docstrings_to_model_forward(MPLUG_OWL_VISION_INPUTS_DOCSTRING)
657
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=MplugOwlVisionConfig)
658
+ def forward(
659
+ self,
660
+ pixel_values: Optional[torch.FloatTensor] = None,
661
+ output_attentions: Optional[bool] = None,
662
+ output_hidden_states: Optional[bool] = None,
663
+ return_dict: Optional[bool] = None,
664
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
665
+ r"""
666
+ Returns:
667
+
668
+ """
669
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
670
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
671
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
672
+
673
+ if pixel_values is None:
674
+ raise ValueError("You have to specify pixel_values")
675
+
676
+ hidden_states = self.embeddings(pixel_values) # [B, T, N, D]
677
+
678
+ encoder_outputs = self.encoder(
679
+ inputs_embeds=hidden_states,
680
+ output_attentions=output_attentions,
681
+ output_hidden_states=output_hidden_states,
682
+ return_dict=return_dict,
683
+ )
684
+
685
+ last_hidden_state = encoder_outputs[0]
686
+ last_hidden_state = self.post_layernorm(last_hidden_state)
687
+
688
+ pooled_output = last_hidden_state[:, :, 0, :].mean(1)
689
+ pooled_output = self.post_layernorm(pooled_output)
690
+
691
+ if not return_dict:
692
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
693
+
694
+ return BaseModelOutputWithPooling(
695
+ last_hidden_state=last_hidden_state,
696
+ pooler_output=pooled_output,
697
+ hidden_states=encoder_outputs.hidden_states,
698
+ attentions=encoder_outputs.attentions,
699
+ )
700
+
701
+ def get_input_embeddings(self):
702
+ return self.embeddings
703
+
704
+
705
+ class MplugOwlVisualAbstractorMLP(nn.Module):
706
+ def __init__(self, config: MplugOwlVisualAbstractorConfig):
707
+ super().__init__()
708
+ self.config = config
709
+ in_features = config.hidden_size
710
+ hidden_features = config.intermediate_size
711
+ if hidden_features != 2816:
712
+ hidden_features = int(2 * hidden_features / 3)
713
+ multiple_of = 256
714
+ hidden_features = multiple_of * ((hidden_features + multiple_of - 1) // multiple_of)
715
+ self.act = nn.SiLU()
716
+
717
+ self.w1 = nn.Linear(in_features, hidden_features)
718
+ self.w2 = nn.Linear(hidden_features, in_features)
719
+ self.w3 = nn.Linear(in_features, hidden_features)
720
+ self.ffn_ln = LayerNormFp32(hidden_features, eps=config.layer_norm_eps)
721
+
722
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
723
+ hidden_states = self.act(self.w1(hidden_states)) * self.w3(hidden_states)
724
+ hidden_states = self.ffn_ln(hidden_states)
725
+ hidden_states = self.w2(hidden_states)
726
+ return hidden_states
727
+
728
+
729
+ class MplugOwlVisualAbstractorMultiHeadAttention(nn.Module):
730
+ def __init__(self, config: MplugOwlVisualAbstractorConfig):
731
+ super().__init__()
732
+ self.config = config
733
+ if config.hidden_size % config.num_attention_heads != 0:
734
+ raise ValueError("The hidden size (%d) is not a multiple of the number of attention heads (%d)" % (config.hidden_size, config.num_attention_heads))
735
+
736
+ self.num_attention_heads = config.num_attention_heads
737
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
738
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
739
+
740
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
741
+ self.key = nn.Linear(config.encoder_hidden_size, self.all_head_size)
742
+ self.value = nn.Linear(config.encoder_hidden_size, self.all_head_size)
743
+
744
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
745
+ self.save_attention = False
746
+
747
+ def save_attn_gradients(self, attn_gradients):
748
+ self.attn_gradients = attn_gradients
749
+
750
+ def get_attn_gradients(self):
751
+ return self.attn_gradients
752
+
753
+ def save_attention_map(self, attention_map):
754
+ self.attention_map = attention_map
755
+
756
+ def get_attention_map(self):
757
+ return self.attention_map
758
+
759
+ def transpose_for_scores(self, x):
760
+ new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
761
+ x = x.view(*new_x_shape)
762
+ return x.permute(0, 2, 1, 3)
763
+
764
+ def forward(
765
+ self,
766
+ hidden_states,
767
+ attention_mask=None,
768
+ head_mask=None,
769
+ encoder_hidden_states=None,
770
+ encoder_attention_mask=None,
771
+ past_key_value=None,
772
+ output_attentions=False,
773
+ ):
774
+ # If this is instantiated as a cross-attention module, the keys
775
+ # and values come from an encoder; the attention mask needs to be
776
+ # such that the encoder's padding tokens are not attended to.
777
+ key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
778
+ value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
779
+ attention_mask = encoder_attention_mask
780
+
781
+ mixed_query_layer = self.query(hidden_states)
782
+
783
+ query_layer = self.transpose_for_scores(mixed_query_layer)
784
+
785
+ past_key_value = (key_layer, value_layer)
786
+
787
+ # Take the dot product between "query" and "key" to get the raw attention scores.
788
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
789
+
790
+ attention_scores = attention_scores / math.sqrt(self.attention_head_size)
791
+
792
+ if attention_mask is not None:
793
+ # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
794
+ attention_scores = attention_scores + attention_mask
795
+
796
+ # Normalize the attention scores to probabilities.
797
+ attention_probs = nn.Softmax(dim=-1)(attention_scores)
798
+
799
+ if self.save_attention:
800
+ self.save_attention_map(attention_probs)
801
+ attention_probs.register_hook(self.save_attn_gradients)
802
+
803
+ # This is actually dropping out entire tokens to attend to, which might
804
+ # seem a bit unusual, but is taken from the original Transformer paper.
805
+ attention_probs_dropped = self.dropout(attention_probs)
806
+
807
+ # Mask heads if we want to
808
+ if head_mask is not None:
809
+ attention_probs_dropped = attention_probs_dropped * head_mask
810
+
811
+ context_layer = torch.matmul(attention_probs_dropped, value_layer)
812
+
813
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
814
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
815
+ context_layer = context_layer.view(*new_context_layer_shape)
816
+
817
+ outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
818
+
819
+ outputs = outputs + (past_key_value,)
820
+ return outputs
821
+
822
+
823
+ class MplugOwlVisualAbstractorCrossOutput(nn.Module):
824
+ def __init__(self, config: MplugOwlVisualAbstractorConfig):
825
+ super().__init__()
826
+ dim = config.hidden_size
827
+ self.out_proj = nn.Linear(dim, dim, bias=True)
828
+ self.norm2 = LayerNormFp32(dim)
829
+ self.mlp = MplugOwlVisualAbstractorMLP(config)
830
+
831
+ def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor:
832
+ input_tensor = input_tensor + self.out_proj(hidden_states)
833
+ input_tensor = input_tensor + self.mlp(self.norm2(input_tensor))
834
+ return input_tensor
835
+
836
+
837
+ class MplugOwlVisualAbstractorAttention(nn.Module):
838
+ def __init__(self, config: MplugOwlVisualAbstractorConfig):
839
+ super().__init__()
840
+ self.attention = MplugOwlVisualAbstractorMultiHeadAttention(config)
841
+ self.output = MplugOwlVisualAbstractorCrossOutput(config)
842
+ self.pruned_heads = set()
843
+ self.norm1 = LayerNormFp32(config.hidden_size)
844
+ self.normk = LayerNormFp32(config.hidden_size)
845
+
846
+ def prune_heads(self, heads):
847
+ if len(heads) == 0:
848
+ return
849
+ heads, index = find_pruneable_heads_and_indices(heads, self.attention.num_attention_heads, self.attention.attention_head_size, self.pruned_heads)
850
+
851
+ # Prune linear layers
852
+ self.attention.query = prune_linear_layer(self.attention.query, index)
853
+ self.attention.key = prune_linear_layer(self.attention.key, index)
854
+ self.attention.value = prune_linear_layer(self.attention.value, index)
855
+ self.output.dense = prune_linear_layer(self.output.out_proj, index, dim=1)
856
+
857
+ # Update hyper params and store pruned heads
858
+ self.attention.num_attention_heads = self.attention.num_attention_heads - len(heads)
859
+ self.attention.all_head_size = self.attention.attention_head_size * self.attention.num_attention_heads
860
+ self.pruned_heads = self.pruned_heads.union(heads)
861
+
862
+ def forward(
863
+ self,
864
+ hidden_states: torch.Tensor,
865
+ attention_mask: Optional[torch.FloatTensor] = None,
866
+ head_mask: Optional[torch.FloatTensor] = None,
867
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
868
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
869
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
870
+ output_attentions: Optional[bool] = False,
871
+ ) -> Tuple[torch.Tensor]:
872
+ # HACK we apply norm on q and k
873
+ hidden_states = self.norm1(hidden_states)
874
+ encoder_hidden_states = self.normk(encoder_hidden_states)
875
+ encoder_hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1)
876
+ encoder_attention_mask = torch.cat([attention_mask, encoder_attention_mask], dim=-1)
877
+ self_outputs = self.attention(
878
+ hidden_states,
879
+ attention_mask,
880
+ head_mask,
881
+ encoder_hidden_states,
882
+ encoder_attention_mask,
883
+ past_key_value,
884
+ output_attentions,
885
+ )
886
+ attention_output = self.output(self_outputs[0], hidden_states)
887
+ # add attentions if we output them
888
+ outputs = (attention_output,) + self_outputs[1:]
889
+ return outputs
890
+
891
+
892
+ class MplugOwlVisualAbstractorLayer(nn.Module):
893
+ def __init__(self, config, layer_idx):
894
+ super().__init__()
895
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
896
+ self.seq_len_dim = 1
897
+
898
+ self.layer_idx = layer_idx
899
+
900
+ self.crossattention = MplugOwlVisualAbstractorAttention(config)
901
+ self.has_cross_attention = True
902
+
903
+ def forward(
904
+ self,
905
+ hidden_states,
906
+ attention_mask=None,
907
+ head_mask=None,
908
+ encoder_hidden_states=None,
909
+ encoder_attention_mask=None,
910
+ output_attentions=False,
911
+ ):
912
+ if encoder_hidden_states is None:
913
+ raise ValueError("encoder_hidden_states must be given for cross-attention layers")
914
+ cross_attention_outputs = self.crossattention(
915
+ hidden_states,
916
+ attention_mask,
917
+ head_mask,
918
+ encoder_hidden_states,
919
+ encoder_attention_mask,
920
+ output_attentions=output_attentions,
921
+ )
922
+ query_attention_output = cross_attention_outputs[0]
923
+
924
+ outputs = (query_attention_output,)
925
+ return outputs
926
+
927
+
928
+ class MplugOwlVisualAbstractorEncoder(nn.Module):
929
+ def __init__(self, config):
930
+ super().__init__()
931
+ self.config = config
932
+ self.layers = nn.ModuleList([MplugOwlVisualAbstractorLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)])
933
+ self.gradient_checkpointing = False
934
+
935
+ def forward(
936
+ self,
937
+ hidden_states,
938
+ attention_mask=None,
939
+ head_mask=None,
940
+ encoder_hidden_states=None,
941
+ encoder_attention_mask=None,
942
+ past_key_values=None,
943
+ output_attentions=False,
944
+ output_hidden_states=False,
945
+ return_dict=True,
946
+ ):
947
+ all_hidden_states = () if output_hidden_states else None
948
+
949
+ for i in range(self.config.num_hidden_layers):
950
+ layer_module = self.layers[i]
951
+ if output_hidden_states:
952
+ all_hidden_states = all_hidden_states + (hidden_states,)
953
+
954
+ layer_head_mask = head_mask[i] if head_mask is not None else None
955
+ past_key_value = past_key_values[i] if past_key_values is not None else None
956
+
957
+ if getattr(self.config, "gradient_checkpointing", False) and self.training:
958
+
959
+ def create_custom_forward(module):
960
+ def custom_forward(*inputs):
961
+ return module(*inputs, past_key_value, output_attentions)
962
+
963
+ return custom_forward
964
+
965
+ layer_outputs = torch.utils.checkpoint.checkpoint(
966
+ create_custom_forward(layer_module),
967
+ hidden_states,
968
+ attention_mask,
969
+ layer_head_mask,
970
+ encoder_hidden_states,
971
+ encoder_attention_mask,
972
+ )
973
+ else:
974
+ layer_outputs = layer_module(
975
+ hidden_states,
976
+ attention_mask,
977
+ layer_head_mask,
978
+ encoder_hidden_states,
979
+ encoder_attention_mask,
980
+ output_attentions,
981
+ )
982
+
983
+ hidden_states = layer_outputs[0]
984
+
985
+ return BaseModelOutput(
986
+ last_hidden_state=hidden_states,
987
+ )
988
+
989
+
990
+ class MplugOwlVisualAbstractorModel(MplugOwlPreTrainedModel):
991
+ def __init__(self, config: MplugOwlVisualAbstractorConfig, language_hidden_size):
992
+ super().__init__(config)
993
+ self.config = config
994
+
995
+ self.encoder = MplugOwlVisualAbstractorEncoder(config)
996
+ self.visual_fc = torch.nn.Linear(config.hidden_size, language_hidden_size)
997
+ self.temporal_visual_fc = torch.nn.Linear(config.hidden_size, language_hidden_size)
998
+ self.vit_eos = torch.nn.Parameter(torch.randn(1, 1, language_hidden_size))
999
+ nn.init.trunc_normal_(self.vit_eos, mean=0.0, std=self.config.initializer_range)
1000
+ self.post_init()
1001
+
1002
+ def _prune_heads(self, heads_to_prune):
1003
+ """
1004
+ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
1005
+ class PreTrainedModel
1006
+ """
1007
+ for layer, heads in heads_to_prune.items():
1008
+ self.encoder.layer[layer].attention.prune_heads(heads)
1009
+
1010
+ def get_extended_attention_mask(
1011
+ self,
1012
+ attention_mask: torch.Tensor,
1013
+ input_shape: Tuple[int],
1014
+ device: torch.device,
1015
+ ) -> torch.Tensor:
1016
+ """
1017
+ Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
1018
+
1019
+ Arguments:
1020
+ attention_mask (`torch.Tensor`):
1021
+ Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
1022
+ input_shape (`Tuple[int]`):
1023
+ The shape of the input to the model.
1024
+ device: (`torch.device`):
1025
+ The device of the input to the model.
1026
+
1027
+ Returns:
1028
+ `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
1029
+ """
1030
+ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
1031
+ # ourselves in which case we just need to make it broadcastable to all heads.
1032
+ if attention_mask.dim() == 3:
1033
+ extended_attention_mask = attention_mask[:, None, :, :]
1034
+ elif attention_mask.dim() == 2:
1035
+ # Provided a padding mask of dimensions [batch_size, seq_length]
1036
+ # - the model is an encoder, so make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
1037
+ extended_attention_mask = attention_mask[:, None, None, :]
1038
+ else:
1039
+ raise ValueError("Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(input_shape, attention_mask.shape))
1040
+
1041
+ # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
1042
+ # masked positions, this operation will create a tensor which is 0.0 for
1043
+ # positions we want to attend and -10000.0 for masked positions.
1044
+ # Since we are adding it to the raw scores before the softmax, this is
1045
+ # effectively the same as removing these entirely.
1046
+ extended_attention_mask = extended_attention_mask.to(dtype=self.dtype) # fp16 compatibility
1047
+ extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
1048
+ return extended_attention_mask
1049
+
1050
+ def forward(
1051
+ self,
1052
+ query_embeds,
1053
+ temporal_query_embeds=None,
1054
+ attention_mask=None,
1055
+ head_mask=None,
1056
+ encoder_hidden_states=None,
1057
+ encoder_attention_mask=None,
1058
+ past_key_values=None,
1059
+ output_attentions=None,
1060
+ output_hidden_states=None,
1061
+ return_dict=None,
1062
+ ):
1063
+ r"""
1064
+ encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`):
1065
+ Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
1066
+ the model is configured as a decoder.
1067
+ encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, `optional`):
1068
+ Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
1069
+ the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:
1070
+ - 1 for tokens that are **not masked**,
1071
+ - 0 for tokens that are **masked**.
1072
+ past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of:
1073
+ shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and
1074
+ value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are
1075
+ used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key
1076
+ value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape
1077
+ `(batch_size, sequence_length)`.
1078
+ """
1079
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1080
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1081
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1082
+
1083
+ T = encoder_hidden_states.size(1)
1084
+ if T == 1 or temporal_query_embeds is None:
1085
+ embedding_output = query_embeds
1086
+ else:
1087
+ embedding_output = torch.cat([query_embeds, temporal_query_embeds], dim=1)
1088
+ input_shape = embedding_output.size()[:-1]
1089
+ batch_size, seq_length = input_shape
1090
+ device = embedding_output.device
1091
+
1092
+ encoder_hidden_states = einops.rearrange(encoder_hidden_states, "b t n d -> b (t n) d")
1093
+
1094
+ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
1095
+ # ourselves in which case we just need to make it broadcastable to all heads.
1096
+ if attention_mask is None:
1097
+ attention_mask = torch.ones((embedding_output.shape[0], embedding_output.shape[1]), dtype=torch.long, device=embedding_output.device)
1098
+ extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, device)
1099
+
1100
+ # If a 2D or 3D attention mask is provided for the cross-attention
1101
+ # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
1102
+ if encoder_hidden_states is not None:
1103
+ if type(encoder_hidden_states) == list:
1104
+ encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
1105
+ else:
1106
+ (
1107
+ encoder_batch_size,
1108
+ encoder_sequence_length,
1109
+ _,
1110
+ ) = encoder_hidden_states.size()
1111
+ encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
1112
+
1113
+ if type(encoder_attention_mask) == list:
1114
+ encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
1115
+ elif encoder_attention_mask is None:
1116
+ encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
1117
+ encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
1118
+ else:
1119
+ encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
1120
+ else:
1121
+ encoder_extended_attention_mask = None
1122
+
1123
+ # Prepare head mask if needed
1124
+ # 1.0 in head_mask indicate we keep the head
1125
+ # attention_probs has shape bsz x n_heads x N x N
1126
+ # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
1127
+ # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
1128
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
1129
+
1130
+ encoder_outputs = self.encoder(
1131
+ embedding_output,
1132
+ attention_mask=extended_attention_mask,
1133
+ head_mask=head_mask,
1134
+ encoder_hidden_states=encoder_hidden_states,
1135
+ encoder_attention_mask=encoder_extended_attention_mask,
1136
+ past_key_values=past_key_values,
1137
+ output_attentions=output_attentions,
1138
+ output_hidden_states=output_hidden_states,
1139
+ return_dict=return_dict,
1140
+ )
1141
+ sequence_output = encoder_outputs[0]
1142
+ pooled_output = sequence_output[:, 0, :]
1143
+
1144
+ if T == 1 or temporal_query_embeds is None:
1145
+ temporal_sequence_output = None
1146
+ else:
1147
+ temporal_sequence_output = sequence_output[:, query_embeds.size(1) :]
1148
+ sequence_output = sequence_output[:, : query_embeds.size(1)]
1149
+
1150
+ sequence_output = self.visual_fc(sequence_output)
1151
+ if temporal_sequence_output is not None:
1152
+ sequence_output += self.temporal_visual_fc(temporal_sequence_output)
1153
+ sequence_output = torch.cat([sequence_output, self.vit_eos.repeat(sequence_output.shape[0], 1, 1)], dim=1)
1154
+
1155
+ return BaseModelOutputWithPooling(
1156
+ last_hidden_state=sequence_output,
1157
+ pooler_output=pooled_output,
1158
+ hidden_states=encoder_outputs.hidden_states,
1159
+ )
1160
+
1161
+
1162
+ @add_start_docstrings(
1163
+ """
1164
+ mPLUG-Owl Model for generating text and image features. The model consists of a vision encoder, Querying Transformer
1165
+ (Q-Former) and a language model.
1166
+ """,
1167
+ MPLUG_OWL_START_DOCSTRING,
1168
+ )
1169
+ class MplugOwlModel(MplugOwlPreTrainedModel):
1170
+ config_class = MplugOwlConfig
1171
+ main_input_name = "pixel_values"
1172
+
1173
+ def __init__(self, config: MplugOwlConfig, *inputs, **kwargs):
1174
+ super().__init__(config, *inputs, **kwargs)
1175
+
1176
+ self.vision_model = MplugOwlVisionModel(config.vision_config)
1177
+
1178
+ self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
1179
+ self.temporal_query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
1180
+ self.abstractor = MplugOwlVisualAbstractorModel(config.visual_abstractor_config, config.text_config.hidden_size)
1181
+
1182
+ # if config.use_decoder_only_language_model:
1183
+ # from llama.modeling_llama import LlamaForCausalLM
1184
+ language_model = AutoModelForCausalLM.from_config(config.text_config)
1185
+ # else:
1186
+ # language_model = AutoModelForSeq2SeqLM.from_config(config.text_config)
1187
+ self.language_model = language_model
1188
+
1189
+ # Initialize weights and apply final processing
1190
+ self.post_init()
1191
+
1192
+ def get_input_embeddings(self):
1193
+ return self.language_model.get_input_embeddings()
1194
+
1195
+ def set_input_embeddings(self, value):
1196
+ self.language_model.set_input_embeddings(value)
1197
+
1198
+ def set_output_embeddings(self, new_embeddings):
1199
+ self.language_model.set_output_embeddings(new_embeddings)
1200
+
1201
+ def get_output_embeddings(self) -> nn.Module:
1202
+ return self.language_model.get_output_embeddings()
1203
+
1204
+ def get_encoder(self):
1205
+ return self.language_model.get_encoder()
1206
+
1207
+ def get_decoder(self):
1208
+ return self.language_model.get_decoder()
1209
+
1210
+ def _tie_weights(self):
1211
+ if not self.config.use_decoder_only_language_model:
1212
+ self.language_model.encoder.embed_tokens = self.language_model.shared
1213
+ self.language_model.decoder.embed_tokens = self.language_model.shared
1214
+
1215
+ def get_text_features(
1216
+ self,
1217
+ input_ids: Optional[torch.Tensor] = None,
1218
+ attention_mask: Optional[torch.Tensor] = None,
1219
+ decoder_input_ids: Optional[torch.Tensor] = None,
1220
+ decoder_attention_mask: Optional[torch.Tensor] = None,
1221
+ labels: Optional[torch.Tensor] = None,
1222
+ output_attentions: Optional[bool] = None,
1223
+ output_hidden_states: Optional[bool] = None,
1224
+ return_dict: Optional[bool] = None,
1225
+ ):
1226
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1227
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1228
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1229
+
1230
+ if self.config.use_decoder_only_language_model:
1231
+ text_outputs = self.language_model(
1232
+ input_ids=input_ids,
1233
+ attention_mask=attention_mask,
1234
+ output_attentions=output_attentions,
1235
+ output_hidden_states=output_hidden_states,
1236
+ return_dict=return_dict,
1237
+ )
1238
+ else:
1239
+ inputs_embeds = self.language_model.get_input_embeddings()(input_ids)
1240
+
1241
+ text_outputs = self.language_model(
1242
+ inputs_embeds=inputs_embeds,
1243
+ attention_mask=attention_mask,
1244
+ decoder_input_ids=decoder_input_ids,
1245
+ decoder_attention_mask=decoder_attention_mask,
1246
+ output_attentions=output_attentions,
1247
+ output_hidden_states=output_hidden_states,
1248
+ return_dict=return_dict,
1249
+ labels=labels,
1250
+ )
1251
+
1252
+ return text_outputs
1253
+
1254
+ def get_image_features(
1255
+ self,
1256
+ pixel_values: Optional[torch.FloatTensor] = None,
1257
+ output_attentions: Optional[bool] = None,
1258
+ output_hidden_states: Optional[bool] = None,
1259
+ return_dict: Optional[bool] = None,
1260
+ ):
1261
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1262
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1263
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1264
+
1265
+ vision_outputs = self.vision_model(
1266
+ pixel_values=pixel_values,
1267
+ output_attentions=output_attentions,
1268
+ output_hidden_states=output_hidden_states,
1269
+ return_dict=return_dict,
1270
+ )
1271
+
1272
+ return vision_outputs
1273
+
1274
+
1275
+ def get_media_indices(my_list):
1276
+ if isinstance(my_list, torch.Tensor):
1277
+ my_list = my_list.cpu().tolist()
1278
+ result = []
1279
+ for i in range(len(my_list)):
1280
+ if i == 0 and my_list[i] < 0:
1281
+ result.append(i)
1282
+ elif my_list[i] != my_list[i - 1] and my_list[i] < 0:
1283
+ result.append(i)
1284
+ return result
1285
+
1286
+
1287
+ def get_media_types(tensors, positions):
1288
+ if isinstance(tensors, torch.Tensor):
1289
+ tensors = tensors.cpu().tolist()
1290
+ result = []
1291
+ for pos in positions:
1292
+ result.append(tensors[pos])
1293
+ return result
1294
+
1295
+
1296
+ @add_start_docstrings(
1297
+ """
1298
+ mPLUG-Owl Model for generating text given an image and an optional text prompt.
1299
+ """,
1300
+ MPLUG_OWL_START_DOCSTRING,
1301
+ )
1302
+ class MplugOwlForConditionalGeneration(MplugOwlPreTrainedModel):
1303
+ config_class = MplugOwlConfig
1304
+ main_input_name = "pixel_values"
1305
+
1306
+ def __init__(self, config: MplugOwlConfig):
1307
+ super().__init__(config)
1308
+
1309
+ self.vision_model = MplugOwlVisionModel(config.vision_config)
1310
+
1311
+ self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
1312
+ self.temporal_query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.visual_abstractor_config.hidden_size))
1313
+ self.abstractor = MplugOwlVisualAbstractorModel(config.visual_abstractor_config, config.text_config.hidden_size)
1314
+
1315
+ # if config.use_decoder_only_language_model:
1316
+ # from llama.modeling_llama import LlamaForCausalLM
1317
+ language_model = AutoModelForCausalLM.from_config(config.text_config)
1318
+ # else:
1319
+ # language_model = AutoModelForSeq2SeqLM.from_config(config.text_config)
1320
+ self.language_model = language_model
1321
+
1322
+ # Initialize weights and apply final processing
1323
+ self.post_init()
1324
+ self.main_input_name = "input_ids"
1325
+ from transformers import GenerationConfig
1326
+
1327
+ self.generation_config = GenerationConfig(max_length=512, do_sample=True, top_k=3, pad_token_id=0, unk_token_id=0, bos_token_id=1, eos_token_id=2)
1328
+
1329
+ # Hack Bloom
1330
+ if config.text_config.model_type == "bloom":
1331
+ bound_method = bloom_forward.__get__(self.language_model.transformer, self.language_model.transformer.__class__)
1332
+ setattr(self.language_model.transformer, "forward", bound_method)
1333
+
1334
+ def get_input_embeddings(self):
1335
+ return self.language_model.get_input_embeddings()
1336
+
1337
+ def set_input_embeddings(self, value):
1338
+ self.language_model.set_input_embeddings(value)
1339
+
1340
+ def set_output_embeddings(self, new_embeddings):
1341
+ self.language_model.set_output_embeddings(new_embeddings)
1342
+
1343
+ def get_output_embeddings(self) -> nn.Module:
1344
+ return self.language_model.get_output_embeddings()
1345
+
1346
+ def get_encoder(self):
1347
+ return self.language_model.get_encoder()
1348
+
1349
+ def get_decoder(self):
1350
+ return self.language_model.get_decoder()
1351
+
1352
+ def _tie_weights(self):
1353
+ if not self.config.use_decoder_only_language_model:
1354
+ self.language_model.encoder.embed_tokens = self.language_model.shared
1355
+ self.language_model.decoder.embed_tokens = self.language_model.shared
1356
+
1357
+ def _preprocess_accelerate(self):
1358
+ r"""
1359
+ Some pre-processing hacks to make the model `accelerate` compatible. Check
1360
+ https://github.com/huggingface/transformers/pull/21707 for more details.
1361
+ """
1362
+ hf_device_map = self.hf_device_map
1363
+
1364
+ if len(hf_device_map) > 1 and "language_model" not in hf_device_map and torch.cuda.device_count() > 1:
1365
+ # warn users about unexpected behavior when using multi-GPU + mPLUG-Owl + `accelerate`.
1366
+ logger.warning(
1367
+ "The `language_model` is not in the `hf_device_map` dictionary and you are running your script"
1368
+ " in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`."
1369
+ " Please pass a `device_map` that contains `language_model` to remove this warning."
1370
+ " Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for"
1371
+ " more details on creating a `device_map` for large models.",
1372
+ )
1373
+
1374
+ if hasattr(self.language_model, "_hf_hook"):
1375
+ self.language_model._hf_hook.io_same_device = True # For `generate` compatibility
1376
+
1377
+ @add_start_docstrings_to_model_forward(MPLUG_OWL_INPUTS_DOCSTRING)
1378
+ @replace_return_docstrings(output_type=MplugOwlForConditionalGenerationModelOutput, config_class=MplugOwlVisionConfig)
1379
+ def forward(
1380
+ self,
1381
+ pixel_values: torch.FloatTensor,
1382
+ video_pixel_values: torch.FloatTensor,
1383
+ input_ids: torch.FloatTensor,
1384
+ num_images,
1385
+ num_videos,
1386
+ non_padding_mask: Optional[torch.LongTensor] = None,
1387
+ non_media_mask: Optional[torch.LongTensor] = None,
1388
+ prompt_mask: Optional[torch.LongTensor] = None,
1389
+ attention_mask: Optional[torch.LongTensor] = None,
1390
+ decoder_input_ids: Optional[torch.LongTensor] = None,
1391
+ decoder_attention_mask: Optional[torch.LongTensor] = None,
1392
+ output_attentions: Optional[bool] = None,
1393
+ output_hidden_states: Optional[bool] = None,
1394
+ labels: Optional[torch.LongTensor] = None,
1395
+ return_dict: Optional[bool] = None,
1396
+ ) -> Union[Tuple, MplugOwlForConditionalGenerationModelOutput]:
1397
+ r"""
1398
+ Returns:
1399
+
1400
+ Examples:
1401
+
1402
+ Image captioning (without providing a text prompt):
1403
+
1404
+ ```python
1405
+ >>> from PIL import Image
1406
+ >>> import requests
1407
+ >>> from transformers import MplugOwlProcessor, MplugOwlForConditionalGeneration
1408
+ >>> import torch
1409
+
1410
+ >>> device = "cuda" if torch.cuda.is_available() else "cpu"
1411
+
1412
+ >>> processor = MplugOwlProcessor.from_pretrained("x-plug/x_plug-llama-7b")
1413
+ >>> model = MplugOwlForConditionalGeneration.from_pretrained(
1414
+ ... "x-plug/x_plug-llama-7b", torch_dtype=torch.float16
1415
+ ... )
1416
+ >>> model.to(device) # doctest: +IGNORE_RESULT
1417
+
1418
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1419
+ >>> image = Image.open(requests.get(url, stream=True).raw)
1420
+
1421
+ >>> inputs = processor(images=image, return_tensors="pt").to(device, torch.float16)
1422
+
1423
+ >>> generated_ids = model.generate(**inputs)
1424
+ >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
1425
+ >>> print(generated_text)
1426
+ two cats laying on a couch
1427
+ ```
1428
+
1429
+ Visual question answering (prompt = question):
1430
+
1431
+ ```python
1432
+ >>> from PIL import Image
1433
+ >>> import requests
1434
+ >>> from transformers import MplugOwlProcessor, MplugOwlForConditionalGeneration
1435
+ >>> import torch
1436
+
1437
+ >>> device = "cuda" if torch.cuda.is_available() else "cpu"
1438
+
1439
+ >>> processor = MplugOwlProcessor.from_pretrained("x-plug/x_plug-llama-7b")
1440
+ >>> model = MplugOwlForConditionalGeneration.from_pretrained(
1441
+ ... "x-plug/x_plug-llama-7b", torch_dtype=torch.float16
1442
+ ... )
1443
+ >>> model.to(device) # doctest: +IGNORE_RESULT
1444
+
1445
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1446
+ >>> image = Image.open(requests.get(url, stream=True).raw)
1447
+
1448
+ >>> prompt = "Question: how many cats are there? Answer:"
1449
+ >>> inputs = processor(images=image, text=prompt, return_tensors="pt").to(device, torch.float16)
1450
+
1451
+ >>> generated_ids = model.generate(**inputs)
1452
+ >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
1453
+ >>> print(generated_text)
1454
+ two
1455
+ ```"""
1456
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1457
+
1458
+ # get text embedding
1459
+ text_tokens_ = input_ids.clone()
1460
+ batch_size = input_ids.shape[0]
1461
+ # labels = text_tokens_[:, 1:].clone().contiguous()
1462
+
1463
+ media_token_indices = [
1464
+ # [:-1] since we would not use the last token for embedding
1465
+ get_media_indices(text_tokens_[i][:-1])
1466
+ for i in range(batch_size)
1467
+ ]
1468
+
1469
+ media_token_types = [get_media_types(text_tokens_[i][:-1], media_token_indices[i]) for i in range(batch_size)]
1470
+
1471
+ text_tokens_[text_tokens_ < 0] = 1 # Not used
1472
+ # text_tokens = text_tokens_[:, :-1].contiguous()
1473
+ text_embeds = self.get_input_embeddings()(text_tokens_) # Temporally Embedding
1474
+
1475
+ if pixel_values is not None:
1476
+ image_embeds = self.vision_model(pixel_values, return_dict=True).last_hidden_state
1477
+
1478
+ image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
1479
+ query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
1480
+ temporal_query_tokens = self.temporal_query_tokens.expand(image_embeds.shape[0], -1, -1)
1481
+
1482
+ query_features = self.abstractor(
1483
+ query_embeds=query_tokens,
1484
+ encoder_hidden_states=image_embeds,
1485
+ encoder_attention_mask=image_attention_mask,
1486
+ )["last_hidden_state"]
1487
+ img_seq_length = query_features.shape[1]
1488
+
1489
+ if video_pixel_values is not None:
1490
+ video_embeds = self.vision_model(video_pixel_values, return_dict=True).last_hidden_state
1491
+
1492
+ video_attention_mask = torch.ones(video_embeds.size()[:-1], dtype=torch.long, device=video_embeds.device)
1493
+ video_attention_mask = einops.rearrange(video_attention_mask, "b t n -> b (t n)")
1494
+ query_tokens = self.query_tokens.expand(video_embeds.shape[0], -1, -1)
1495
+ temporal_query_tokens = self.temporal_query_tokens.expand(video_embeds.shape[0], -1, -1)
1496
+
1497
+ video_query_features = self.abstractor(
1498
+ query_embeds=query_tokens,
1499
+ temporal_query_embeds=temporal_query_tokens,
1500
+ encoder_hidden_states=video_embeds,
1501
+ encoder_attention_mask=video_attention_mask,
1502
+ )["last_hidden_state"]
1503
+ vid_seq_length = video_query_features.shape[1]
1504
+
1505
+ num_images_per_sample = num_images.long().cpu().tolist()
1506
+ num_videos_per_sample = num_videos.long().cpu().tolist()
1507
+
1508
+ text_chunk_embeds = []
1509
+ img_idx = 0
1510
+ for b in range(batch_size):
1511
+ start = 0
1512
+ result = []
1513
+ if len(media_token_indices[b]) > 0:
1514
+ for i, pos in enumerate(media_token_indices[b]):
1515
+ if pos > start:
1516
+ result.append(text_embeds[b, start:pos])
1517
+ result.append(query_features[img_idx + i])
1518
+ start = pos + img_seq_length
1519
+ if start < text_embeds.shape[1]:
1520
+ result.append(text_embeds[b, start:])
1521
+
1522
+ img_idx += num_images_per_sample[b]
1523
+ text_chunk_embeds.append(torch.cat(result, dim=0))
1524
+
1525
+ # Actual Input Embeddings
1526
+ input_embeds = torch.stack(text_chunk_embeds, dim=0)
1527
+
1528
+ # if pixel_values is None and self.language_model.is_gradient_checkpointing:
1529
+ # # Hack here when gradient checkpoint is enable.
1530
+ # # Keep the compute graph static
1531
+ # image_embeds = self.vision_model(torch.zeros(1,3,224,224,device=input_embeds.device,dtype=input_embeds.dtype), return_dict=True).last_hidden_state
1532
+ # query_tokens = self.query_tokens.expand(
1533
+ # image_embeds.shape[0], -1, -1)
1534
+ # query_features = self.abstractor(query_embeds=query_tokens,
1535
+ # encoder_hidden_states=image_embeds,)['last_hidden_state']
1536
+
1537
+ # input_embeds = input_embeds + query_features.mean()*0
1538
+
1539
+ # Create causal mask and position ids
1540
+ _, loss_mask, position_ids = get_ltor_masks_and_position_ids_from_embeddings(input_embeds)
1541
+
1542
+ # Calculate the loss_mask
1543
+ non_padding_mask = non_padding_mask.long()
1544
+ non_media_mask = non_media_mask.long()
1545
+ prompt_mask = prompt_mask.long() # TODO How to deal with prompt mask
1546
+ # from icecream import ic
1547
+ # non_padding_mask = non_padding_mask[:,:-1]
1548
+ # non_media_mask = non_media_mask[:,:-1]
1549
+ # prompt_mask = prompt_mask[:,:-1]
1550
+ # attention_mask = attention_mask[:,:-1]
1551
+ loss_mask = loss_mask[:, :-1]
1552
+
1553
+ loss_mask = loss_mask * non_padding_mask * non_media_mask * prompt_mask
1554
+ labels[:, 1:][loss_mask != 1] = -100
1555
+ # Forward into GPT
1556
+ outputs = self.language_model(
1557
+ inputs_embeds=input_embeds,
1558
+ attention_mask=attention_mask,
1559
+ labels=labels,
1560
+ return_dict=return_dict,
1561
+ output_attentions=self.config.output_attentions,
1562
+ )
1563
+ # outputs.loss = (outputs.loss * loss_mask.view(-1)
1564
+ # ).sum()/loss_mask.sum()
1565
+ return outputs
1566
+
1567
+ @torch.no_grad()
1568
+ def generate(
1569
+ self,
1570
+ pixel_values: torch.FloatTensor = None,
1571
+ video_pixel_values: torch.FloatTensor = None,
1572
+ input_ids: Optional[torch.LongTensor] = None,
1573
+ attention_mask: Optional[torch.LongTensor] = None,
1574
+ isdecoder=True,
1575
+ **generate_kwargs,
1576
+ ) -> torch.LongTensor:
1577
+ """
1578
+ Overrides `generate` function to be able to use the model as a conditional generator.
1579
+
1580
+ Args:
1581
+ pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width)):
1582
+ Input images to be processed.
1583
+ input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
1584
+ The sequence used as a prompt for the generation.
1585
+ attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
1586
+ Mask to avoid performing attention on padding token indices
1587
+
1588
+ Returns:
1589
+ captions (list): A list of strings of length batch_size * num_captions.
1590
+ """
1591
+ if input_ids is None:
1592
+ return self.language_model.generate(attention_mask=attention_mask, **generate_kwargs)
1593
+
1594
+ if attention_mask is None:
1595
+ attention_mask = input_ids.new_ones(*input_ids.shape)
1596
+
1597
+ batch_size = input_ids.size(0)
1598
+ media_token_indices = [get_media_indices(input_ids[i]) for i in range(batch_size)]
1599
+ media_token_types = [get_media_types(input_ids[i], media_token_indices[i]) for i in range(batch_size)]
1600
+ num_images_per_sample = [len([y for y in x if y == -1]) for x in media_token_types]
1601
+ num_videos_per_sample = [len([y for y in x if y < -1]) for x in media_token_types]
1602
+ input_ids = input_ids.clone() # prevent inplace modify
1603
+ input_ids[input_ids < 0] = 0 # Not used
1604
+
1605
+ if hasattr(self, "hf_device_map"):
1606
+ # preprocess for `accelerate`
1607
+ self._preprocess_accelerate()
1608
+ batch_size = input_ids.shape[0]
1609
+ # get text embedding
1610
+ inputs_embeds = self.get_input_embeddings()(input_ids)
1611
+ if hasattr(self.language_model, "transformer") and hasattr(self.language_model.transformer, "word_embeddings_layernorm"):
1612
+ inputs_embeds = self.language_model.transformer.word_embeddings_layernorm(inputs_embeds)
1613
+ # get visual embedding
1614
+ if pixel_values is not None:
1615
+ pixel_values = pixel_values.to(input_ids.device)
1616
+ with torch.no_grad():
1617
+ image_embeds = self.vision_model(pixel_values, return_dict=True).last_hidden_state
1618
+ image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
1619
+ query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
1620
+ query_outputs = self.abstractor(
1621
+ query_embeds=query_tokens,
1622
+ encoder_hidden_states=image_embeds,
1623
+ encoder_attention_mask=image_attention_mask,
1624
+ return_dict=True,
1625
+ )
1626
+ query_output = query_outputs["last_hidden_state"]
1627
+ image_embeds = query_output
1628
+ img_seq_length = image_embeds.shape[1]
1629
+
1630
+ if video_pixel_values is not None:
1631
+ video_pixel_values = video_pixel_values.to(input_ids.device)
1632
+ with torch.no_grad():
1633
+ video_embeds = self.vision_model(video_pixel_values, return_dict=True).last_hidden_state
1634
+ video_attention_mask = torch.ones(video_embeds.size()[:-1], dtype=torch.long, device=video_embeds.device)
1635
+ video_attention_mask = einops.rearrange(video_attention_mask, "b t n -> b (t n)")
1636
+ query_tokens = self.query_tokens.expand(video_embeds.shape[0], -1, -1)
1637
+ temporal_query_tokens = self.temporal_query_tokens.expand(video_embeds.shape[0], -1, -1)
1638
+ query_outputs = self.abstractor(
1639
+ query_embeds=query_tokens,
1640
+ temporal_query_embeds=temporal_query_tokens,
1641
+ encoder_hidden_states=video_embeds,
1642
+ encoder_attention_mask=video_attention_mask,
1643
+ return_dict=True,
1644
+ )
1645
+ query_output = query_outputs["last_hidden_state"]
1646
+ video_embeds = query_output
1647
+ vid_seq_length = video_embeds.shape[1]
1648
+
1649
+ # ===================
1650
+ # Get actual input embeddings
1651
+ # ===================
1652
+ text_chunk_embeds = []
1653
+ text_chunk_attns = []
1654
+ img_idx = 0
1655
+ vid_idx = 0
1656
+
1657
+ for b in range(batch_size):
1658
+ start = 0
1659
+ result = []
1660
+ result_attn = []
1661
+ for i, pos in enumerate(media_token_indices[b]):
1662
+ curr_image_idx, curr_video_idx = 0, 0
1663
+ if pos > start:
1664
+ result.append(inputs_embeds[b, start:pos])
1665
+ result_attn.append(attention_mask[b, start:pos])
1666
+ if media_token_types[b][i] == -1:
1667
+ result.append(image_embeds[img_idx + curr_image_idx])
1668
+ result_attn.append(torch.ones(image_embeds[img_idx + curr_image_idx].shape[0], device=inputs_embeds.device))
1669
+ start = pos + img_seq_length
1670
+ curr_image_idx += 1
1671
+ else:
1672
+ result.append(video_embeds[vid_idx + curr_video_idx])
1673
+ result_attn.append(torch.ones(video_embeds[img_idx + curr_video_idx].shape[0], device=inputs_embeds.device))
1674
+ start = pos + vid_seq_length
1675
+ curr_video_idx += 1
1676
+ if start < inputs_embeds.shape[1]:
1677
+ result.append(inputs_embeds[b, start:])
1678
+ result_attn.append(attention_mask[b, start:])
1679
+
1680
+ img_idx += num_images_per_sample[b]
1681
+ vid_idx += num_videos_per_sample[b]
1682
+ text_chunk_embeds.append(torch.cat(result, dim=0))
1683
+ text_chunk_attns.append(torch.cat(result_attn, dim=0))
1684
+ inputs_embeds = torch.stack(text_chunk_embeds, dim=0)
1685
+ attention_mask = torch.stack(text_chunk_attns, dim=0)
1686
+
1687
+ outputs = self.language_model.generate(
1688
+ inputs_embeds=inputs_embeds,
1689
+ # input_ids=input_ids,
1690
+ attention_mask=attention_mask,
1691
+ **generate_kwargs,
1692
+ )
1693
+
1694
+ return outputs
1695
+
1696
+ def prepare_inputs_for_generation(self, input_ids, pixel_values=None, video_pixel_values=None, past_key_values=None, attention_mask=None, **model_kwargs):
1697
+ input_shape = input_ids.shape
1698
+ # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
1699
+ if attention_mask is None:
1700
+ attention_mask = input_ids.new_ones(input_shape)
1701
+
1702
+ # # cut decoder_input_ids if past_key_values is used
1703
+ # if past_key_values is not None:
1704
+ # input_ids = input_ids[:, -1:]
1705
+
1706
+ return {
1707
+ "input_ids": input_ids,
1708
+ "pixel_values": pixel_values,
1709
+ "video_pixel_values": video_pixel_values,
1710
+ "attention_mask": attention_mask,
1711
+ # "past_key_values": past_key_values,
1712
+ # "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None),
1713
+ # "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None),
1714
+ "is_decoder": True,
1715
+ }
1716
+
1717
+
1718
+ def bloom_forward(
1719
+ self,
1720
+ input_ids: Optional[torch.LongTensor] = None,
1721
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
1722
+ attention_mask: Optional[torch.Tensor] = None,
1723
+ head_mask: Optional[torch.LongTensor] = None,
1724
+ inputs_embeds: Optional[torch.LongTensor] = None,
1725
+ use_cache: Optional[bool] = None,
1726
+ output_attentions: Optional[bool] = None,
1727
+ output_hidden_states: Optional[bool] = None,
1728
+ return_dict: Optional[bool] = None,
1729
+ **deprecated_arguments,
1730
+ ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
1731
+ if deprecated_arguments.pop("position_ids", False) is not False:
1732
+ # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None`
1733
+ warnings.warn(
1734
+ "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore" " passing `position_ids`.",
1735
+ FutureWarning,
1736
+ )
1737
+ if len(deprecated_arguments) > 0:
1738
+ raise ValueError(f"Got unexpected arguments: {deprecated_arguments}")
1739
+
1740
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1741
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1742
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
1743
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1744
+
1745
+ if input_ids is not None and inputs_embeds is not None:
1746
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
1747
+ elif input_ids is not None:
1748
+ batch_size, seq_length = input_ids.shape
1749
+ elif inputs_embeds is not None:
1750
+ batch_size, seq_length, _ = inputs_embeds.shape
1751
+ else:
1752
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
1753
+
1754
+ if past_key_values is None:
1755
+ past_key_values = tuple([None] * len(self.h))
1756
+
1757
+ # Prepare head mask if needed
1758
+ # 1.0 in head_mask indicate we keep the head
1759
+ # attention_probs has shape batch_size x num_heads x N x N
1760
+ # head_mask has shape n_layer x batch x num_heads x N x N
1761
+ head_mask = self.get_head_mask(head_mask, self.config.n_layer)
1762
+
1763
+ if inputs_embeds is None:
1764
+ inputs_embeds = self.word_embeddings(input_ids)
1765
+ inputs_embeds = self.word_embeddings_layernorm(inputs_embeds)
1766
+
1767
+ hidden_states = inputs_embeds
1768
+
1769
+ presents = () if use_cache else None
1770
+ all_self_attentions = () if output_attentions else None
1771
+ all_hidden_states = () if output_hidden_states else None
1772
+
1773
+ if self.gradient_checkpointing and self.training:
1774
+ if use_cache:
1775
+ logger.warning_once("`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...")
1776
+ use_cache = False
1777
+
1778
+ # Compute alibi tensor: check build_alibi_tensor documentation
1779
+ seq_length_with_past = seq_length
1780
+ past_key_values_length = 0
1781
+ if past_key_values[0] is not None:
1782
+ past_key_values_length = past_key_values[0][0].shape[2]
1783
+ seq_length_with_past = seq_length_with_past + past_key_values_length
1784
+ if attention_mask is None:
1785
+ attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
1786
+ else:
1787
+ attention_mask = attention_mask.to(hidden_states.device)
1788
+
1789
+ alibi = self.build_alibi_tensor(attention_mask, self.num_heads, dtype=hidden_states.dtype)
1790
+
1791
+ causal_mask = self._prepare_attn_mask(
1792
+ attention_mask,
1793
+ input_shape=(batch_size, seq_length),
1794
+ past_key_values_length=past_key_values_length,
1795
+ )
1796
+
1797
+ for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
1798
+ if output_hidden_states:
1799
+ all_hidden_states = all_hidden_states + (hidden_states,)
1800
+
1801
+ if self.gradient_checkpointing and self.training:
1802
+
1803
+ def create_custom_forward(module):
1804
+ def custom_forward(*inputs):
1805
+ # None for past_key_value
1806
+ return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)
1807
+
1808
+ return custom_forward
1809
+
1810
+ outputs = torch.utils.checkpoint.checkpoint(
1811
+ create_custom_forward(block),
1812
+ hidden_states,
1813
+ alibi,
1814
+ causal_mask,
1815
+ layer_past,
1816
+ head_mask[i],
1817
+ )
1818
+ else:
1819
+ outputs = block(
1820
+ hidden_states,
1821
+ layer_past=layer_past,
1822
+ attention_mask=causal_mask,
1823
+ head_mask=head_mask[i],
1824
+ use_cache=use_cache,
1825
+ output_attentions=output_attentions,
1826
+ alibi=alibi,
1827
+ )
1828
+
1829
+ hidden_states = outputs[0]
1830
+ if use_cache is True:
1831
+ presents = presents + (outputs[1],)
1832
+
1833
+ if output_attentions:
1834
+ all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
1835
+
1836
+ # Add last hidden state
1837
+ hidden_states = self.ln_f(hidden_states)
1838
+
1839
+ if output_hidden_states:
1840
+ all_hidden_states = all_hidden_states + (hidden_states,)
1841
+
1842
+ if not return_dict:
1843
+ return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
1844
+
1845
+ return BaseModelOutputWithPastAndCrossAttentions(
1846
+ last_hidden_state=hidden_states,
1847
+ past_key_values=presents,
1848
+ hidden_states=all_hidden_states,
1849
+ attentions=all_self_attentions,
1850
+ )
LaViDa/eval/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+
3
+ import numpy as np
4
+ import torch
5
+ import torch.utils.checkpoint
6
+ from decord import VideoReader
7
+ from PIL import Image
8
+ from transformers.models.clip.image_processing_clip import CLIPImageProcessor
9
+ from transformers.processing_utils import ProcessorMixin
10
+ from transformers.tokenization_utils_base import BatchEncoding
11
+
12
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
13
+
14
+ from .tokenization_mplug_owl import MplugOwlTokenizer
15
+
16
+
17
+ def get_index(num_frames, num_segments):
18
+ seg_size = float(num_frames - 1) / num_segments
19
+ start = int(seg_size / 2)
20
+ offsets = np.array([start + int(np.round(seg_size * idx)) for idx in range(num_segments)])
21
+ return offsets
22
+
23
+
24
+ def load_video(path, num_frames=4):
25
+ """vr = VideoReader(path, height=224, width=224)
26
+ total_frames = len(vr)
27
+ frame_indices = get_index(total_frames, num_frames)
28
+ images_group = list()
29
+ for frame_index in frame_indices:
30
+ img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
31
+ images_group.append(img)
32
+ return images_group"""
33
+ # Change a bit here from the original code
34
+ # I use pyav instead of decord because it is much more safer
35
+ # The operations here are the same, we load video and return a list of PIL Image
36
+ # Load video frames
37
+ video_frames = read_video_pyav(path, num_frm=num_frames)
38
+ target_h, target_w = 224, 224
39
+ # If image shape is not as target, resize it
40
+ if video_frames.shape[-3] != target_h or video_frames.shape[-2] != target_w:
41
+ video_frames = torch.from_numpy(video_frames).permute(0, 3, 1, 2).float()
42
+ video_frames = torch.nn.functional.interpolate(video_frames, size=(target_h, target_w))
43
+ video_frames = video_frames.permute(0, 2, 3, 1).to(torch.uint8).numpy()
44
+ video_frames = [Image.fromarray(frame) for frame in video_frames]
45
+ if len(video_frames) > num_frames:
46
+ video_frames = video_frames[:num_frames]
47
+ return video_frames
48
+
49
+
50
+ class MplugOwlProcessor(ProcessorMixin):
51
+ attributes = []
52
+ tokenizer_class = "MplugOwlTokenizer"
53
+
54
+ def __init__(self, image_processor=None, tokenizer=None, **kwargs):
55
+ super().__init__(**kwargs)
56
+ self.tokens_to_generate = 0
57
+ self.image_processor = image_processor
58
+ self.tokenizer = tokenizer
59
+ self.add_BOS = True
60
+
61
+ def __call__(self, text=None, images=None, videos=None, num_frames=4, return_tensors=None, **kwargs):
62
+ if text is None and images is None:
63
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
64
+
65
+ if text is not None:
66
+ encoding = tokenize_prompts(
67
+ prompts=text,
68
+ tokens_to_generate=self.tokens_to_generate,
69
+ add_BOS=self.add_BOS,
70
+ tokenizer=self.tokenizer,
71
+ ignore_dist=True,
72
+ **kwargs,
73
+ )
74
+ # encoding = self.tokenizer(text, return_tensors=return_tensors, **kwargs)
75
+
76
+ if images is not None:
77
+ image_features = self.image_processor(images, return_tensors=return_tensors, **kwargs)
78
+
79
+ if videos is not None:
80
+ video_features = []
81
+ for video in videos:
82
+ video_frames = load_video(video, num_frames)
83
+ video_feature = self.image_processor(video_frames, return_tensors=return_tensors, **kwargs)["pixel_values"]
84
+ video_features.append(video_feature)
85
+ video_features = torch.stack(video_features, dim=0)
86
+ video_features = video_features.permute(0, 2, 1, 3, 4)
87
+
88
+ if text is not None and images is not None:
89
+ encoding["pixel_values"] = image_features.pixel_values
90
+ return encoding
91
+ if text is not None and videos is not None:
92
+ encoding["video_pixel_values"] = video_features
93
+ return encoding
94
+ elif text is not None:
95
+ return encoding
96
+ elif images is not None:
97
+ return BatchEncoding(data=dict(**image_features), tensor_type=return_tensors)
98
+ else:
99
+ return BatchEncoding(data=dict(video_pixel_values=video_pixel_values), tensor_type=return_tensors)
100
+
101
+ def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
102
+ """
103
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
104
+ refer to the docstring of this method for more information.
105
+ """
106
+ return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
107
+
108
+ def decode(self, skip_special_tokens=True, *args, **kwargs):
109
+ """
110
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
111
+ the docstring of this method for more information.
112
+ """
113
+ return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
114
+
115
+
116
+ class MplugOwlImageProcessor(CLIPImageProcessor):
117
+ pass
118
+
119
+
120
+ def detokenize_generations(tokens_gpu_tensor, lengths_gpu_tensor, return_segments, tokenizer):
121
+ """Detokenize the generated tokens."""
122
+
123
+ prompts_plus_generations = []
124
+ if return_segments:
125
+ prompts_plus_generations_segments = []
126
+
127
+ tokens = tokens_gpu_tensor.cpu().numpy().tolist()
128
+ lengths = lengths_gpu_tensor.cpu().numpy().tolist()
129
+ for sequence_tokens, length in zip(tokens, lengths):
130
+ sequence_tokens = sequence_tokens[:length]
131
+ prompts_plus_generations.append(tokenizer.detokenize(sequence_tokens))
132
+ if return_segments:
133
+ from tokenizers.decoders import Metaspace
134
+
135
+ if hasattr(tokenizer, "tokenizer"):
136
+ if isinstance(tokenizer.tokenizer.decoder, Metaspace):
137
+ words = tokenizer.tokenizer.decode(sequence_tokens)
138
+ else:
139
+ words = []
140
+ for token in sequence_tokens:
141
+ word = tokenizer.tokenizer.decoder[token]
142
+ word = bytearray([tokenizer.tokenizer.byte_decoder[c] for c in word]).decode("utf-8", errors="replace")
143
+ words.append(word)
144
+ prompts_plus_generations_segments.append(words)
145
+ else:
146
+ words = tokenizer.detokenize(sequence_tokens)
147
+ # else:
148
+ # words = []
149
+ # for token in sequence_tokens:
150
+ # word = tokenizer.tokenizer.decoder[token]
151
+ # word = bytearray(
152
+ # [tokenizer.tokenizer.byte_decoder[c] for c in word]).decode(
153
+ # 'utf-8', errors='replace')
154
+ # words.append(word)
155
+ prompts_plus_generations_segments.append(words)
156
+
157
+ if return_segments:
158
+ return tokens, prompts_plus_generations, prompts_plus_generations_segments
159
+
160
+ return tokens, prompts_plus_generations
161
+
162
+
163
+ def tokenize_prompts(prompts=None, tokens_to_generate=None, add_BOS=None, rank=0, tokenizer=None, ignore_dist=False, **kwargs):
164
+ """Tokenize prompts and make them avaiable on all ranks."""
165
+
166
+ # On all ranks set to None so we can pass them to functions
167
+ prompts_tokens_cuda_long_tensor = None
168
+ prompts_length_cuda_long_tensor = None
169
+
170
+ # On the specified rank, build the above.
171
+ attention_mask = None
172
+ if ignore_dist or torch.distributed.get_rank() == rank:
173
+ assert prompts is not None
174
+ assert tokens_to_generate is not None
175
+ # Tensor of tokens padded and their unpadded length.
176
+ prompts_tokens_cuda_long_tensor, prompts_length_cuda_long_tensor, attention_mask = _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS, tokenizer, **kwargs)
177
+ # We need the sizes of these tensors for the boradcast
178
+ [
179
+ prompts_tokens_cuda_long_tensor.size(0), # Batch size
180
+ prompts_tokens_cuda_long_tensor.size(1),
181
+ ] # Sequence lenght
182
+
183
+ return {
184
+ "input_ids": prompts_tokens_cuda_long_tensor,
185
+ "attention_mask": attention_mask,
186
+ # "prompt_length": prompts_length_cuda_long_tensor,
187
+ }
188
+
189
+
190
+ def _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS, tokenizer, **kwargs):
191
+ """Given a set of prompts and number of tokens to generate:
192
+ - tokenize prompts
193
+ - set the sequence length to be the max of length of prompts
194
+ plus the number of tokens we would like to generate
195
+ - pad all the sequences to this length so we can convert them
196
+ into a 2D tensor.
197
+ """
198
+
199
+ # Tokenize all the prompts.
200
+ # if add_BOS:
201
+ # prompts_tokens = [[tokenizer.bos] + tokenizer.tokenize(prompt)
202
+ # for prompt in prompts]
203
+ # else:
204
+ # prompts_tokens = [tokenizer.tokenize(prompt) for prompt in prompts]
205
+
206
+ prompts_tokens = [_tokenize_prompt(prompt, tokenizer, add_BOS, **kwargs) for prompt in prompts]
207
+
208
+ # Now we have a list of list of tokens which each list has a different
209
+ # size. We want to extend this list to:
210
+ # - incorporate the tokens that need to be generated
211
+ # - make all the sequences equal length.
212
+ # Get the prompts length.
213
+ prompts_length = [len(prompt_tokens) for prompt_tokens in prompts_tokens]
214
+ # Get the max prompts length.
215
+ max_prompt_len = max(prompts_length)
216
+ # Number of tokens in the each sample of the batch.
217
+ samples_length = max_prompt_len + tokens_to_generate
218
+ # Now update the list of list to be of the same size: samples_length.
219
+ for prompt_tokens, prompt_length in zip(prompts_tokens, prompts_length):
220
+ padding_size = samples_length - prompt_length
221
+ prompt_tokens.extend([tokenizer.eos_token_id] * padding_size)
222
+
223
+ # Now we are in a structured format, we can convert to tensors.
224
+ prompts_tokens_tensor = torch.LongTensor(prompts_tokens)
225
+ prompts_length_tensor = torch.LongTensor(prompts_length)
226
+ attention_mask = torch.zeros(prompts_tokens_tensor.shape[:2])
227
+ for i, l in enumerate(prompts_length_tensor):
228
+ attention_mask[i, :l] = 1
229
+ return prompts_tokens_tensor, prompts_length_tensor, attention_mask
230
+
231
+
232
+ def _tokenize_prompt(prompt, tokenizer, add_BOS=False, media_info={"<image>": 65, "<|video|>": 65}, **kwargs):
233
+ media_tokens = {k: -int(i + 1) for i, k in enumerate(media_info.keys())}
234
+ media_lengths = media_info.copy()
235
+
236
+ if add_BOS:
237
+ prompt_chunk = [tokenizer.bos_token_id]
238
+ else:
239
+ prompt_chunk = []
240
+
241
+ # Pure Text
242
+ if all([media_token not in prompt for media_token in media_tokens.keys()]):
243
+ enc_chunk = prompt_chunk + tokenizer(prompt, add_special_tokens=False, **kwargs)["input_ids"]
244
+
245
+ # Multi-Modal Text
246
+ else:
247
+ enc_chunk = prompt_chunk
248
+ pattern = "|".join(map(re.escape, list(media_tokens.keys())))
249
+ chunk_strs = re.split(f"({pattern})", prompt)
250
+ chunk_strs = [x for x in chunk_strs if len(x) > 0]
251
+ for idx, chunk_str in enumerate(chunk_strs):
252
+ if chunk_str in media_tokens:
253
+ enc_chunk += [media_tokens[chunk_str]] * media_lengths[chunk_str]
254
+ else:
255
+ tmp_chunk = tokenizer(chunk_str, add_special_tokens=False)["input_ids"]
256
+ # if idx < len(chunk_strs) - 1: # Last chunk should not have eos
257
+ # tmp_chunk += [tokenizer.eod_id]
258
+ enc_chunk += tmp_chunk
259
+ return enc_chunk
260
+
261
+
262
+ if __name__ == "__main__":
263
+ pass
LaViDa/eval/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 x-plug and The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """Tokenization classes for MplugOwl."""
16
+
17
+ from loguru import logger
18
+ from transformers.models.llama.tokenization_llama import LlamaTokenizer
19
+
20
+ VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
21
+
22
+ PRETRAINED_VOCAB_FILES_MAP = {
23
+ "vocab_file": {
24
+ "MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/vocab.txt",
25
+ },
26
+ }
27
+
28
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
29
+ "MAGAer13/mplug-owl-llama-7b": 2048,
30
+ }
31
+
32
+
33
+ class MplugOwlTokenizer(LlamaTokenizer):
34
+ def __init__(
35
+ self,
36
+ vocab_file,
37
+ unk_token="<unk>",
38
+ bos_token="<s>",
39
+ eos_token="</s>",
40
+ pad_token="<unk>",
41
+ sp_model_kwargs=None,
42
+ add_bos_token=False,
43
+ add_eos_token=False,
44
+ clean_up_tokenization_spaces=False,
45
+ **kwargs,
46
+ ):
47
+ super().__init__(
48
+ vocab_file,
49
+ unk_token,
50
+ bos_token,
51
+ eos_token,
52
+ pad_token,
53
+ sp_model_kwargs,
54
+ add_bos_token,
55
+ add_eos_token,
56
+ clean_up_tokenization_spaces,
57
+ **kwargs,
58
+ )
59
+ self.eod_id = self.eos_token_id
LaViDa/eval/lmms_eval/models/ola.py ADDED
@@ -0,0 +1,709 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ os.environ["LOWRES_RESIZE"] = "384x32"
4
+ os.environ["HIGHRES_BASE"] = "0x32"
5
+ os.environ["VIDEO_RESIZE"] = "0x64"
6
+ os.environ["VIDEO_MAXRES"] = "480"
7
+ os.environ["VIDEO_MINRES"] = "288"
8
+ os.environ["MAXRES"] = "1536"
9
+ os.environ["MINRES"] = "0"
10
+ os.environ["FORCE_NO_DOWNSAMPLE"] = "1"
11
+ os.environ["LOAD_VISION_EARLY"] = "1"
12
+ os.environ["PAD2STRIDE"] = "1"
13
+ os.environ["USE_SPEECH"] = "1"
14
+ import copy
15
+ import logging
16
+ from datetime import timedelta
17
+ from pathlib import Path
18
+ from typing import List, Optional, Tuple, Union
19
+
20
+ import librosa
21
+ import numpy as np
22
+ import PIL
23
+ import soundfile as sf
24
+ import torch
25
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
26
+ from accelerate.state import AcceleratorState
27
+ from decord import VideoReader, cpu
28
+ from tqdm import tqdm
29
+ from transformers import AutoConfig
30
+
31
+ from lmms_eval import utils
32
+ from lmms_eval.api.instance import Instance
33
+ from lmms_eval.api.model import lmms
34
+ from lmms_eval.api.registry import register_model
35
+ from lmms_eval.models.model_utils.audio_processing import downsample_audio
36
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
37
+
38
+ eval_logger = logging.getLogger("lmms-eval")
39
+
40
+ import sys
41
+
42
+ wd = Path(__file__).parent.parent.parent.parent.resolve()
43
+ sys.path.append(os.path.join(str(wd), "Ola"))
44
+
45
+ import whisper
46
+ from ola.constants import (
47
+ DEFAULT_IM_END_TOKEN,
48
+ DEFAULT_IM_START_TOKEN,
49
+ DEFAULT_IMAGE_TOKEN,
50
+ DEFAULT_SPEECH_TOKEN,
51
+ IMAGE_TOKEN_INDEX,
52
+ SPEECH_TOKEN_INDEX,
53
+ )
54
+ from ola.conversation import SeparatorStyle, conv_templates
55
+ from ola.datasets.preprocess import (
56
+ tokenizer_image_token,
57
+ tokenizer_speech_image_token,
58
+ tokenizer_speech_token,
59
+ )
60
+ from ola.mm_utils import (
61
+ KeywordsStoppingCriteria,
62
+ get_model_name_from_path,
63
+ process_anyres_highres_image,
64
+ process_anyres_video,
65
+ )
66
+ from ola.model.builder import load_pretrained_model
67
+
68
+ try:
69
+ from ola.model.language_model.ola_qwen import OlaConfigQwen
70
+
71
+ AutoConfig.register("ola_qwen", OlaConfigQwen)
72
+ except:
73
+ eval_logger.debug("")
74
+ from moviepy.video.io.VideoFileClip import VideoFileClip
75
+
76
+ if "USE_SPEECH" in os.environ:
77
+ USE_SPEECH = os.environ["USE_SPEECH"]
78
+ print("USE_SPEECH is set")
79
+ else:
80
+ USE_SPEECH = None
81
+
82
+
83
+ @register_model("ola")
84
+ class Ola(lmms):
85
+ """
86
+ How to run lmms-eval with Ola model:
87
+
88
+ 1. Install Ola:
89
+ https://github.com/Ola-Omni/Ola?tab=readme-ov-file#installation
90
+
91
+ 2. Download the pretrained weight from https://huggingface.co/THUdyh/Ola-7b
92
+ or skip this step to use the online weights directly
93
+
94
+ 3.Download audio encoder from https://huggingface.co/THUdyh/Ola_speech_encoders/tree/main
95
+ and put the weights large-v3.pt and BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt
96
+ under llms-eval repository (ensure your current directory can see large-v3.pt and BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt)
97
+
98
+ The path for the project should be like this:
99
+ Folder/contains/lmms-eval/and/Ola
100
+ ├── lmms-eval (current directory)
101
+ │ ├── lmms_eval/
102
+ │ ├── ...
103
+ │ ├── large-v3.pt
104
+ │ ├── BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt
105
+ ├── Ola
106
+ │ ├── ...
107
+
108
+ 4. Run the the command to start evaluate the modeL. For example:
109
+ ```bash
110
+ python3 -m accelerate.commands.launch \
111
+ --num_processes=8 \
112
+ -m lmms_eval \
113
+ --model ola\
114
+ --tasks mme \
115
+ --batch_size 1 \
116
+ --log_samples \
117
+ --log_samples_suffix mme_ola \
118
+ --output_path ./logs/
119
+ ```
120
+ """
121
+
122
+ def __init__(
123
+ self,
124
+ pretrained: str = "THUdyh/Ola-7b",
125
+ truncation: Optional[bool] = True,
126
+ device: Optional[str] = "cuda:0",
127
+ batch_size: Optional[Union[int, str]] = 1,
128
+ attn_implementation=(
129
+ "sdpa" if torch.__version__ >= "2.1.2" else "eager"
130
+ ), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
131
+ device_map="",
132
+ conv_template="qwen_1_5",
133
+ use_cache=True,
134
+ truncate_context=False,
135
+ max_frames_num: int = 64,
136
+ mm_resampler_type: str = "spatial_pool",
137
+ overwrite: bool = True,
138
+ video_decode_backend: str = "decord",
139
+ **kwargs,
140
+ ) -> None:
141
+ super().__init__()
142
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
143
+
144
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
145
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
146
+ if accelerator.num_processes > 1:
147
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
148
+ self.device_map = f"cuda:{accelerator.local_process_index}"
149
+ elif accelerator.num_processes == 1 and device_map == "auto":
150
+ self._device = torch.device(device)
151
+ self.device_map = device_map
152
+ else:
153
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
154
+ self.device_map = f"cuda:{accelerator.local_process_index}"
155
+
156
+ self.pretrained = pretrained
157
+ self.model_name = get_model_name_from_path(pretrained)
158
+ self.video_decode_backend = video_decode_backend
159
+ # self._config = AutoConfig.from_pretrained(self.pretrained)
160
+ self.overwrite = overwrite
161
+ self.mm_resampler_type = mm_resampler_type
162
+ self.max_frames_num = int(max_frames_num)
163
+ if self.overwrite == True:
164
+ overwrite_config = {}
165
+ overwrite_config["patchify_video_feature"] = False
166
+ overwrite_config["attn_implementation"] = attn_implementation
167
+
168
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
169
+
170
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, device=self.device_map)
171
+ else:
172
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(
173
+ pretrained,
174
+ None,
175
+ device_map=self.device_map,
176
+ )
177
+
178
+ self._config = self._model.config
179
+ self.model.to(self.device).eval().bfloat16()
180
+ self.model.tie_weights()
181
+ self.truncation = truncation
182
+ self.batch_size_per_gpu = int(batch_size)
183
+ self.conv_template = conv_template
184
+ self.use_cache = use_cache
185
+ self.truncate_context = truncate_context
186
+ if accelerator.num_processes > 1:
187
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
188
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
189
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
190
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
191
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
192
+ kwargs = {
193
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
194
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
195
+ }
196
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
197
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
198
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
199
+ self._model = accelerator.prepare(self.model)
200
+ else:
201
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
202
+ self.accelerator = accelerator
203
+ if self.accelerator.is_local_main_process:
204
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
205
+ self._rank = self.accelerator.local_process_index
206
+ self._world_size = self.accelerator.num_processes
207
+ elif accelerator.num_processes == 1 and device_map == "auto":
208
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
209
+ self._rank = 0
210
+ self._world_size = 1
211
+ else:
212
+ eval_logger.info(f"Using single device: {self._device}")
213
+ self._rank = 0
214
+ self._world_size = 1
215
+ self.accelerator = accelerator
216
+
217
+ @property
218
+ def config(self):
219
+ # return the associated transformers.AutoConfig for the given pretrained model.
220
+ return self._config
221
+
222
+ @property
223
+ def tokenizer(self):
224
+ return self._tokenizer
225
+
226
+ @property
227
+ def model(self):
228
+ # returns the model, unwrapping it if using Accelerate
229
+ if hasattr(self, "accelerator"):
230
+ return self.accelerator.unwrap_model(self._model)
231
+ else:
232
+ return self._model
233
+
234
+ @property
235
+ def eot_token_id(self):
236
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
237
+ return self.tokenizer.eos_token_id
238
+
239
+ @property
240
+ def max_length(self):
241
+ return self._max_length
242
+
243
+ def pad_sequence(self, input_ids, batch_first, padding_value):
244
+ if self.tokenizer.padding_side == "left":
245
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
246
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
247
+ if self.tokenizer.padding_side == "left":
248
+ input_ids = torch.flip(input_ids, [1])
249
+ return input_ids
250
+
251
+ @property
252
+ def batch_size(self):
253
+ return self.batch_size_per_gpu
254
+
255
+ @property
256
+ def device(self):
257
+ return self._device
258
+
259
+ @property
260
+ def rank(self):
261
+ return self._rank
262
+
263
+ @property
264
+ def world_size(self):
265
+ return self._world_size
266
+
267
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
268
+ """ """
269
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
270
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
271
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
272
+ if left_truncate_len:
273
+ encoding = encoding[-left_truncate_len:]
274
+ return encoding
275
+
276
+ def load_video(self, video_path, max_frames_num):
277
+ vr = VideoReader(video_path, ctx=cpu(0))
278
+ total_frame_num = len(vr)
279
+ fps = round(vr.get_avg_fps())
280
+
281
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
282
+ frame_idx = uniform_sampled_frames.tolist()
283
+
284
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
285
+ video = [PIL.Image.fromarray(frame) for frame in spare_frames]
286
+ return video, frame_idx
287
+
288
+ def tok_decode(self, tokens):
289
+ return self.tokenizer.decode(tokens)
290
+
291
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
292
+ res = []
293
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
294
+
295
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
296
+ # encode, pad, and truncate contexts for this batch
297
+ if type(doc_to_target) == str:
298
+ continuation = doc_to_target
299
+ else:
300
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
301
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
302
+ visuals = self.flatten(visuals)
303
+ videos = []
304
+ for visual in visuals:
305
+ video = self.load_video(visual, self.max_frames_num)
306
+ video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].bfloat16().to(self.device)
307
+ videos.append(video)
308
+
309
+ qs = contexts
310
+ if self.model.config.mm_use_im_start_end:
311
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
312
+ else:
313
+ qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
314
+
315
+ conv = conv_templates[self.conv_template].copy()
316
+ conv.append_message(conv.roles[0], qs)
317
+ conv.append_message(conv.roles[1], None)
318
+ prompt = conv.get_prompt()
319
+
320
+ contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
321
+
322
+ conv = conv_templates[self.conv_template].copy()
323
+ conv.append_message(conv.roles[0], qs)
324
+ conv.append_message(conv.roles[1], continuation)
325
+ prompt = conv.get_prompt()
326
+
327
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
328
+
329
+ labels = input_ids.clone()
330
+ # Context part no need to calculate for loss
331
+ labels[0, : contxt_id.shape[1]] = -100
332
+
333
+ with torch.inference_mode():
334
+ outputs = self.model(input_ids=input_ids, labels=labels, images=videos, modalities="video")
335
+
336
+ loss = outputs["loss"]
337
+ # loss = torch.exp(loss)
338
+ logits = outputs["logits"]
339
+ greedy_tokens = logits.argmax(dim=-1)
340
+ cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
341
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
342
+ max_equal = (greedy_tokens == cont_toks).all()
343
+ res.append((float(loss.item()), bool(max_equal)))
344
+ pbar.update(1)
345
+ pbar.close()
346
+ return res
347
+
348
+ def flatten(self, input):
349
+ new_list = []
350
+ for i in input:
351
+ for j in i:
352
+ new_list.append(j)
353
+ return new_list
354
+
355
+ def extract_audio(self, videos_file_path):
356
+ my_clip = VideoFileClip(videos_file_path)
357
+ return my_clip.audio
358
+
359
+ def load_audio(self, audio_file_name):
360
+ CHUNK_LIM = 480000
361
+ import librosa
362
+
363
+ audio, samplerate = librosa.load(audio_file_name, sr=16000)
364
+ audio = audio.astype(np.float32)
365
+
366
+ if len(audio.shape) > 1:
367
+ audio = audio[:, 0]
368
+
369
+ speechs = []
370
+ speech_wavs = []
371
+ if len(audio) <= CHUNK_LIM:
372
+ audio = whisper.pad_or_trim(audio)
373
+ speechs.append(audio)
374
+ speech_wavs.append(torch.from_numpy(audio).unsqueeze(0))
375
+ else:
376
+ for i in range(0, len(audio), CHUNK_LIM):
377
+ chunk = audio[i : i + CHUNK_LIM]
378
+ if len(chunk) < CHUNK_LIM:
379
+ chunk = whisper.pad_or_trim(chunk)
380
+ speechs.append(chunk)
381
+ speech_wavs.append(torch.from_numpy(chunk).unsqueeze(0))
382
+ mels = []
383
+ for chunk in speechs:
384
+ chunk = whisper.log_mel_spectrogram(chunk, n_mels=128).permute(1, 0).unsqueeze(0)
385
+ mels.append(chunk)
386
+
387
+ mels = torch.cat(mels, dim=0)
388
+ speech_wavs = torch.cat(speech_wavs, dim=0)
389
+ if mels.shape[0] > 20:
390
+ mels = mels[:20]
391
+ speech_wavs = speech_wavs[:20]
392
+
393
+ speech_length = torch.LongTensor([mels.shape[1]] * mels.shape[0])
394
+ speech_chunks = torch.LongTensor([mels.shape[0]])
395
+
396
+ return mels, speech_length, speech_chunks, speech_wavs
397
+
398
+ def process_audio(self, audio_array, sampling_rate):
399
+ """
400
+ Process audio array to format of Ola model
401
+ """
402
+ audio = audio_array.astype(np.float32)
403
+ if len(audio.shape) > 1:
404
+ audio = audio[:, 0]
405
+ target_sr = 16000
406
+ CHUNK_LIM = 480000
407
+
408
+ if sampling_rate != target_sr:
409
+ speech_wav = librosa.resample(audio_array, orig_sr=sampling_rate, target_sr=target_sr).astype(np.float32)
410
+ else:
411
+ speech_wav = audio_array.astype(np.float32)
412
+ speechs = []
413
+ speech_wavs = []
414
+
415
+ if len(speech_wav) <= CHUNK_LIM:
416
+ speech = whisper.pad_or_trim(speech_wav)
417
+ speech_wav = whisper.pad_or_trim(speech_wav)
418
+ speechs.append(speech)
419
+ speech_wavs.append(torch.from_numpy(speech_wav).unsqueeze(0))
420
+ else:
421
+ for i in range(0, len(speech_wav), CHUNK_LIM):
422
+ chunk = speech_wav[i : i + CHUNK_LIM]
423
+ if len(chunk) < CHUNK_LIM:
424
+ chunk = whisper.pad_or_trim(chunk)
425
+ speechs.append(chunk)
426
+ speech_wavs.append(torch.from_numpy(chunk).unsqueeze(0))
427
+ mels = []
428
+ for chunk in speechs:
429
+ chunk = whisper.log_mel_spectrogram(chunk, n_mels=128).permute(1, 0).unsqueeze(0)
430
+ mels.append(chunk)
431
+
432
+ mels = torch.cat(mels, dim=0)
433
+ speech_wavs = torch.cat(speech_wavs, dim=0)
434
+ if mels.shape[0] > 25:
435
+ mels = mels[:25]
436
+ speech_wavs = speech_wavs[:25]
437
+
438
+ speech_length = torch.LongTensor([mels.shape[1]] * mels.shape[0])
439
+ speech_chunks = torch.LongTensor([mels.shape[0]])
440
+ return mels, speech_length, speech_chunks, speech_wavs
441
+
442
+ def generate_until(self, requests) -> List[str]:
443
+ MODALITY = None
444
+ res = []
445
+
446
+ def _collate(x):
447
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
448
+ # - time estimates will always be over not underestimates, which is more useful for planning
449
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
450
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
451
+ # automatic adaptive batches much much easier to implement
452
+ # - any OOMs will happen right away rather than near the end
453
+ toks = self.tok_encode(x[0])
454
+ return -len(toks), x[0]
455
+
456
+ # we group requests by their generation_kwargs,
457
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
458
+ # in the same batch.
459
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
460
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
461
+ num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
462
+ pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
463
+ for chunk in chunks:
464
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
465
+ task = task[0]
466
+ split = split[0]
467
+ context = contexts[0]
468
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
469
+ visuals = self.flatten(visuals) # Len = 1. just an audio tho
470
+
471
+ speechs, speech_lengths, speech_wavs, speech_chunks = [], [], [], []
472
+ images, images_highres = [], [] # For dummy image passed in audio modality
473
+ image_sizes = []
474
+ image_tensor, image_highres_tensor = [], [] # For image
475
+ video_processed = [] # For video only
476
+ for visual in visuals:
477
+ if isinstance(visual, str): # For Video
478
+ if MODALITY is None:
479
+ MODALITY = "VIDEO"
480
+ # Process audio of video
481
+ try:
482
+ video, frame_idx = self.load_video(visual, self.max_frames_num)
483
+ except Exception as e:
484
+ eval_logger.info(f"{e}")
485
+ eval_logger.info(f"Video {visuals} can not load, check the source")
486
+ continue
487
+ audio = self.extract_audio(visual)
488
+ audio.write_audiofile(f"./video_audio_{self.rank}.wav")
489
+ video_audio_path = f"./video_audio_{self.rank}.wav"
490
+ speech, speech_length, speech_chunk, speech_wav = self.load_audio(video_audio_path)
491
+ speechs.append(speech.bfloat16().to(self.device))
492
+ speech_lengths.append(speech_length.to(self.device))
493
+ speech_chunks.append(speech_chunk.to(self.device))
494
+ speech_wavs.append(speech_wav.to(self.device))
495
+ os.remove(video_audio_path)
496
+
497
+ # Process images of video
498
+ for idx, frame in enumerate(video):
499
+ self._image_processor.do_resize = False
500
+ self._image_processor.do_center_crop = False
501
+ frame = process_anyres_video(frame, self._image_processor)
502
+
503
+ if frame_idx is not None and idx in frame_idx:
504
+ video_processed.append(frame.unsqueeze(0))
505
+ elif frame_idx is None:
506
+ video_processed.append(frame.unsqueeze(0))
507
+
508
+ if frame_idx is None:
509
+ frame_idx = np.arange(0, len(video_processed), dtype=int).tolist()
510
+
511
+ video_processed = torch.cat(video_processed, dim=0).bfloat16().to(self.device)
512
+ video_processed = (video_processed, video_processed)
513
+
514
+ video_data = (video_processed, (384, 384), "video")
515
+
516
+ elif isinstance(visual, PIL.Image.Image): # For Image
517
+ if MODALITY is None:
518
+ MODALITY = "IMAGE"
519
+ self._image_processor.do_resize = False
520
+ self._image_processor.do_center_crop = False
521
+ image_sizes.append(visual.size)
522
+ image_tensor_, image_highres_tensor_ = process_anyres_highres_image(visual, self._image_processor)
523
+ image_tensor.append(image_tensor_)
524
+ image_highres_tensor.append(image_highres_tensor_)
525
+
526
+ elif isinstance(visual, dict) and "array" in visual: # For Audio
527
+ if MODALITY is None:
528
+ MODALITY = "AUDIO"
529
+ mels, speech_length, speech_chunk, speech_wav = self.process_audio(visual["array"], visual["sampling_rate"])
530
+ speechs.append(mels.bfloat16().to(self.device))
531
+ speech_lengths.append(speech_length.to(self.device))
532
+ speech_chunks.append(speech_chunk.to(self.device))
533
+ speech_wavs.append(speech_wav.to(self.device))
534
+
535
+ # Processing dummy images, as required by model
536
+ images.append(torch.zeros(1, 3, 224, 224).to(dtype=torch.bfloat16, device=self.device, non_blocking=True))
537
+ images_highres.append(torch.zeros(1, 3, 224, 224).to(dtype=torch.bfloat16, device=self.device, non_blocking=True))
538
+ image_sizes.append((224, 224))
539
+
540
+ if not video_processed and MODALITY == "VIDEO":
541
+ # If video is not processed, skip the iteration
542
+ pbar.update(1)
543
+ continue
544
+
545
+ if MODALITY == "IMAGE":
546
+ if all(x.shape == image_tensor[0].shape for x in image_tensor):
547
+ image_tensor = torch.stack(image_tensor, dim=0)
548
+ if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor):
549
+ image_highres_tensor = torch.stack(image_highres_tensor, dim=0)
550
+ if type(image_tensor) is list:
551
+ image_tensor = [_image.bfloat16().to("cuda") for _image in image_tensor]
552
+ else:
553
+ image_tensor = image_tensor.bfloat16().to("cuda")
554
+ if type(image_highres_tensor) is list:
555
+ image_highres_tensor = [_image.bfloat16().to("cuda") for _image in image_highres_tensor]
556
+ else:
557
+ image_highres_tensor = image_highres_tensor.bfloat16().to("cuda")
558
+
559
+ # Processing dummy audio, as required by model
560
+ speechs.append(torch.zeros(1, 3000, 128).bfloat16().to("cuda"))
561
+ speech_lengths.append(torch.LongTensor([3000]).to("cuda"))
562
+ speech_wavs.append(torch.zeros([1, 480000]).to("cuda"))
563
+ speech_chunks.append(torch.LongTensor([1]).to("cuda"))
564
+
565
+ # we assume all gen kwargs in the batch are the same
566
+ # this is safe to assume because the `grouper` object ensures it.
567
+ gen_kwargs = all_gen_kwargs[0]
568
+
569
+ # Set default values for until and max_new_tokens
570
+ until = [self.tokenizer.decode(self.eot_token_id)]
571
+
572
+ # Update values from gen_kwargs if present
573
+ if "until" in gen_kwargs:
574
+ until = gen_kwargs.pop("until")
575
+ if isinstance(until, str):
576
+ until = [until]
577
+ elif not isinstance(until, list):
578
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
579
+ assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
580
+ # Okay be I am assuming bs always == 1
581
+ qs = list(contexts)[0]
582
+ if self.model.config.mm_use_im_start_end:
583
+ if MODALITY == "AUDIO":
584
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_SPEECH_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
585
+ elif MODALITY == "IMAGE":
586
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
587
+ elif MODALITY == "VIDEO":
588
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_SPEECH_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
589
+ else:
590
+ if MODALITY == "AUDIO":
591
+ qs = DEFAULT_SPEECH_TOKEN + "\n" + qs
592
+ elif MODALITY == "IMAGE":
593
+ qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
594
+ elif MODALITY == "VIDEO":
595
+ qs = DEFAULT_SPEECH_TOKEN + DEFAULT_IMAGE_TOKEN + "\n" + qs
596
+
597
+ conv = conv_templates[self.conv_template].copy()
598
+ conv.append_message(conv.roles[0], qs)
599
+ conv.append_message(conv.roles[1], None)
600
+ prompt = conv.get_prompt()
601
+
602
+ if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
603
+ eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{prompt}\n")
604
+
605
+ if MODALITY == "AUDIO":
606
+ input_ids = tokenizer_speech_token(prompt, self.tokenizer, SPEECH_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
607
+ elif MODALITY == "IMAGE":
608
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
609
+ elif MODALITY == "VIDEO":
610
+ input_ids = tokenizer_speech_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
611
+ pad_token_ids = 151643
612
+ attention_masks = input_ids.ne(pad_token_ids).long().to(self.device)
613
+
614
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
615
+ keywords = [stop_str]
616
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
617
+
618
+ if "max_new_tokens" not in gen_kwargs:
619
+ gen_kwargs["max_new_tokens"] = 256
620
+ if "temperature" not in gen_kwargs:
621
+ gen_kwargs["temperature"] = 0
622
+ if "top_p" not in gen_kwargs:
623
+ gen_kwargs["top_p"] = None
624
+ if "num_beams" not in gen_kwargs:
625
+ gen_kwargs["num_beams"] = 1
626
+
627
+ try:
628
+ with torch.inference_mode():
629
+ if MODALITY == "AUDIO":
630
+ output_ids = self.model.generate(
631
+ input_ids,
632
+ images=images,
633
+ images_highres=images_highres,
634
+ image_sizes=image_sizes,
635
+ modalities=["text"],
636
+ speech=speechs,
637
+ speech_lengths=speech_lengths,
638
+ speech_chunks=speech_chunks,
639
+ speech_wav=speech_wavs,
640
+ attention_mask=attention_masks,
641
+ use_cache=True,
642
+ stopping_criteria=[stopping_criteria],
643
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
644
+ temperature=gen_kwargs["temperature"],
645
+ top_p=gen_kwargs["top_p"],
646
+ num_beams=gen_kwargs["num_beams"],
647
+ max_new_tokens=gen_kwargs["max_new_tokens"],
648
+ )
649
+ elif MODALITY == "IMAGE":
650
+ output_ids = self.model.generate(
651
+ inputs=input_ids,
652
+ images=image_tensor,
653
+ images_highres=image_highres_tensor,
654
+ image_sizes=image_sizes,
655
+ modalities=["image"],
656
+ speech=speechs,
657
+ speech_lengths=speech_lengths,
658
+ speech_chunks=speech_chunks,
659
+ speech_wav=speech_wavs,
660
+ attention_mask=attention_masks,
661
+ use_cache=True,
662
+ stopping_criteria=[stopping_criteria],
663
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
664
+ temperature=gen_kwargs["temperature"],
665
+ top_p=gen_kwargs["top_p"],
666
+ num_beams=gen_kwargs["num_beams"],
667
+ max_new_tokens=gen_kwargs["max_new_tokens"],
668
+ )
669
+ elif MODALITY == "VIDEO":
670
+ output_ids = self.model.generate(
671
+ inputs=input_ids,
672
+ images=video_data[0][0],
673
+ images_highres=video_data[0][1],
674
+ modalities=video_data[2],
675
+ speech=speechs,
676
+ speech_lengths=speech_lengths,
677
+ speech_chunks=speech_chunks,
678
+ speech_wav=speech_wavs,
679
+ attention_mask=attention_masks,
680
+ use_cache=True,
681
+ stopping_criteria=[stopping_criteria],
682
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
683
+ temperature=gen_kwargs["temperature"],
684
+ top_p=gen_kwargs["top_p"],
685
+ num_beams=gen_kwargs["num_beams"],
686
+ max_new_tokens=gen_kwargs["max_new_tokens"],
687
+ )
688
+ except Exception as e:
689
+ eval_logger.error(f"Error {e} in generating")
690
+ outputs = ""
691
+ res.append(outputs)
692
+ pbar.update(1)
693
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), outputs)
694
+ continue
695
+ outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
696
+ if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
697
+ eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{outputs}\n")
698
+
699
+ res.append(outputs)
700
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), outputs)
701
+ pbar.update(1)
702
+ # reorder this group of results back to original unsorted form
703
+ res = re_ords.get_original(res)
704
+
705
+ pbar.close()
706
+ return res
707
+
708
+ def generate_until_multi_round(self, requests) -> List[str]:
709
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/openai_compatible.py ADDED
@@ -0,0 +1,225 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import json
3
+ import os
4
+ import time
5
+ from io import BytesIO
6
+ from typing import List, Tuple, Union
7
+
8
+ import numpy as np
9
+ import requests as url_requests
10
+ from accelerate import Accelerator, DistributedType
11
+ from tqdm import tqdm
12
+
13
+ from lmms_eval.api.instance import Instance
14
+ from lmms_eval.api.model import lmms
15
+ from lmms_eval.api.registry import register_model
16
+
17
+ try:
18
+ from decord import VideoReader, cpu
19
+ except ImportError:
20
+ pass
21
+
22
+ from dotenv import find_dotenv, load_dotenv
23
+ from loguru import logger as eval_logger
24
+ from openai import OpenAI
25
+ from PIL import Image
26
+
27
+ load_dotenv(verbose=True)
28
+
29
+
30
+ @register_model("openai_compatible")
31
+ class OpenAICompatible(lmms):
32
+ def __init__(
33
+ self,
34
+ model_version: str = "grok-2-latest",
35
+ timeout: int = 120,
36
+ max_retries: int = 5,
37
+ max_size_in_mb: int = 20,
38
+ continual_mode: bool = False,
39
+ response_persistent_folder: str = None,
40
+ **kwargs,
41
+ ) -> None:
42
+ super().__init__()
43
+ self.model_version = model_version
44
+ self.timeout = timeout
45
+ self.max_retries = max_retries
46
+ self.max_size_in_mb = max_size_in_mb # some models have a limit on the size of the image
47
+ self.continual_mode = continual_mode
48
+ if self.continual_mode:
49
+ if response_persistent_folder is None:
50
+ raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
51
+
52
+ os.makedirs(response_persistent_folder, exist_ok=True)
53
+ self.response_persistent_folder = response_persistent_folder
54
+ self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
55
+
56
+ if os.path.exists(self.response_persistent_file):
57
+ with open(self.response_persistent_file, "r") as f:
58
+ self.response_cache = json.load(f)
59
+ self.cache_mode = "resume"
60
+ else:
61
+ self.response_cache = {}
62
+ self.cache_mode = "start"
63
+
64
+ self.client = OpenAI(api_key=os.getenv("OPENAI_COMPATIBLE_API_KEY"), base_url=os.getenv("OPENAI_COMPATIBLE_API_URL"))
65
+
66
+ accelerator = Accelerator()
67
+ # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
68
+ if accelerator.num_processes > 1:
69
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
70
+ self.accelerator = accelerator
71
+ if self.accelerator.is_local_main_process:
72
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
73
+ self._rank = self.accelerator.local_process_index
74
+ self._world_size = self.accelerator.num_processes
75
+ else:
76
+ self.accelerator = accelerator
77
+ self._rank = self.accelerator.local_process_index
78
+ self._world_size = self.accelerator.num_processes
79
+
80
+ self.device = self.accelerator.device
81
+
82
+ # Function to encode the image
83
+ def encode_image(self, image: Union[Image.Image, str]):
84
+ max_size = self.max_size_in_mb * 1024 * 1024 # 20MB in bytes
85
+ if isinstance(image, str):
86
+ img = Image.open(image).convert("RGB")
87
+ else:
88
+ img = image.copy()
89
+
90
+ output_buffer = BytesIO()
91
+ img.save(output_buffer, format="PNG")
92
+ byte_data = output_buffer.getvalue()
93
+
94
+ # If image is too large, resize it while maintaining aspect ratio
95
+ while len(byte_data) > max_size and img.size[0] > 100 and img.size[1] > 100:
96
+ new_size = (int(img.size[0] * 0.75), int(img.size[1] * 0.75))
97
+ img = img.resize(new_size, Image.Resampling.LANCZOS)
98
+
99
+ output_buffer = BytesIO()
100
+ img.save(output_buffer, format="PNG")
101
+ byte_data = output_buffer.getvalue()
102
+
103
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
104
+ return base64_str
105
+
106
+ # Function to encode the video
107
+ def encode_video(self, video_path, for_get_frames_num):
108
+ vr = VideoReader(video_path, ctx=cpu(0))
109
+ total_frame_num = len(vr)
110
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int)
111
+
112
+ # Ensure the last frame is included
113
+ if total_frame_num - 1 not in uniform_sampled_frames:
114
+ uniform_sampled_frames = np.append(uniform_sampled_frames, total_frame_num - 1)
115
+
116
+ frame_idx = uniform_sampled_frames.tolist()
117
+ frames = vr.get_batch(frame_idx).asnumpy()
118
+
119
+ base64_frames = []
120
+ for frame in frames:
121
+ img = Image.fromarray(frame)
122
+ output_buffer = BytesIO()
123
+ img.save(output_buffer, format="PNG")
124
+ byte_data = output_buffer.getvalue()
125
+ base64_str = base64.b64encode(byte_data).decode("utf-8")
126
+ base64_frames.append(base64_str)
127
+
128
+ return base64_frames
129
+
130
+ def flatten(self, input):
131
+ new_list = []
132
+ for i in input:
133
+ for j in i:
134
+ new_list.append(j)
135
+ return new_list
136
+
137
+ def generate_until(self, requests) -> List[str]:
138
+ res = []
139
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
140
+
141
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
142
+ if self.continual_mode is True and self.cache_mode == "resume":
143
+ doc_uuid = f"{task}___{split}___{doc_id}"
144
+ if doc_uuid in self.response_cache:
145
+ response_text = self.response_cache[doc_uuid]
146
+ if response_text:
147
+ res.append(response_text)
148
+ pbar.update(1)
149
+ continue
150
+
151
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
152
+ if None in visuals:
153
+ visuals = []
154
+ imgs = []
155
+ else:
156
+ visuals = self.flatten(visuals)
157
+ imgs = [] # multiple images or frames for video
158
+ for visual in visuals:
159
+ if isinstance(visual, str) and (".mp4" in visual or ".avi" in visual or ".mov" in visual or ".flv" in visual or ".wmv" in visual):
160
+ frames = self.encode_video(visual, self.max_frames_num)
161
+ imgs.extend(frames)
162
+ elif isinstance(visual, str) and (".jpg" in visual or ".jpeg" in visual or ".png" in visual or ".gif" in visual or ".bmp" in visual or ".tiff" in visual or ".webp" in visual):
163
+ img = self.encode_image(visual)
164
+ imgs.append(img)
165
+ elif isinstance(visual, Image.Image):
166
+ img = self.encode_image(visual)
167
+ imgs.append(img)
168
+
169
+ payload = {"messages": []}
170
+ payload["model"] = self.model_version
171
+
172
+ # When there is no image token in the context, append the image to the text
173
+ payload["messages"].append({"role": "user", "content": []})
174
+ payload["messages"][0]["content"].append({"type": "text", "text": contexts})
175
+ for img in imgs:
176
+ payload["messages"][0]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}})
177
+
178
+ if "max_new_tokens" not in gen_kwargs:
179
+ gen_kwargs["max_new_tokens"] = 1024
180
+ if gen_kwargs["max_new_tokens"] > 4096:
181
+ gen_kwargs["max_new_tokens"] = 4096
182
+ if "temperature" not in gen_kwargs:
183
+ gen_kwargs["temperature"] = 0
184
+ if "top_p" not in gen_kwargs:
185
+ gen_kwargs["top_p"] = None
186
+ if "num_beams" not in gen_kwargs:
187
+ gen_kwargs["num_beams"] = 1
188
+
189
+ payload["max_tokens"] = gen_kwargs["max_new_tokens"]
190
+ payload["temperature"] = gen_kwargs["temperature"]
191
+
192
+ for attempt in range(self.max_retries):
193
+ try:
194
+ response = self.client.chat.completions.create(**payload)
195
+ response_text = response.choices[0].message.content
196
+ break # If successful, break out of the loop
197
+
198
+ except Exception as e:
199
+ error_msg = str(e)
200
+ eval_logger.info(f"Attempt {attempt + 1}/{self.max_retries} failed with error: {error_msg}")
201
+
202
+ # On last attempt, log error and set empty response
203
+ if attempt == self.max_retries - 1:
204
+ eval_logger.error(f"All {self.max_retries} attempts failed. Last error: {error_msg}")
205
+ response_text = ""
206
+ else:
207
+ time.sleep(self.timeout)
208
+
209
+ res.append(response_text)
210
+ pbar.update(1)
211
+
212
+ if self.continual_mode is True: # Cache the response
213
+ doc_uuid = f"{task}___{split}___{doc_id}"
214
+ self.response_cache[doc_uuid] = response_text
215
+ with open(self.response_persistent_file, "w") as f:
216
+ json.dump(self.response_cache, f)
217
+
218
+ pbar.close()
219
+ return res
220
+
221
+ def generate_until_multi_round(self, requests) -> List[str]:
222
+ raise NotImplementedError("TODO: Implement multi-round generation for OpenAI compatible models")
223
+
224
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
225
+ raise NotImplementedError("TODO: Implement loglikelihood for OpenAI compatible models")
LaViDa/eval/lmms_eval/models/oryx.py ADDED
@@ -0,0 +1,475 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import logging
3
+ import math
4
+ from datetime import timedelta
5
+ from typing import List, Optional, Sequence, Tuple, Union
6
+
7
+ import numpy as np
8
+ import torch
9
+ from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
10
+ from accelerate.state import AcceleratorState
11
+ from decord import VideoReader, cpu
12
+ from PIL import Image
13
+ from tqdm import tqdm
14
+ from transformers import AutoConfig
15
+
16
+ from lmms_eval.api.instance import Instance
17
+ from lmms_eval.api.model import lmms
18
+ from lmms_eval.api.registry import register_model
19
+ from lmms_eval.models.model_utils.load_video import read_video_pyav
20
+
21
+ eval_logger = logging.getLogger("lmms-eval")
22
+ import os
23
+ import sys
24
+
25
+ try:
26
+ from oryx.constants import (
27
+ DEFAULT_IM_END_TOKEN,
28
+ DEFAULT_IM_START_TOKEN,
29
+ DEFAULT_IMAGE_TOKEN,
30
+ IMAGE_TOKEN_INDEX,
31
+ )
32
+ from oryx.conversation import SeparatorStyle, conv_templates
33
+ from oryx.mm_utils import (
34
+ KeywordsStoppingCriteria,
35
+ get_model_name_from_path,
36
+ process_anyres_highres_image_genli,
37
+ process_anyres_video_genli,
38
+ tokenizer_image_token,
39
+ )
40
+ from oryx.model.builder import load_pretrained_model
41
+ from oryx.model.language_model.oryx_llama import OryxConfig
42
+ except ImportError:
43
+ eval_logger.debug("Oryx is not installed. Please install Oryx to use this model.")
44
+
45
+ try:
46
+ from oryx.model.language_model.oryx_qwen import OryxQwenConfig
47
+
48
+ AutoConfig.register("oryx_qwen", OryxQwenConfig)
49
+ except:
50
+ eval_logger.debug("")
51
+
52
+
53
+ @register_model("oryx")
54
+ class Oryx(lmms):
55
+ def __init__(
56
+ self,
57
+ pretrained: str = "",
58
+ truncation: Optional[bool] = True,
59
+ device: Optional[str] = "cuda:0",
60
+ batch_size: Optional[Union[int, str]] = 1,
61
+ attn_implementation=(
62
+ "sdpa" if torch.__version__ >= "2.1.2" else "eager"
63
+ ), # inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
64
+ device_map="",
65
+ conv_template="qwen_1_5",
66
+ use_cache=True,
67
+ truncate_context=False,
68
+ max_frames_num: int = 32,
69
+ mm_resampler_type: str = "spatial_pool",
70
+ overwrite: bool = True,
71
+ video_decode_backend: str = "decord",
72
+ **kwargs,
73
+ ) -> None:
74
+ super().__init__()
75
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
76
+
77
+ accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
78
+ accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
79
+ if accelerator.num_processes > 1:
80
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
81
+ self.device_map = f"cuda:{accelerator.local_process_index}"
82
+ elif accelerator.num_processes == 1 and device_map == "auto":
83
+ self._device = torch.device(device)
84
+ self.device_map = device_map
85
+ else:
86
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
87
+ self.device_map = f"cuda:{accelerator.local_process_index}"
88
+
89
+ self.pretrained = pretrained
90
+ self.model_name = get_model_name_from_path(pretrained)
91
+ self.video_decode_backend = video_decode_backend
92
+ # self._config = AutoConfig.from_pretrained(self.pretrained)
93
+ self.overwrite = overwrite
94
+ self.mm_resampler_type = mm_resampler_type
95
+ self.max_frames_num = int(max_frames_num)
96
+ if self.overwrite == True:
97
+ overwrite_config = {}
98
+ overwrite_config["mm_resampler_type"] = self.mm_resampler_type
99
+ overwrite_config["patchify_video_feature"] = False
100
+ overwrite_config["attn_implementation"] = attn_implementation
101
+
102
+ cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
103
+
104
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, self.model_name, device_map=self.device_map, overwrite_config=overwrite_config)
105
+ else:
106
+ self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(
107
+ pretrained,
108
+ None,
109
+ self.model_name,
110
+ device_map=self.device_map,
111
+ )
112
+
113
+ self._config = self._model.config
114
+ self.model.eval()
115
+ self.model.tie_weights()
116
+ self.truncation = truncation
117
+ self.batch_size_per_gpu = int(batch_size)
118
+ self.conv_template = conv_template
119
+ self.use_cache = use_cache
120
+ self.truncate_context = truncate_context
121
+ if accelerator.num_processes > 1:
122
+ assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
123
+ # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
124
+ # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
125
+ # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
126
+ if accelerator.distributed_type == DistributedType.DEEPSPEED:
127
+ kwargs = {
128
+ "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
129
+ "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
130
+ }
131
+ AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
132
+ eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
133
+ if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
134
+ self._model = accelerator.prepare(self.model)
135
+ else:
136
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
137
+ self.accelerator = accelerator
138
+ if self.accelerator.is_local_main_process:
139
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
140
+ self._rank = self.accelerator.local_process_index
141
+ self._world_size = self.accelerator.num_processes
142
+ elif accelerator.num_processes == 1 and device_map == "auto":
143
+ eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
144
+ self._rank = 0
145
+ self._world_size = 1
146
+ else:
147
+ eval_logger.info(f"Using single device: {self._device}")
148
+ self.model.to(self._device)
149
+ self._rank = 0
150
+ self._world_size = 1
151
+
152
+ @property
153
+ def config(self):
154
+ # return the associated transformers.AutoConfig for the given pretrained model.
155
+ return self._config
156
+
157
+ @property
158
+ def tokenizer(self):
159
+ return self._tokenizer
160
+
161
+ @property
162
+ def model(self):
163
+ # returns the model, unwrapping it if using Accelerate
164
+ if hasattr(self, "accelerator"):
165
+ return self.accelerator.unwrap_model(self._model)
166
+ else:
167
+ return self._model
168
+
169
+ @property
170
+ def eot_token_id(self):
171
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
172
+ return self.tokenizer.eos_token_id
173
+
174
+ @property
175
+ def max_length(self):
176
+ return self._max_length
177
+
178
+ def pad_sequence(self, input_ids, batch_first, padding_value):
179
+ if self.tokenizer.padding_side == "left":
180
+ input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
181
+ input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
182
+ if self.tokenizer.padding_side == "left":
183
+ input_ids = torch.flip(input_ids, [1])
184
+ return input_ids
185
+
186
+ @property
187
+ def batch_size(self):
188
+ return self.batch_size_per_gpu
189
+
190
+ @property
191
+ def device(self):
192
+ return self._device
193
+
194
+ @property
195
+ def rank(self):
196
+ return self._rank
197
+
198
+ @property
199
+ def world_size(self):
200
+ return self._world_size
201
+
202
+ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
203
+ """ """
204
+ add_special_tokens = False if add_special_tokens is None else add_special_tokens
205
+ encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
206
+ # left-truncate the encoded context to be at most `left_truncate_len` tokens long
207
+ if left_truncate_len:
208
+ encoding = encoding[-left_truncate_len:]
209
+ return encoding
210
+
211
+ def load_video(self, video_path, max_frames_num):
212
+ vr = VideoReader(video_path, ctx=cpu(0))
213
+ total_frame_num = len(vr)
214
+ fps = round(vr.get_avg_fps())
215
+
216
+ uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
217
+ frame_idx = uniform_sampled_frames.tolist()
218
+ modality = "video"
219
+
220
+ spare_frames = vr.get_batch(frame_idx).asnumpy()
221
+ return spare_frames, modality # (frames, height, width, channels)
222
+
223
+ def tok_decode(self, tokens):
224
+ return self.tokenizer.decode(tokens)
225
+
226
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
227
+ res = []
228
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
229
+
230
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
231
+ # encode, pad, and truncate contexts for this batch
232
+ if type(doc_to_target) == str:
233
+ continuation = doc_to_target
234
+ else:
235
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
236
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
237
+ visuals = self.flatten(visuals)
238
+ videos = []
239
+ # video
240
+ if type(visuals[0][0]) == str:
241
+ for visual in visuals:
242
+ video = self.load_video(visual, self.max_frames_num)
243
+ video = self._image_processor.preprocess(video, return_tensors="pt")["pixel_values"].bfloat16().to(self.device)
244
+ videos.append(video)
245
+ task_type = "video"
246
+ # image
247
+ else:
248
+ for visual in visuals:
249
+ image_tensor_, image_highres_tensor_ = process_anyres_highres_image_genli(visual, self._image_processor)
250
+ image_tensor.append(image_tensor_)
251
+ image_highres_tensor.append(image_highres_tensor_)
252
+ if all(x.shape == image_tensor[0].shape for x in image_tensor):
253
+ image_tensor = torch.stack(image_tensor, dim=0)
254
+ if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor):
255
+ image_highres_tensor = torch.stack(image_highres_tensor, dim=0)
256
+ if type(image_tensor) is list:
257
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
258
+ else:
259
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
260
+ if type(image_highres_tensor) is list:
261
+ image_highres_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_highres_tensor]
262
+ else:
263
+ image_highres_tensor = image_highres_tensor.to(dtype=torch.bfloat16, device=self.device)
264
+
265
+ image_sizes = [visuals[idx].size for idx in range(len(visuals))]
266
+ task_type = "image"
267
+
268
+ qs = contexts
269
+ if self.model.config.mm_use_im_start_end:
270
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
271
+ else:
272
+ qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
273
+
274
+ conv = conv_templates[self.conv_template].copy()
275
+ conv.append_message(conv.roles[0], qs)
276
+ conv.append_message(conv.roles[1], None)
277
+ prompt = conv.get_prompt()
278
+
279
+ contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
280
+
281
+ conv = conv_templates[self.conv_template].copy()
282
+ conv.append_message(conv.roles[0], qs)
283
+ conv.append_message(conv.roles[1], continuation)
284
+ prompt = conv.get_prompt()
285
+
286
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
287
+
288
+ labels = input_ids.clone()
289
+ # Context part no need to calculate for loss
290
+ labels[0, : contxt_id.shape[1]] = -100
291
+
292
+ with torch.inference_mode():
293
+ if task_type == "video":
294
+ outputs = self.model(
295
+ input_ids=input_ids,
296
+ labels=labels,
297
+ modalities=["video"],
298
+ images=videos,
299
+ images_highres=videos,
300
+ )
301
+ else:
302
+ outputs = self.model(
303
+ input_ids=input_ids,
304
+ labels=labels,
305
+ modalities=["image"] * len(image_sizes),
306
+ images=image_tensor,
307
+ images_highres=image_highres_tensor,
308
+ image_sizes=image_sizes,
309
+ )
310
+
311
+ loss = outputs["loss"]
312
+ # loss = torch.exp(loss)
313
+ logits = outputs["logits"]
314
+ greedy_tokens = logits.argmax(dim=-1)
315
+ cont_toks = input_ids[:, contxt_id.shape[1] :] # [1, seq]
316
+ greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]] # [1, seq]
317
+ max_equal = (greedy_tokens == cont_toks).all()
318
+ res.append((float(loss.item()), bool(max_equal)))
319
+ pbar.update(1)
320
+ pbar.close()
321
+ return res
322
+
323
+ def flatten(self, input):
324
+ new_list = []
325
+ for i in input:
326
+ for j in i:
327
+ new_list.append(j)
328
+ return new_list
329
+
330
+ def generate_until(self, requests) -> List[str]:
331
+ res = []
332
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
333
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
334
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
335
+ visuals = self.flatten(visuals)
336
+ videos = []
337
+ modalities = []
338
+ try:
339
+ if task == "mvbench_episodic_reasoning":
340
+ sampled_frm = min(len(visuals), self.max_frames_num)
341
+ indices = np.linspace(0, len(visuals) - 1, sampled_frm, dtype=int)
342
+ frames = [visuals[i] for i in indices]
343
+ video = np.stack([np.array(x) for x in frames])
344
+ modality = "video"
345
+ frames = []
346
+ for frame in video:
347
+ self._image_processor.do_resize = False
348
+ self._image_processor.do_center_crop = False
349
+ frames.append(process_anyres_video_genli(Image.fromarray(frame).convert("RGB"), self._image_processor))
350
+ video = torch.stack(frames, dim=0).bfloat16().to(self.device)
351
+ videos.append(video)
352
+ modalities.append(modality)
353
+ else:
354
+ if type(visuals[0][0]) == str:
355
+ for visual in visuals:
356
+ if self.video_decode_backend == "decord":
357
+ video, modality = self.load_video(visual, self.max_frames_num)
358
+ elif self.video_decode_backend == "pyav":
359
+ video, modality = read_video_pyav(visual, num_frm=self.max_frames_num)
360
+ # video = self.load_video(visual, self.max_frames_num)
361
+ frames = []
362
+ for frame in video:
363
+ self._image_processor.do_resize = False
364
+ self._image_processor.do_center_crop = False
365
+ frames.append(process_anyres_video_genli(Image.fromarray(frame).convert("RGB"), self._image_processor))
366
+ video = torch.stack(frames, dim=0).bfloat16().to(self.device)
367
+ videos.append(video)
368
+ modalities.append(modality)
369
+ task_type = "video"
370
+ else:
371
+ self._image_processor.do_resize = False
372
+ self._image_processor.do_center_crop = False
373
+ image_tensor, image_highres_tensor = [], []
374
+ for visual in visuals:
375
+ image_tensor_, image_highres_tensor_ = process_anyres_highres_image_genli(visual, self._image_processor)
376
+ image_tensor.append(image_tensor_)
377
+ image_highres_tensor.append(image_highres_tensor_)
378
+ if all(x.shape == image_tensor[0].shape for x in image_tensor):
379
+ image_tensor = torch.stack(image_tensor, dim=0)
380
+ if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor):
381
+ image_highres_tensor = torch.stack(image_highres_tensor, dim=0)
382
+ if type(image_tensor) is list:
383
+ image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
384
+ else:
385
+ image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
386
+ if type(image_highres_tensor) is list:
387
+ image_highres_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_highres_tensor]
388
+ else:
389
+ image_highres_tensor = image_highres_tensor.to(dtype=torch.bfloat16, device=self.device)
390
+ task_type = "image"
391
+
392
+ except Exception as e:
393
+ eval_logger.info(f"{e}")
394
+ eval_logger.info(f"Video {visuals} can not load, check the source")
395
+ video_path = "\n".join(visuals)
396
+ res.append(f"Video {video_path} can not load, check the source")
397
+ pbar.update(1)
398
+ continue
399
+
400
+ qs = contexts
401
+ if self.model.config.mm_use_im_start_end:
402
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
403
+ else:
404
+ qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
405
+
406
+ conv = conv_templates[self.conv_template].copy()
407
+
408
+ conv.append_message(conv.roles[0], qs)
409
+ conv.append_message(conv.roles[1], None)
410
+ prompt = conv.get_prompt()
411
+
412
+ input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
413
+ pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
414
+ attention_masks = input_ids.ne(pad_token_ids).long().to(self.device)
415
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
416
+ keywords = [stop_str]
417
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
418
+
419
+ cur_prompt = contexts
420
+ if task_type == "image":
421
+ gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
422
+ if "max_new_tokens" not in gen_kwargs:
423
+ gen_kwargs["max_new_tokens"] = 1024
424
+ if "temperature" not in gen_kwargs:
425
+ gen_kwargs["temperature"] = 0.2
426
+ if "top_p" not in gen_kwargs:
427
+ gen_kwargs["top_p"] = None
428
+ if "num_beams" not in gen_kwargs:
429
+ gen_kwargs["num_beams"] = 1
430
+
431
+ try:
432
+ with torch.inference_mode():
433
+ if task_type == "video":
434
+ output_ids = self.model.generate(
435
+ inputs=input_ids,
436
+ images=videos,
437
+ images_highres=videos,
438
+ attention_mask=attention_masks,
439
+ modalities=modalities,
440
+ use_cache=self.use_cache,
441
+ stopping_criteria=[stopping_criteria],
442
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
443
+ temperature=gen_kwargs["temperature"],
444
+ top_p=gen_kwargs["top_p"],
445
+ num_beams=gen_kwargs["num_beams"],
446
+ max_new_tokens=gen_kwargs["max_new_tokens"],
447
+ )
448
+ else:
449
+ output_ids = self.model.generate(
450
+ input_ids,
451
+ attention_mask=attention_masks,
452
+ pad_token_id=pad_token_ids,
453
+ modalities=["image"] * len(gen_kwargs["image_sizes"]),
454
+ images=image_tensor,
455
+ images_highres=image_highres_tensor,
456
+ image_sizes=gen_kwargs["image_sizes"],
457
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
458
+ temperature=gen_kwargs["temperature"],
459
+ top_p=gen_kwargs["top_p"],
460
+ num_beams=gen_kwargs["num_beams"],
461
+ max_new_tokens=gen_kwargs["max_new_tokens"],
462
+ use_cache=self.use_cache,
463
+ )
464
+ outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
465
+ # print(outputs)
466
+ res.append(outputs)
467
+ pbar.update(1)
468
+ except Exception as e:
469
+ eval_logger.info(f"{e}")
470
+ eval_logger.info(f"Video {visuals} generate failed, check the source")
471
+ video_path = "\n".join(visuals)
472
+ res.append(f"Video {video_path} generate failed, check the source")
473
+ pbar.update(1)
474
+ continue
475
+ return res
LaViDa/eval/lmms_eval/models/phi3v.py ADDED
@@ -0,0 +1,220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List, Optional, Tuple, Union
2
+
3
+ import torch
4
+ from accelerate import Accelerator, DistributedType
5
+ from loguru import logger as eval_logger
6
+ from tqdm import tqdm
7
+ from transformers import AutoModelForCausalLM, AutoProcessor
8
+
9
+ from lmms_eval import utils
10
+ from lmms_eval.api.instance import Instance
11
+ from lmms_eval.api.model import lmms
12
+ from lmms_eval.api.registry import register_model
13
+
14
+
15
+ @register_model("phi3v")
16
+ class Phi3v(lmms):
17
+ """
18
+ This class implements inference for the microsoft/Phi-3-vision-128k-instruct model.
19
+ To learn more about this model please visit the following links:
20
+ 1. https://huggingface.co/microsoft/Phi-3-vision-128k-instruct
21
+ 2. https://azure.microsoft.com/en-us/blog/new-models-added-to-the-phi-3-family-available-on-microsoft-azure/
22
+ 3. https://github.com/microsoft/Phi-3CookBook
23
+
24
+ NOTE: This class was adapted from quen_vl.py and llava_hf.py.
25
+
26
+ Example:
27
+
28
+ accelerate launch --num_processes=4 -m lmms_eval --model phi3v --tasks mmmu_val \
29
+ --batch_size 1 --log_samples --log_samples_suffix phi3v_mmmu --output_path ./logs/
30
+ """
31
+
32
+ def __init__(
33
+ self,
34
+ model_id_name: str = "microsoft/Phi-3-vision-128k-instruct",
35
+ device: str = "cuda",
36
+ dtype: Optional[Union[str, torch.dtype]] = "auto",
37
+ batch_size: int = 1,
38
+ trust_remote_code: Optional[bool] = True,
39
+ use_cache: bool = True,
40
+ **kwargs,
41
+ ) -> None:
42
+ super().__init__()
43
+ # Do not use kwargs for now
44
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
45
+ # Setup accelerator.
46
+ accelerator = Accelerator()
47
+ if accelerator.num_processes > 1:
48
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
49
+ else:
50
+ self._device = device
51
+ # Load model.
52
+ self._model = AutoModelForCausalLM.from_pretrained(model_id_name, device_map=device, trust_remote_code=trust_remote_code, torch_dtype=dtype)
53
+ self._processor = AutoProcessor.from_pretrained(model_id_name, trust_remote_code=trust_remote_code)
54
+ self._processor.tokenizer.padding_side = "left"
55
+ self._tokenizer = self._processor.tokenizer
56
+ self._config = self._model.config
57
+ self.batch_size_per_gpu = int(batch_size)
58
+ assert self.batch_size_per_gpu == 1, "batch_size_per_gpu > 1 is not supported for now."
59
+ self.use_cache = use_cache
60
+ if accelerator.num_processes > 1:
61
+ distributed_type_list = [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
62
+ assert accelerator.distributed_type in distributed_type_list, "Unsupported distributed type provided. Only DDP and FSDP are supported."
63
+ if accelerator.distributed_type == DistributedType.FSDP:
64
+ self._model = accelerator.prepare(self.model)
65
+ else:
66
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
67
+ self.accelerator = accelerator
68
+ if self.accelerator.is_local_main_process:
69
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
70
+ self._rank = self.accelerator.local_process_index
71
+ self._world_size = self.accelerator.num_processes
72
+ else:
73
+ eval_logger.info(f"Using single device: {self._device}")
74
+ self.model.to(self._device)
75
+ self._rank = 0
76
+ self._world_size = 1
77
+
78
+ @property
79
+ def config(self):
80
+ # return the associated transformers.AutoConfig for the given pretrained model.
81
+ return self._config
82
+
83
+ @property
84
+ def tokenizer(self):
85
+ return self._tokenizer
86
+
87
+ @property
88
+ def model(self):
89
+ # returns the model, unwrapping it if using Accelerate
90
+ if hasattr(self, "accelerator"):
91
+ return self.accelerator.unwrap_model(self._model)
92
+ else:
93
+ return self._model
94
+
95
+ @property
96
+ def eot_token_id(self):
97
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
98
+ return self.tokenizer.eos_token_id
99
+
100
+ @property
101
+ def max_length(self):
102
+ return self._max_length
103
+
104
+ @property
105
+ def batch_size(self):
106
+ return self.batch_size_per_gpu
107
+
108
+ @property
109
+ def device(self):
110
+ return self._device
111
+
112
+ @property
113
+ def rank(self):
114
+ return self._rank
115
+
116
+ @property
117
+ def world_size(self):
118
+ return self._world_size
119
+
120
+ def flatten(self, input):
121
+ new_list = []
122
+ for i in input:
123
+ for j in i:
124
+ new_list.append(j)
125
+ return new_list
126
+
127
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
128
+ raise NotImplementedError("Not implemented for Phi3v.")
129
+
130
+ def generate_until(self, requests: List[Instance]) -> List[str]:
131
+ res = []
132
+
133
+ def _collate(x):
134
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
135
+ # - time estimates will always be over not underestimates, which is more useful for planning
136
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
137
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
138
+ # automatic adaptive batches much much easier to implement
139
+ # - any OOMs will happen right away rather than near the end
140
+ toks = self.tokenizer.encode(x[0])
141
+ return -len(toks), x[0]
142
+
143
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
144
+ # we group requests by their generation_kwargs,
145
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
146
+ # in the same batch.
147
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
148
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
149
+ for chunk in chunks:
150
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
151
+ task = task[0]
152
+ split = split[0]
153
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
154
+ visuals = self.flatten(visuals)
155
+ # We assume all gen kwargs in the batch are the same
156
+ # this is safe to assume because the `grouper` object ensures it.
157
+ gen_kwargs = all_gen_kwargs[0]
158
+ # Set default values for until and max_new_tokens
159
+ until = [self.tokenizer.decode(self.eot_token_id)]
160
+ # Update values from gen_kwargs if present
161
+ if "until" in gen_kwargs:
162
+ until = gen_kwargs.pop("until")
163
+ if isinstance(until, str):
164
+ until = [until]
165
+ elif not isinstance(until, list):
166
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
167
+ if isinstance(contexts, tuple):
168
+ contexts = list(contexts)
169
+ for i in range(len(contexts)):
170
+ if "<image>" in contexts[i]:
171
+ query = "" + contexts[i]
172
+ img_placeholder_count = 1
173
+ while "<image>" in query:
174
+ query = query.replace("<image>", f"<|image_{img_placeholder_count}|>", 1)
175
+ img_placeholder_count += 1
176
+ else:
177
+ query = ""
178
+ for placeholder_id in range(len(visuals)):
179
+ query += f"<|image_{placeholder_id+1}|>\n"
180
+ query += contexts[i]
181
+ messages = [{"role": "user", "content": query}]
182
+ contexts[i] = self._tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
183
+ assert len(contexts) == 1
184
+ #
185
+ context = contexts[0]
186
+ input_ids = self._processor(text=context, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
187
+ # Setting default parameters.
188
+ if "max_new_tokens" not in gen_kwargs:
189
+ gen_kwargs["max_new_tokens"] = 1024
190
+ if "temperature" not in gen_kwargs:
191
+ gen_kwargs["temperature"] = 0
192
+ if "top_p" not in gen_kwargs:
193
+ gen_kwargs["top_p"] = None
194
+ if "num_beams" not in gen_kwargs:
195
+ gen_kwargs["num_beams"] = 1
196
+ # Generate answer.
197
+ pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eod_id
198
+ generate_ids = self.model.generate(
199
+ **input_ids,
200
+ eos_token_id=self.tokenizer.eos_token_id,
201
+ pad_token_id=pad_token_id,
202
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
203
+ temperature=gen_kwargs["temperature"],
204
+ top_p=gen_kwargs["top_p"],
205
+ num_beams=gen_kwargs["num_beams"],
206
+ max_new_tokens=gen_kwargs["max_new_tokens"],
207
+ use_cache=self.use_cache,
208
+ )
209
+ generate_ids = generate_ids[:, input_ids["input_ids"].shape[1] :]
210
+ response = self._processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
211
+ res.append(response)
212
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), response)
213
+ pbar.update(1)
214
+ # reorder this group of results back to original unsorted form
215
+ res = re_ords.get_original(res)
216
+ pbar.close()
217
+ return res
218
+
219
+ def generate_until_multi_round(self, requests) -> List[str]:
220
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/qwen2_5_vl.py ADDED
@@ -0,0 +1,323 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ from io import BytesIO
3
+ from typing import List, Optional, Tuple, Union
4
+
5
+ import decord
6
+ import numpy as np
7
+ import torch
8
+ from accelerate import Accelerator, DistributedType
9
+ from loguru import logger as eval_logger
10
+ from PIL import Image
11
+ from tqdm import tqdm
12
+ from transformers import (
13
+ AutoProcessor,
14
+ AutoTokenizer,
15
+ Qwen2_5_VLForConditionalGeneration,
16
+ )
17
+
18
+ from lmms_eval import utils
19
+ from lmms_eval.api.instance import Instance
20
+ from lmms_eval.api.model import lmms
21
+ from lmms_eval.api.registry import register_model
22
+ from lmms_eval.models.model_utils.load_video import read_video_pyav_base64
23
+
24
+ try:
25
+ from qwen_vl_utils import process_vision_info
26
+ except ImportError:
27
+ eval_logger.warning("Failed to import qwen_vl_utils; Please install it via `pip install qwen-vl-utils`")
28
+
29
+
30
+ @register_model("qwen2_5_vl")
31
+ class Qwen2_5_VL(lmms):
32
+ """
33
+ Qwen2.5_VL Model
34
+ "https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct"
35
+ """
36
+
37
+ def __init__(
38
+ self,
39
+ pretrained: str = "Qwen/Qwen2.5-VL-3B-Instruct",
40
+ device: Optional[str] = "cuda",
41
+ device_map: Optional[str] = "auto",
42
+ batch_size: Optional[Union[int, str]] = 1,
43
+ use_cache=True,
44
+ use_flash_attention_2: Optional[bool] = False,
45
+ min_pixels: int = 256 * 28 * 28,
46
+ max_pixels: int = 1605632,
47
+ max_num_frames: int = 32,
48
+ use_custom_video_loader: Optional[bool] = False,
49
+ fps: Optional[float] = None, # Only applicable if use_custom_video_loader is True
50
+ max_image_size: Optional[int] = None, # Only applicable if use_custom_video_loader is True
51
+ **kwargs,
52
+ ) -> None:
53
+ super().__init__()
54
+ # Do not use kwargs for now
55
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
56
+
57
+ self.use_custom_video_loader = use_custom_video_loader
58
+ self.fps = fps
59
+ # if self.fps and not self.use_custom_video_loader:
60
+ # raise ValueError("FPS is only applicable if use_custom_video_loader is True")
61
+ self.max_image_size = max_image_size
62
+ if self.max_image_size and not self.use_custom_video_loader:
63
+ raise ValueError("max_image_size is only applicable if use_custom_video_loader is True")
64
+
65
+ accelerator = Accelerator()
66
+ if accelerator.num_processes > 1:
67
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
68
+ self.device_map = f"cuda:{accelerator.local_process_index}"
69
+ elif accelerator.num_processes == 1 and device_map == "auto":
70
+ self._device = torch.device(device)
71
+ self.device_map = device_map
72
+ else:
73
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
74
+ self.device_map = f"cuda:{accelerator.local_process_index}"
75
+
76
+ if use_flash_attention_2:
77
+ self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
78
+ pretrained,
79
+ torch_dtype=torch.bfloat16,
80
+ device_map=self.device_map,
81
+ attn_implementation="flash_attention_2",
82
+ ).eval()
83
+ else:
84
+ self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
85
+ self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
86
+ self.max_pixels = max_pixels
87
+ self.min_pixels = min_pixels
88
+ self.max_num_frames = max_num_frames
89
+ self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
90
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
91
+
92
+ self._config = self.model.config
93
+ self.batch_size_per_gpu = int(batch_size)
94
+ self.use_cache = use_cache
95
+
96
+ if accelerator.num_processes > 1:
97
+ assert accelerator.distributed_type in [
98
+ DistributedType.FSDP,
99
+ DistributedType.MULTI_GPU,
100
+ ], "Unsupported distributed type provided. Only DDP and FSDP are supported."
101
+ if accelerator.distributed_type == DistributedType.FSDP:
102
+ self._model = accelerator.prepare(self.model)
103
+ else:
104
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
105
+ self.accelerator = accelerator
106
+ if self.accelerator.is_local_main_process:
107
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
108
+ self._rank = self.accelerator.local_process_index
109
+ self._world_size = self.accelerator.num_processes
110
+ else:
111
+ self._rank = 0
112
+ self._world_size = 1
113
+
114
+ @property
115
+ def config(self):
116
+ # return the associated transformers.AutoConfig for the given pretrained model.
117
+ return self._config
118
+
119
+ @property
120
+ def tokenizer(self):
121
+ return self._tokenizer
122
+
123
+ @property
124
+ def model(self):
125
+ # returns the model, unwrapping it if using Accelerate
126
+ if hasattr(self, "accelerator"):
127
+ return self.accelerator.unwrap_model(self._model)
128
+ else:
129
+ return self._model
130
+
131
+ @property
132
+ def eot_token_id(self):
133
+ return self.tokenizer.eos_token_id
134
+
135
+ @property
136
+ def max_length(self):
137
+ return self._max_length
138
+
139
+ @property
140
+ def batch_size(self):
141
+ return self.batch_size_per_gpu
142
+
143
+ @property
144
+ def device(self):
145
+ return self._device
146
+
147
+ @property
148
+ def rank(self):
149
+ return self._rank
150
+
151
+ @property
152
+ def world_size(self):
153
+ return self._world_size
154
+
155
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
156
+ raise NotImplementedError("Loglikelihood is not implemented for Qwen2.5_VL")
157
+
158
+ def flatten(self, input):
159
+ new_list = []
160
+ for i in input:
161
+ for j in i:
162
+ new_list.append(j)
163
+ return new_list
164
+
165
+ def generate_until(self, requests: List[Instance]) -> List[str]:
166
+ res = []
167
+
168
+ def _collate(x):
169
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
170
+ # - time estimates will always be over not underestimates, which is more useful for planning
171
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
172
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
173
+ # automatic adaptive batches much much easier to implement
174
+ # - any OOMs will happen right away rather than near the end
175
+ toks = self.tokenizer.encode(x[0])
176
+ return -len(toks), x[0]
177
+
178
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
179
+ # we group requests by their generation_kwargs,
180
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
181
+ # in the same batch.
182
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
183
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
184
+ for chunk in chunks:
185
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
186
+ task = task[0]
187
+ split = split[0]
188
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
189
+ visuals = self.flatten(visuals)
190
+
191
+ gen_kwargs = all_gen_kwargs[0]
192
+
193
+ # Set default values for until and max_new_tokens
194
+ until = [self.tokenizer.decode(self.eot_token_id)]
195
+
196
+ # Update values from gen_kwargs if present
197
+ if "until" in gen_kwargs:
198
+ until = gen_kwargs.pop("until")
199
+ if isinstance(until, str):
200
+ until = [until]
201
+ elif not isinstance(until, list):
202
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
203
+
204
+ # if isinstance(contexts, tuple):
205
+ # contexts = list(contexts)
206
+
207
+ # for i in range(len(contexts)):
208
+ # for j in range(32):
209
+ # if f"<image {j}>" in contexts[i]:
210
+ # contexts[i] = contexts[i].replace(f"<image {j}>", "<image>")
211
+ # if f"\\<image {j}\\>" in contexts[i]:
212
+ # contexts[i] = contexts[i].replace(f"\\<image {j}\\>", "<image>")
213
+ # if "<image>" in contexts[i]:
214
+ # contexts[i] = contexts[i].replace("<image>", "")
215
+ # print(contexts[i])
216
+
217
+ # for i in range(len(contexts)):
218
+ # if "<image>" in contexts[i]:
219
+ # contexts[i] = contexts[i].replace("<image>", "")
220
+
221
+ messages = []
222
+ processed_visuals = []
223
+ for i, context in enumerate(contexts):
224
+ # context += "\nPlease think step by step."
225
+ # if "<image>" in context:
226
+ # context = context.replace("<image>", "")
227
+
228
+ message = [{"role": "system", "content": "You are a helpful assistant."}]
229
+
230
+ if len(visuals) > 0:
231
+ visual = visuals[i] if i < len(visuals) else None
232
+ if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")): # Video file
233
+ if self.use_custom_video_loader:
234
+ visual = read_video_pyav_base64(visual, num_frm=self.max_num_frames, fps=self.fps, img_format="JPEG", max_image_size=self.max_image_size)
235
+ image_contents = list(map(lambda x: f"data:image/jpeg;base64,{x}", visual))
236
+ message.append({"role": "user", "content": [{"type": "video", "video": image_contents}, {"type": "text", "text": context}]})
237
+ else:
238
+ vr = decord.VideoReader(visual)
239
+ first_frame = vr[0].asnumpy()
240
+ height, width = first_frame.shape[:2]
241
+ # max_pixels = height * width
242
+ message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": 360 * 420}, {"type": "text", "text": context}]})
243
+ elif isinstance(visual, Image.Image): # Single image
244
+ base64_image = visual.convert("RGB")
245
+ buffer = BytesIO()
246
+ base64_image.save(buffer, format="JPEG")
247
+ base64_bytes = base64.b64encode(buffer.getvalue())
248
+ base64_string = base64_bytes.decode("utf-8")
249
+ message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
250
+ elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual): # Multiple images
251
+ image_content = []
252
+ for v in visual:
253
+ base64_image = v.convert("RGB")
254
+ buffer = BytesIO()
255
+ base64_image.save(buffer, format="JPEG")
256
+ base64_bytes = base64.b64encode(buffer.getvalue())
257
+ base64_string = base64_bytes.decode("utf-8")
258
+ image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
259
+ message.append({"role": "user", "content": image_content + [{"type": "text", "text": context}]})
260
+ else:
261
+ message.append({"role": "user", "content": [{"type": "text", "text": context}]})
262
+ else:
263
+ message.append({"role": "user", "content": [{"type": "text", "text": context}]})
264
+
265
+ messages.append(message)
266
+ # print("message")
267
+
268
+ text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
269
+ image_inputs, video_inputs = process_vision_info(messages)
270
+ inputs = self.processor(
271
+ text=text,
272
+ images=image_inputs,
273
+ videos=video_inputs,
274
+ # fps=self.fps,
275
+ padding=True,
276
+ return_tensors="pt",
277
+ )
278
+
279
+ if self.device_map == "auto":
280
+ inputs = inputs.to("cuda")
281
+ else:
282
+ inputs = inputs.to(self.device)
283
+
284
+ if "max_new_tokens" not in gen_kwargs:
285
+ gen_kwargs["max_new_tokens"] = 4096
286
+ if "temperature" not in gen_kwargs:
287
+ gen_kwargs["temperature"] = 0
288
+ if "top_p" not in gen_kwargs:
289
+ gen_kwargs["top_p"] = None
290
+ if "num_beams" not in gen_kwargs:
291
+ gen_kwargs["num_beams"] = 1
292
+
293
+ pad_token_id = self.tokenizer.pad_token_id
294
+
295
+ cont = self.model.generate(
296
+ **inputs,
297
+ eos_token_id=self.tokenizer.eos_token_id,
298
+ pad_token_id=pad_token_id,
299
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
300
+ temperature=gen_kwargs["temperature"],
301
+ top_p=gen_kwargs["top_p"],
302
+ num_beams=gen_kwargs["num_beams"],
303
+ max_new_tokens=4096,
304
+ use_cache=self.use_cache,
305
+ )
306
+
307
+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
308
+ answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
309
+ for i, ans in enumerate(answers):
310
+ answers[i] = ans
311
+
312
+ for ans, context in zip(answers, contexts):
313
+ res.append(ans)
314
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
315
+ pbar.update(1)
316
+ # reorder this group of results back to original unsorted form
317
+ res = re_ords.get_original(res)
318
+
319
+ pbar.close()
320
+ return res
321
+
322
+ def generate_until_multi_round(self, requests) -> List[str]:
323
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/qwen2_5_vl_interleave.py ADDED
@@ -0,0 +1,408 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import json
3
+ import os
4
+ from io import BytesIO
5
+ from re import I
6
+ from typing import List, Optional, Tuple, Union
7
+
8
+ import decord
9
+ import numpy as np
10
+ import torch
11
+ from accelerate import Accelerator, DistributedType
12
+ from loguru import logger as eval_logger
13
+ from PIL import Image
14
+ from tqdm import tqdm
15
+ from transformers import (
16
+ AutoProcessor,
17
+ AutoTokenizer,
18
+ Qwen2_5_VLForConditionalGeneration,
19
+ )
20
+
21
+ from lmms_eval import utils
22
+ from lmms_eval.api.instance import Instance
23
+ from lmms_eval.api.model import lmms
24
+ from lmms_eval.api.registry import register_model
25
+ from lmms_eval.models.model_utils.load_video import read_video_pyav_base64
26
+
27
+ try:
28
+ from qwen_vl_utils import process_vision_info
29
+ except ImportError:
30
+ eval_logger.warning("Failed to import qwen_vl_utils; Please install it via `pip install qwen-vl-utils`")
31
+
32
+
33
+ @register_model("qwen2_5_vl_interleave")
34
+ class Qwen2_5_VL_Interleave(lmms):
35
+ """
36
+ Qwen2.5_VL Model
37
+ "https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct"
38
+ """
39
+
40
+ def __init__(
41
+ self,
42
+ pretrained: str = "Qwen/Qwen2.5-VL-3B-Instruct",
43
+ device: Optional[str] = "cuda",
44
+ device_map: Optional[str] = "auto",
45
+ batch_size: Optional[Union[int, str]] = 1,
46
+ use_cache=True,
47
+ use_flash_attention_2: Optional[bool] = False,
48
+ min_pixels: int = 256 * 28 * 28,
49
+ max_pixels: int = 256 * 28 * 28,
50
+ max_num_frames: int = 32,
51
+ use_custom_video_loader: Optional[bool] = False,
52
+ fps: Optional[float] = None, # Only applicable if use_custom_video_loader is True
53
+ max_image_size: Optional[int] = None, # Only applicable if use_custom_video_loader is True
54
+ **kwargs,
55
+ ) -> None:
56
+ super().__init__()
57
+ # Do not use kwargs for now
58
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
59
+
60
+ self.use_custom_video_loader = use_custom_video_loader
61
+ self.fps = fps
62
+ # if self.fps and not self.use_custom_video_loader:
63
+ # raise ValueError("FPS is only applicable if use_custom_video_loader is True")
64
+ self.max_image_size = max_image_size
65
+ # if self.max_image_size and not self.use_custom_video_loader:
66
+ # raise ValueError("max_image_size is only applicable if use_custom_video_loader is True")
67
+
68
+ accelerator = Accelerator()
69
+ if accelerator.num_processes > 1:
70
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
71
+ self.device_map = f"cuda:{accelerator.local_process_index}"
72
+ elif accelerator.num_processes == 1 and device_map == "auto":
73
+ self._device = torch.device(device)
74
+ self.device_map = device_map
75
+ else:
76
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
77
+ self.device_map = f"cuda:{accelerator.local_process_index}"
78
+
79
+ if use_flash_attention_2:
80
+ self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
81
+ pretrained,
82
+ torch_dtype=torch.bfloat16,
83
+ device_map=self.device_map,
84
+ attn_implementation="flash_attention_2",
85
+ ).eval()
86
+ else:
87
+ self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
88
+ self.max_pixels = max_pixels
89
+ self.min_pixels = min_pixels
90
+ self.max_num_frames = max_num_frames
91
+ self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
92
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
93
+
94
+ self._config = self.model.config
95
+ self.batch_size_per_gpu = int(batch_size)
96
+ self.use_cache = use_cache
97
+
98
+ if accelerator.num_processes > 1:
99
+ assert accelerator.distributed_type in [
100
+ DistributedType.FSDP,
101
+ DistributedType.MULTI_GPU,
102
+ ], "Unsupported distributed type provided. Only DDP and FSDP are supported."
103
+ if accelerator.distributed_type == DistributedType.FSDP:
104
+ self._model = accelerator.prepare(self.model)
105
+ else:
106
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
107
+ self.accelerator = accelerator
108
+ if self.accelerator.is_local_main_process:
109
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
110
+ self._rank = self.accelerator.local_process_index
111
+ self._world_size = self.accelerator.num_processes
112
+ else:
113
+ self._rank = 0
114
+ self._world_size = 1
115
+
116
+ # self.continual_mode = continual_mode
117
+ # if self.continual_mode and response_persistent_folder is None:
118
+ # raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.")
119
+ # if self.continual_mode:
120
+ # self.response_persistent_folder = response_persistent_folder
121
+ # if not os.path.exists(self.response_persistent_folder):
122
+ # os.makedirs(self.response_persistent_folder)
123
+ # self.model_version = pretrained.split("/")[-1].replace("-", "_").lower()
124
+ # self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
125
+
126
+ # if os.path.exists(self.response_persistent_file):
127
+ # with open(self.response_persistent_file, "r") as f:
128
+ # self.response_cache = json.load(f)
129
+ # self.cache_mode = "resume"
130
+ # else:
131
+ # self.response_cache = {}
132
+ # self.cache_mode = "start"
133
+
134
+ @property
135
+ def config(self):
136
+ # return the associated transformers.AutoConfig for the given pretrained model.
137
+ return self._config
138
+
139
+ @property
140
+ def tokenizer(self):
141
+ return self._tokenizer
142
+
143
+ @property
144
+ def model(self):
145
+ # returns the model, unwrapping it if using Accelerate
146
+ if hasattr(self, "accelerator"):
147
+ return self.accelerator.unwrap_model(self._model)
148
+ else:
149
+ return self._model
150
+
151
+ @property
152
+ def eot_token_id(self):
153
+ return self.tokenizer.eos_token_id
154
+
155
+ @property
156
+ def max_length(self):
157
+ return self._max_length
158
+
159
+ @property
160
+ def batch_size(self):
161
+ return self.batch_size_per_gpu
162
+
163
+ @property
164
+ def device(self):
165
+ return self._device
166
+
167
+ @property
168
+ def rank(self):
169
+ return self._rank
170
+
171
+ @property
172
+ def world_size(self):
173
+ return self._world_size
174
+
175
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
176
+ raise NotImplementedError("Loglikelihood is not implemented for Qwen2.5_VL")
177
+
178
+ def flatten(self, input):
179
+ new_list = []
180
+ for i in input:
181
+ for j in i:
182
+ new_list.append(j)
183
+ return new_list
184
+
185
+ def generate_until(self, requests: List[Instance]) -> List[str]:
186
+ res = []
187
+
188
+ def get_uuid(task, split, doc_id):
189
+ return f"{task}___{split}___{doc_id}"
190
+
191
+ def _collate(x):
192
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
193
+ # - time estimates will always be over not underestimates, which is more useful for planning
194
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
195
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
196
+ # automatic adaptive batches much much easier to implement
197
+ # - any OOMs will happen right away rather than near the end
198
+ toks = self.tokenizer.encode(x[0])
199
+ return -len(toks), x[0]
200
+
201
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
202
+ # we group requests by their generation_kwargs,
203
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
204
+ # in the same batch.
205
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
206
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
207
+ for chunk in chunks:
208
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
209
+
210
+ task = task[0]
211
+ split = split[0]
212
+ # if self.continual_mode and self.cache_mode == "resume":
213
+ # doc_uuid = get_uuid(task, split, doc_id)
214
+ # # print(doc_uuid)
215
+ # if doc_uuid in self.response_cache:
216
+ # content = self.response_cache[doc_uuid]
217
+ # if content:
218
+ # res.extend(content)
219
+ # pbar.update(1)
220
+ # continue
221
+ visuals = [doc_to_visual[i](self.task_dict[task][split][ids]) for i, ids in enumerate(doc_id)]
222
+
223
+ gen_kwargs = all_gen_kwargs[0]
224
+
225
+ # Set default values for until and max_new_tokens
226
+ until = [self.tokenizer.decode(self.eot_token_id)]
227
+
228
+ # Update values from gen_kwargs if present
229
+ if "until" in gen_kwargs:
230
+ until = gen_kwargs.pop("until")
231
+ if isinstance(until, str):
232
+ until = [until]
233
+ elif not isinstance(until, list):
234
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
235
+
236
+ if isinstance(contexts, tuple):
237
+ contexts = list(contexts)
238
+
239
+ # for i in range(len(contexts)):
240
+ # for j in range(32):
241
+ # if f"<image {j}>" in contexts[i]:
242
+ # contexts[i] = contexts[i].replace(f"<image {j}>", "<image>")
243
+ # if f"\\<image {j}\\>" in contexts[i]:
244
+ # contexts[i] = contexts[i].replace(f"\\<image {j}\\>", "<image>")
245
+ # if "<image>" in contexts[i]:
246
+ # contexts[i] = contexts[i].replace("<image>", "")
247
+ # print(contexts[i])
248
+
249
+ messages = []
250
+ processed_visuals = []
251
+ for i, context in enumerate(contexts):
252
+ # context += "\nPlease think step by step."
253
+
254
+ # print("context", context)
255
+
256
+ if "<image 1>" in context:
257
+ context = context.split("<image 1>")
258
+ # print(json.dumps(context, indent=4))
259
+ else:
260
+ context = [context]
261
+
262
+ message = [{"role": "system", "content": "You are a helpful assistant."}]
263
+
264
+ if len(visuals) > 0:
265
+ visual = visuals[i] if i < len(visuals) else None
266
+ # print("visuals", visual)
267
+ if isinstance(visual, Image.Image):
268
+ visual = [visual]
269
+ if isinstance(visual, (list, tuple)) and isinstance(visual[0], str):
270
+ assert len(visual) == 1, f"Expected a single video file but got {len(visual)} files"
271
+ visual = visual[0]
272
+ if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")): # Video file
273
+ if self.use_custom_video_loader:
274
+ try:
275
+ visual = read_video_pyav_base64(
276
+ visual,
277
+ num_frm=self.max_num_frames,
278
+ fps=self.fps,
279
+ img_format="JPEG",
280
+ max_image_size=self.max_image_size,
281
+ )
282
+ except Exception as e:
283
+ eval_logger.error(f"Failed to load video: {visual}. Error: {e}")
284
+ continue
285
+
286
+ image_contents = list(map(lambda x: f"data:image/jpeg;base64,{x}", visual))
287
+ if len(context) == 2:
288
+ if len(image_contents) == 1:
289
+ message.append(
290
+ {
291
+ "role": "user",
292
+ "content": [{"type": "video", "video": image_contents[:-1]}, {"type": "text", "text": context[0]}, {"type": "image", "image": image_contents[-1]}, {"type": "text", "text": context[1]}],
293
+ }
294
+ )
295
+ else:
296
+ message.append({"role": "user", "content": [{"type": "text", "text": context[0]}, {"type": "image", "image": image_contents[-1]}, {"type": "text", "text": context[1]}]})
297
+ else:
298
+ message.append({"role": "user", "content": [{"type": "video", "video": image_contents}, {"type": "text", "text": context}]})
299
+ # with open("alb.json", "w") as f:
300
+ # json.dump(message, f, indent=4)
301
+ else:
302
+ vr = decord.VideoReader(visual)
303
+ first_frame = vr[0].asnumpy()
304
+ height, width = first_frame.shape[:2]
305
+ # max_pixels = height * width
306
+ message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": self.max_pixels}, {"type": "text", "text": context}]})
307
+ elif isinstance(visual, Image.Image): # Single image
308
+ base64_image = visual.convert("RGB")
309
+ buffer = BytesIO()
310
+ base64_image.save(buffer, format="JPEG")
311
+ base64_bytes = base64.b64encode(buffer.getvalue())
312
+ base64_string = base64_bytes.decode("utf-8")
313
+ message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
314
+ elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual): # Multiple images
315
+ image_content = []
316
+ i = 0
317
+ for v in visual:
318
+ base64_image = v.convert("RGB")
319
+ buffer = BytesIO()
320
+ base64_image.save(buffer, format="JPEG")
321
+ base64_bytes = base64.b64encode(buffer.getvalue())
322
+ base64_string = base64_bytes.decode("utf-8")
323
+ image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
324
+ v.save(f"test_{i}.jpg")
325
+ i += 1
326
+ # message.append({"role": "user", "content": image_content + [{"type": "text", "text": context}]})
327
+ assert len(image_content) + 1 == len(context), f"Number of images and context do not match, {len(image_content)} images and {len(context)} context\n{json.dumps(context)}"
328
+ content = []
329
+ for i in range(len(image_content)):
330
+ content.append({"type": "text", "text": context[i]})
331
+ content.append(image_content[i])
332
+ content.append({"type": "text", "text": context[-1]})
333
+ # print("content", content)
334
+ else:
335
+ message.append({"role": "user", "content": [{"type": "text", "text": context}]})
336
+ else:
337
+ message.append({"role": "user", "content": [{"type": "text", "text": context}]})
338
+
339
+ messages.append(message)
340
+ # print(messages)
341
+
342
+ texts = [self.processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True) for msg in messages]
343
+ image_inputs, video_inputs = process_vision_info(messages)
344
+
345
+ inputs = self.processor(
346
+ text=texts,
347
+ images=image_inputs,
348
+ videos=video_inputs,
349
+ padding=True,
350
+ # fps=self.fps,
351
+ return_tensors="pt",
352
+ )
353
+
354
+ if self.device_map == "auto":
355
+ inputs = inputs.to("cuda")
356
+ else:
357
+ inputs = inputs.to(self.device)
358
+
359
+ if "max_new_tokens" not in gen_kwargs:
360
+ gen_kwargs["max_new_tokens"] = 128
361
+ if "temperature" not in gen_kwargs:
362
+ gen_kwargs["temperature"] = 0
363
+ if "top_p" not in gen_kwargs:
364
+ gen_kwargs["top_p"] = None
365
+ if "num_beams" not in gen_kwargs:
366
+ gen_kwargs["num_beams"] = 1
367
+
368
+ pad_token_id = self.tokenizer.pad_token_id
369
+
370
+ cont = self.model.generate(
371
+ **inputs,
372
+ eos_token_id=self.tokenizer.eos_token_id,
373
+ pad_token_id=pad_token_id,
374
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
375
+ temperature=gen_kwargs["temperature"],
376
+ top_p=gen_kwargs["top_p"],
377
+ num_beams=gen_kwargs["num_beams"],
378
+ max_new_tokens=gen_kwargs["max_new_tokens"],
379
+ use_cache=self.use_cache,
380
+ )
381
+ # cont = self.model.generate(**inputs, max_new_tokens=1024)
382
+
383
+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
384
+ answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
385
+ for i, ans in enumerate(answers):
386
+ answers[i] = ans
387
+
388
+ content = []
389
+ for ans, context in zip(answers, contexts):
390
+ res.append(ans)
391
+ content.append(ans)
392
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
393
+ pbar.update(1)
394
+ # reorder this group of results back to original unsorted form
395
+
396
+ # if self.continual_mode is True: # Cache the response
397
+ # doc_uuid = get_uuid(task, split, doc_id)
398
+ # self.response_cache[doc_uuid] = content
399
+ # with open(self.response_persistent_file, "w") as f:
400
+ # json.dump(self.response_cache, f)
401
+
402
+ res = re_ords.get_original(res)
403
+
404
+ pbar.close()
405
+ return res
406
+
407
+ def generate_until_multi_round(self, requests) -> List[str]:
408
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/qwen2_audio.py ADDED
@@ -0,0 +1,284 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ from io import BytesIO
3
+ from typing import List, Optional, Tuple, Union
4
+
5
+ import decord
6
+ import torch
7
+ from accelerate import Accelerator, DistributedType
8
+ from loguru import logger as eval_logger
9
+ from tqdm import tqdm
10
+ from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
11
+
12
+ from lmms_eval import utils
13
+ from lmms_eval.api.instance import Instance
14
+ from lmms_eval.api.model import lmms
15
+ from lmms_eval.api.registry import register_model
16
+ from lmms_eval.models.model_utils.audio_processing import downsample_audio
17
+
18
+
19
+ @register_model("qwen2_audio")
20
+ class Qwen2_Audio(lmms):
21
+ """
22
+ Qwen2_Audio Model
23
+ "https://github.com/QwenLM/Qwen2-Audio"
24
+ """
25
+
26
+ def __init__(
27
+ self,
28
+ pretrained: str = "Qwen/Qwen2-Audio-7B", # Qwen/Qwen2-Audio-7B-Instruct
29
+ device: Optional[str] = "cuda",
30
+ device_map: Optional[str] = "cuda",
31
+ batch_size: Optional[Union[int, str]] = 1,
32
+ use_cache=True,
33
+ add_generation_prompt: bool = True,
34
+ add_system_prompt: bool = True,
35
+ simple_prompt: bool = False,
36
+ **kwargs,
37
+ ) -> None:
38
+ super().__init__()
39
+ # Do not use kwargs for now
40
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
41
+
42
+ accelerator = Accelerator()
43
+ self.add_generation_prompt = add_generation_prompt
44
+ self.add_system_prompt = add_system_prompt
45
+ # If using simple prompt, only add "<|audio_bos|><|AUDIO|><|audio_eos|>"
46
+ # and then prompt to align with original Qwen2 Audio
47
+ self.simple_prompt = simple_prompt
48
+ if accelerator.num_processes > 1:
49
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
50
+ self.device_map = f"cuda:{accelerator.local_process_index}"
51
+ elif accelerator.num_processes == 1 and device_map == "auto":
52
+ self._device = torch.device(device)
53
+ self.device_map = device_map
54
+ else:
55
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
56
+ self.device_map = f"cuda:{accelerator.local_process_index}"
57
+
58
+ self._model = Qwen2AudioForConditionalGeneration.from_pretrained(
59
+ pretrained,
60
+ torch_dtype="auto",
61
+ device_map=self.device_map,
62
+ ).eval()
63
+
64
+ self.processor = AutoProcessor.from_pretrained(pretrained)
65
+ self.processor.tokenizer.padding_side = "left"
66
+ self._tokenizer = self.processor.tokenizer
67
+
68
+ if not self.add_system_prompt:
69
+ # Overwrite chat template to exclude system prompt
70
+ self.processor.chat_template = (
71
+ "{% set audio_count = namespace(value=0) %}"
72
+ "{% for message in messages %}"
73
+ "<|im_start|>{{ message['role'] }}\n"
74
+ "{% if message['content'] is string %}"
75
+ "{{ message['content'] }}<|im_end|>\n"
76
+ "{% else %}"
77
+ "{% for content in message['content'] %}"
78
+ "{% if 'audio' in content or 'audio_url' in content %}"
79
+ "{% set audio_count.value = audio_count.value + 1 %}"
80
+ "Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
81
+ "{% elif 'text' in content %}"
82
+ "{{ content['text'] }}"
83
+ "{% endif %}"
84
+ "{% endfor %}"
85
+ "<|im_end|>\n"
86
+ "{% endif %}"
87
+ "{% endfor %}"
88
+ "{% if add_generation_prompt %}"
89
+ "<|im_start|>assistant\n"
90
+ "{% endif %}"
91
+ )
92
+
93
+ self._config = self.model.config
94
+ self.batch_size_per_gpu = int(batch_size)
95
+ self.use_cache = use_cache
96
+
97
+ if accelerator.num_processes > 1:
98
+ assert accelerator.distributed_type in [
99
+ DistributedType.FSDP,
100
+ DistributedType.MULTI_GPU,
101
+ ], "Unsupported distributed type provided. Only DDP and FSDP are supported."
102
+ if accelerator.distributed_type == DistributedType.FSDP:
103
+ self._model = accelerator.prepare(self.model)
104
+ else:
105
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
106
+ self.accelerator = accelerator
107
+ if self.accelerator.is_local_main_process:
108
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
109
+ self._rank = self.accelerator.local_process_index
110
+ self._world_size = self.accelerator.num_processes
111
+ else:
112
+ self.model.to(self._device)
113
+ self._rank = 0
114
+ self._world_size = 1
115
+
116
+ @property
117
+ def config(self):
118
+ # return the associated transformers.AutoConfig for the given pretrained model.
119
+ return self._config
120
+
121
+ @property
122
+ def tokenizer(self):
123
+ return self._tokenizer
124
+
125
+ @property
126
+ def model(self):
127
+ # returns the model, unwrapping it if using Accelerate
128
+ if hasattr(self, "accelerator"):
129
+ return self.accelerator.unwrap_model(self._model)
130
+ else:
131
+ return self._model
132
+
133
+ @property
134
+ def eot_token_id(self):
135
+ return self.tokenizer.eos_token_id
136
+
137
+ @property
138
+ def max_length(self):
139
+ return self._max_length
140
+
141
+ @property
142
+ def batch_size(self):
143
+ return self.batch_size_per_gpu
144
+
145
+ @property
146
+ def device(self):
147
+ return self._device
148
+
149
+ @property
150
+ def rank(self):
151
+ return self._rank
152
+
153
+ @property
154
+ def world_size(self):
155
+ return self._world_size
156
+
157
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
158
+ raise NotImplementedError("Loglikelihood is not implemented for Qwen2_Audio")
159
+
160
+ def flatten(self, input):
161
+ new_list = []
162
+ for i in input:
163
+ for j in i:
164
+ new_list.append(j)
165
+ return new_list
166
+
167
+ def generate_until(self, requests: List[Instance]) -> List[str]:
168
+ res = []
169
+
170
+ def _collate(x):
171
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
172
+ # - time estimates will always be over not underestimates, which is more useful for planning
173
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
174
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
175
+ # automatic adaptive batches much much easier to implement
176
+ # - any OOMs will happen right away rather than near the end
177
+ toks = self.tokenizer.encode(x[0])
178
+ return -len(toks), x[0]
179
+
180
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
181
+ # we group requests by their generation_kwargs,
182
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
183
+ # in the same batch.
184
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
185
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
186
+ for chunk in chunks:
187
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
188
+ task = task[0]
189
+ split = split[0]
190
+ batched_audios = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
191
+ flattened_audios = self.flatten(batched_audios)
192
+
193
+ # we assume all gen kwargs in the batch are the same
194
+ # this is safe to assume because the `grouper` object ensures it.
195
+ gen_kwargs = all_gen_kwargs[0]
196
+
197
+ # Set default values for until and max_new_tokens
198
+ until = [self.tokenizer.decode(self.eot_token_id)]
199
+
200
+ # Update values from gen_kwargs if present
201
+ if "until" in gen_kwargs:
202
+ until = gen_kwargs.pop("until")
203
+ if isinstance(until, str):
204
+ until = [until]
205
+ elif not isinstance(until, list):
206
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
207
+
208
+ # contexts = "<|audio_bos|><|AUDIO|><|audio_eos|>" + contexts
209
+
210
+ if isinstance(contexts, tuple):
211
+ contexts = list(contexts)
212
+
213
+ if not self.simple_prompt:
214
+ conversations = []
215
+ for idx, context in enumerate(contexts):
216
+ conv = [{"role": "user", "content": []}]
217
+ for _ in batched_audios[idx]:
218
+ # This placeholder is just use to make chat template work
219
+ # We already have the sampled audio array
220
+ conv[0]["content"].append({"type": "audio", "audio_url": "placeholder.wav"})
221
+ conv[0]["content"].append({"type": "text", "text": context})
222
+ conversations.append(conv)
223
+
224
+ text = [self.processor.apply_chat_template(conversation, add_generation_prompt=self.add_generation_prompt, tokenize=False) for conversation in conversations]
225
+ else:
226
+ text = ["<|audio_bos|><|AUDIO|><|audio_eos|>" + context for context in contexts]
227
+ audios = [downsample_audio(audio["array"], audio["sampling_rate"], self.processor.feature_extractor.sampling_rate) for audio in flattened_audios]
228
+
229
+ inputs = self.processor(text=text, audios=audios, return_tensors="pt", padding=True, sampling_rate=self.processor.feature_extractor.sampling_rate)
230
+
231
+ if self.device_map == "auto":
232
+ inputs = inputs.to("cuda")
233
+ else:
234
+ inputs = inputs.to(self.device)
235
+
236
+ if "max_new_tokens" not in gen_kwargs:
237
+ gen_kwargs["max_new_tokens"] = 256
238
+ if "temperature" not in gen_kwargs:
239
+ gen_kwargs["temperature"] = 0
240
+ if "top_p" not in gen_kwargs:
241
+ gen_kwargs["top_p"] = None
242
+ if "num_beams" not in gen_kwargs:
243
+ gen_kwargs["num_beams"] = 1
244
+
245
+ try:
246
+ cont = self.model.generate(
247
+ **inputs,
248
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
249
+ temperature=gen_kwargs["temperature"],
250
+ top_p=gen_kwargs["top_p"],
251
+ num_beams=gen_kwargs["num_beams"],
252
+ max_new_tokens=gen_kwargs["max_new_tokens"],
253
+ min_new_tokens=1,
254
+ use_cache=self.use_cache,
255
+ )
256
+
257
+ # cont = self.model.generate(**inputs, max_new_tokens=256, min_new_tokens=1, do_sample=False)
258
+
259
+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
260
+ # generated_ids_trimmed = cont[:, inputs.input_ids.size(1):]
261
+ answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
262
+ for i, ans in enumerate(answers):
263
+ for term in until:
264
+ if len(term) > 0:
265
+ ans = ans.split(term)[0]
266
+ answers[i] = ans
267
+
268
+ except Exception as e:
269
+ eval_logger.debug(f"Error while generating: {e}. It is possibly due to blank audio in {contexts}")
270
+ answers = [""] * len(contexts)
271
+
272
+ for ans, context in zip(answers, contexts):
273
+ res.append(ans)
274
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
275
+ pbar.update(1)
276
+
277
+ # reorder this group of results back to original unsorted form
278
+ res = re_ords.get_original(res)
279
+
280
+ pbar.close()
281
+ return res
282
+
283
+ def generate_until_multi_round(self, requests) -> List[str]:
284
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/qwen2_vl.py ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ from io import BytesIO
3
+ from typing import List, Optional, Tuple, Union
4
+
5
+ import decord
6
+ import numpy as np
7
+ import torch
8
+ from accelerate import Accelerator, DistributedType
9
+ from loguru import logger as eval_logger
10
+ from PIL import Image
11
+ from tqdm import tqdm
12
+ from transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration
13
+
14
+ from lmms_eval import utils
15
+ from lmms_eval.api.instance import Instance
16
+ from lmms_eval.api.model import lmms
17
+ from lmms_eval.api.registry import register_model
18
+ from lmms_eval.models.model_utils.load_video import load_video_decord
19
+
20
+ try:
21
+ from qwen_vl_utils import process_vision_info
22
+ except ImportError:
23
+ eval_logger.warning("Failed to import qwen_vl_utils; Please install it via `pip install qwen-vl-utils`")
24
+
25
+
26
+ @register_model("qwen2_vl")
27
+ class Qwen2_VL(lmms):
28
+ """
29
+ Qwen2_VL Model
30
+ "https://github.com/QwenLM/Qwen2-VL"
31
+ """
32
+
33
+ def __init__(
34
+ self,
35
+ pretrained: str = "Qwen/Qwen2-VL-7B-Instruct",
36
+ device: Optional[str] = "cuda",
37
+ device_map: Optional[str] = "cuda",
38
+ batch_size: Optional[Union[int, str]] = 1,
39
+ use_cache=True,
40
+ use_flash_attention_2: Optional[bool] = False,
41
+ max_pixels: int = 12845056,
42
+ min_pixels: int = 3136,
43
+ max_num_frames: int = 32,
44
+ **kwargs,
45
+ ) -> None:
46
+ super().__init__()
47
+ # Do not use kwargs for now
48
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
49
+
50
+ accelerator = Accelerator()
51
+ if accelerator.num_processes > 1:
52
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
53
+ self.device_map = f"cuda:{accelerator.local_process_index}"
54
+ elif accelerator.num_processes == 1 and device_map == "auto":
55
+ self._device = torch.device(device)
56
+ self.device_map = device_map
57
+ else:
58
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
59
+ self.device_map = f"cuda:{accelerator.local_process_index}"
60
+
61
+ if use_flash_attention_2:
62
+ self._model = Qwen2VLForConditionalGeneration.from_pretrained(
63
+ pretrained,
64
+ torch_dtype="auto",
65
+ device_map=self.device_map,
66
+ attn_implementation="flash_attention_2",
67
+ ).eval()
68
+ else:
69
+ self._model = Qwen2VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
70
+ self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
71
+ self.max_pixels = max_pixels
72
+ self.min_pixels = min_pixels
73
+ self.max_num_frames = max_num_frames
74
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
75
+
76
+ self._config = self.model.config
77
+ self.batch_size_per_gpu = int(batch_size)
78
+ self.use_cache = use_cache
79
+
80
+ if accelerator.num_processes > 1:
81
+ assert accelerator.distributed_type in [
82
+ DistributedType.FSDP,
83
+ DistributedType.MULTI_GPU,
84
+ ], "Unsupported distributed type provided. Only DDP and FSDP are supported."
85
+ if accelerator.distributed_type == DistributedType.FSDP:
86
+ self._model = accelerator.prepare(self.model)
87
+ else:
88
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
89
+ self.accelerator = accelerator
90
+ if self.accelerator.is_local_main_process:
91
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
92
+ self._rank = self.accelerator.local_process_index
93
+ self._world_size = self.accelerator.num_processes
94
+ else:
95
+ self._rank = 0
96
+ self._world_size = 1
97
+
98
+ @property
99
+ def config(self):
100
+ # return the associated transformers.AutoConfig for the given pretrained model.
101
+ return self._config
102
+
103
+ @property
104
+ def tokenizer(self):
105
+ return self._tokenizer
106
+
107
+ @property
108
+ def model(self):
109
+ # returns the model, unwrapping it if using Accelerate
110
+ if hasattr(self, "accelerator"):
111
+ return self.accelerator.unwrap_model(self._model)
112
+ else:
113
+ return self._model
114
+
115
+ @property
116
+ def eot_token_id(self):
117
+ return self.tokenizer.eos_token_id
118
+
119
+ @property
120
+ def max_length(self):
121
+ return self._max_length
122
+
123
+ @property
124
+ def batch_size(self):
125
+ return self.batch_size_per_gpu
126
+
127
+ @property
128
+ def device(self):
129
+ return self._device
130
+
131
+ @property
132
+ def rank(self):
133
+ return self._rank
134
+
135
+ @property
136
+ def world_size(self):
137
+ return self._world_size
138
+
139
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
140
+ raise NotImplementedError("Loglikelihood is not implemented for Qwen2_VL")
141
+
142
+ def flatten(self, input):
143
+ new_list = []
144
+ for i in input:
145
+ for j in i:
146
+ new_list.append(j)
147
+ return new_list
148
+
149
+ def generate_until(self, requests: List[Instance]) -> List[str]:
150
+ res = []
151
+
152
+ def _collate(x):
153
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
154
+ # - time estimates will always be over not underestimates, which is more useful for planning
155
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
156
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
157
+ # automatic adaptive batches much much easier to implement
158
+ # - any OOMs will happen right away rather than near the end
159
+ toks = self.tokenizer.encode(x[0])
160
+ return -len(toks), x[0]
161
+
162
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
163
+ # we group requests by their generation_kwargs,
164
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
165
+ # in the same batch.
166
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
167
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
168
+ for chunk in chunks:
169
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
170
+ task = task[0]
171
+ split = split[0]
172
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
173
+ if None in visuals:
174
+ visuals = []
175
+ else:
176
+ visuals = self.flatten(visuals)
177
+
178
+ gen_kwargs = all_gen_kwargs[0]
179
+
180
+ # Set default values for until and max_new_tokens
181
+ until = [self.tokenizer.decode(self.eot_token_id)]
182
+
183
+ # Update values from gen_kwargs if present
184
+ if "until" in gen_kwargs:
185
+ until = gen_kwargs.pop("until")
186
+ if isinstance(until, str):
187
+ until = [until]
188
+ elif not isinstance(until, list):
189
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
190
+
191
+ if isinstance(contexts, tuple):
192
+ contexts = list(contexts)
193
+
194
+ for i in range(len(contexts)):
195
+ if "<image>" in contexts[i]:
196
+ contexts[i] = contexts[i].replace("<image>", "")
197
+
198
+ messages = []
199
+ processed_visuals = []
200
+ for i, context in enumerate(contexts):
201
+ if "<image>" in context:
202
+ context = context.replace("<image>", "")
203
+
204
+ message = [{"role": "system", "content": "You are a helpful assistant."}]
205
+
206
+ if len(visuals) > 0:
207
+ visual = visuals[i] if i < len(visuals) else None
208
+ if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")): # Video file
209
+ vr = decord.VideoReader(visual)
210
+ first_frame = vr[0].asnumpy()
211
+ height, width = first_frame.shape[:2]
212
+ # max_pixels = height * width
213
+ message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": self.max_pixels}, {"type": "text", "text": context}]})
214
+ elif isinstance(visual, Image.Image): # Single image
215
+ base64_image = visual.convert("RGB")
216
+ buffer = BytesIO()
217
+ base64_image.save(buffer, format="JPEG")
218
+ base64_bytes = base64.b64encode(buffer.getvalue())
219
+ base64_string = base64_bytes.decode("utf-8")
220
+ message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
221
+ elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual): # Multiple images
222
+ image_content = []
223
+ for v in visual:
224
+ base64_image = v.convert("RGB")
225
+ buffer = BytesIO()
226
+ base64_image.save(buffer, format="JPEG")
227
+ base64_bytes = base64.b64encode(buffer.getvalue())
228
+ base64_string = base64_bytes.decode("utf-8")
229
+ image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
230
+ message.append({"role": "user", "content": image_content + [{"type": "text", "text": context}]})
231
+ else:
232
+ message.append({"role": "user", "content": [{"type": "text", "text": context}]})
233
+ else:
234
+ message.append({"role": "user", "content": [{"type": "text", "text": context}]})
235
+
236
+ messages.append(message)
237
+
238
+ texts = [self.processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True) for msg in messages]
239
+ image_inputs, video_inputs = process_vision_info(messages)
240
+ if video_inputs is not None:
241
+ total_frames = video_inputs[0].shape[0]
242
+ indices = np.linspace(0, total_frames - 1, self.max_num_frames, dtype=int)
243
+ # Append the last frame index if not already included
244
+ if total_frames - 1 not in indices:
245
+ indices = np.append(indices, total_frames - 1)
246
+ video_inputs[0] = video_inputs[0][indices]
247
+ inputs = self.processor(text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt")
248
+
249
+ if self.device_map == "auto":
250
+ inputs = inputs.to("cuda")
251
+ else:
252
+ inputs = inputs.to(self.device)
253
+
254
+ if "max_new_tokens" not in gen_kwargs:
255
+ gen_kwargs["max_new_tokens"] = 128
256
+ if "temperature" not in gen_kwargs:
257
+ gen_kwargs["temperature"] = 0
258
+ if "top_p" not in gen_kwargs:
259
+ gen_kwargs["top_p"] = None
260
+ if "num_beams" not in gen_kwargs:
261
+ gen_kwargs["num_beams"] = 1
262
+
263
+ pad_token_id = self.tokenizer.pad_token_id
264
+
265
+ cont = self.model.generate(
266
+ **inputs,
267
+ eos_token_id=self.tokenizer.eos_token_id,
268
+ pad_token_id=pad_token_id,
269
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
270
+ temperature=gen_kwargs["temperature"],
271
+ top_p=gen_kwargs["top_p"],
272
+ num_beams=gen_kwargs["num_beams"],
273
+ max_new_tokens=gen_kwargs["max_new_tokens"],
274
+ use_cache=self.use_cache,
275
+ )
276
+
277
+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, cont)]
278
+ answers = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
279
+ for i, ans in enumerate(answers):
280
+ for term in until:
281
+ if len(term) > 0:
282
+ ans = ans.split(term)[0]
283
+ answers[i] = ans
284
+
285
+ for ans, context in zip(answers, contexts):
286
+ res.append(ans)
287
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), ans)
288
+ pbar.update(1)
289
+ # reorder this group of results back to original unsorted form
290
+ res = re_ords.get_original(res)
291
+
292
+ pbar.close()
293
+ return res
294
+
295
+ def generate_until_multi_round(self, requests) -> List[str]:
296
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/qwen_vl.py ADDED
@@ -0,0 +1,312 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import uuid
3
+ import warnings
4
+ from typing import List, Optional, Tuple, Union
5
+
6
+ import torch
7
+ from accelerate import Accelerator, DistributedType
8
+ from tqdm import tqdm
9
+
10
+ from lmms_eval import utils
11
+ from lmms_eval.api.instance import Instance
12
+ from lmms_eval.api.model import lmms
13
+ from lmms_eval.api.registry import register_model
14
+ from lmms_eval.models.model_utils.qwen.qwen_generate_utils import make_context
15
+
16
+ warnings.simplefilter("ignore", category=DeprecationWarning)
17
+ warnings.filterwarnings("ignore")
18
+
19
+ from loguru import logger as eval_logger
20
+ from transformers import AutoModelForCausalLM, AutoTokenizer
21
+
22
+
23
+ @register_model("qwen_vl")
24
+ class Qwen_VL(lmms):
25
+ """
26
+ Qwen_VL Model
27
+ https://github.com/QwenLM/Qwen-VL/blob/master/eval_mm/evaluate_vqa.py
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ pretrained: str = "Qwen/Qwen-VL",
33
+ device: Optional[str] = "cuda",
34
+ batch_size: Optional[Union[int, str]] = 1,
35
+ trust_remote_code: Optional[bool] = True,
36
+ use_cache=True,
37
+ **kwargs,
38
+ ) -> None:
39
+ super().__init__()
40
+ # Do not use kwargs for now
41
+ assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
42
+
43
+ accelerator = Accelerator()
44
+ if accelerator.num_processes > 1:
45
+ self._device = torch.device(f"cuda:{accelerator.local_process_index}")
46
+ else:
47
+ self._device = device
48
+ self._model = AutoModelForCausalLM.from_pretrained(pretrained, device_map=self._device, trust_remote_code=trust_remote_code).eval()
49
+ self._tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=trust_remote_code)
50
+ self.tokenizer.padding_side = "left"
51
+ self.tokenizer.pad_token_id = self.tokenizer.eod_id
52
+ self.prompt = "<img>{}</img>{}"
53
+ self._config = self._model.config
54
+ self.model.tie_weights()
55
+ self.batch_size_per_gpu = int(batch_size)
56
+ self.use_cache = use_cache
57
+ if accelerator.num_processes > 1:
58
+ assert accelerator.distributed_type in [
59
+ DistributedType.FSDP,
60
+ DistributedType.MULTI_GPU,
61
+ ], "Unsupported distributed type provided. Only DDP and FSDP are supported."
62
+ if accelerator.distributed_type == DistributedType.FSDP:
63
+ self._model = accelerator.prepare(self.model)
64
+ else:
65
+ self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
66
+ self.accelerator = accelerator
67
+ if self.accelerator.is_local_main_process:
68
+ eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
69
+ self._rank = self.accelerator.local_process_index
70
+ self._world_size = self.accelerator.num_processes
71
+ else:
72
+ self.model.to(self._device)
73
+ self._rank = 0
74
+ self._world_size = 1
75
+
76
+ @property
77
+ def config(self):
78
+ # return the associated transformers.AutoConfig for the given pretrained model.
79
+ return self._config
80
+
81
+ @property
82
+ def tokenizer(self):
83
+ return self._tokenizer
84
+
85
+ @property
86
+ def model(self):
87
+ # returns the model, unwrapping it if using Accelerate
88
+ if hasattr(self, "accelerator"):
89
+ return self.accelerator.unwrap_model(self._model)
90
+ else:
91
+ return self._model
92
+
93
+ @property
94
+ def eot_token_id(self):
95
+ # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
96
+ return self.tokenizer.eod_id
97
+
98
+ @property
99
+ def max_length(self):
100
+ return self._max_length
101
+
102
+ # should be deleted since max_new_tokens is decided by gen_kwargs not a model property
103
+ # @property
104
+ # def max_new_tokens(self) -> int:
105
+ # return 256
106
+
107
+ @property
108
+ def batch_size(self):
109
+ return self.batch_size_per_gpu
110
+
111
+ @property
112
+ def device(self):
113
+ return self._device
114
+
115
+ @property
116
+ def rank(self):
117
+ return self._rank
118
+
119
+ @property
120
+ def world_size(self):
121
+ return self._world_size
122
+
123
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
124
+ res = []
125
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
126
+
127
+ for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
128
+ # encode, pad, and truncate contexts for this batch
129
+ if type(doc_to_target) == str:
130
+ continuation = doc_to_target
131
+ else:
132
+ continuation = doc_to_target(self.task_dict[task][split][doc_id])
133
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
134
+ visuals = self.flatten(visuals)
135
+ query = []
136
+ visual_paths = []
137
+ for visual in visuals:
138
+ name = uuid.uuid4().hex.upper()[0:6]
139
+ visual.save(f"/tmp/{name}.png")
140
+ visual_paths.append(f"/tmp/{name}.png")
141
+ query.append({"image": f"/tmp/{name}.png"})
142
+
143
+ # Make a copy for query to save context (text that needs to be masked)
144
+ context_query = [_ for _ in query]
145
+ context_query.append({"text": contexts})
146
+ query.append({"text": contexts + continuation})
147
+
148
+ context_query = self.tokenizer.from_list_format(context_query)
149
+ query = self.tokenizer.from_list_format(query)
150
+
151
+ raw_contxt_text, context_tokens = make_context(
152
+ self.tokenizer, context_query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format
153
+ )
154
+ context_tokens = torch.tensor([context_tokens])
155
+
156
+ raw_continuation_text, continuation_tokens = make_context(
157
+ self.tokenizer, query, history=None, system="You are a helpful assistant", max_window_size=self.model.generation_config.max_window_size, chat_format=self.model.generation_config.chat_format
158
+ )
159
+ continuation_tokens = torch.tensor([continuation_tokens]).to(self.model.device)
160
+ attn_mask = torch.ones_like(continuation_tokens).to(self.model.device)
161
+ labels = continuation_tokens.clone().to(self.model.device)
162
+ labels[:, : context_tokens.shape[1]] = -100
163
+ with torch.inference_mode():
164
+ outputs = self.model(input_ids=continuation_tokens, labels=labels, attention_mask=attn_mask)
165
+ loss = outputs.loss
166
+ logits = outputs["logits"]
167
+ greedy_tokens = logits.argmax(dim=-1)
168
+ cont_toks = continuation_tokens[:, context_tokens.shape[1] :]
169
+ greedy_tokens = greedy_tokens[:, context_tokens.shape[1] : continuation_tokens.shape[1]] # [1, seq]
170
+ max_equal = (greedy_tokens == cont_toks).all()
171
+ res.append((float(loss.item()), bool(max_equal)))
172
+ pbar.update(1)
173
+
174
+ pbar.close()
175
+ return res
176
+
177
+ def flatten(self, input):
178
+ new_list = []
179
+ for i in input:
180
+ for j in i:
181
+ new_list.append(j)
182
+ return new_list
183
+
184
+ def generate_until(self, requests: List[Instance]) -> List[str]:
185
+ res = []
186
+
187
+ def _collate(x):
188
+ # the negative sign on len(toks) sorts descending - this has a few advantages:
189
+ # - time estimates will always be over not underestimates, which is more useful for planning
190
+ # - to know the size of a batch when going through the list, you know the first one is always the batch
191
+ # padded context length. this is useful to simplify the batching logic and more importantly to make
192
+ # automatic adaptive batches much much easier to implement
193
+ # - any OOMs will happen right away rather than near the end
194
+ toks = self.tokenizer.encode(x[0])
195
+ return -len(toks), x[0]
196
+
197
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
198
+ # we group requests by their generation_kwargs,
199
+ # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
200
+ # in the same batch.
201
+ re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
202
+ chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
203
+ for chunk in chunks:
204
+ contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
205
+ task = task[0]
206
+ split = split[0]
207
+ visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
208
+ visuals = self.flatten(visuals)
209
+ visual_paths = []
210
+ # save images to /tmp, name generated by hash function
211
+ # qwen accept image path. Have to do it here....
212
+ for visual in visuals:
213
+ name = uuid.uuid4().hex.upper()[0:6]
214
+ visual.save(f"/tmp/{name}.png")
215
+ visual_paths.append(f"/tmp/{name}.png")
216
+
217
+ # we assume all gen kwargs in the batch are the same
218
+ # this is safe to assume because the `grouper` object ensures it.
219
+ gen_kwargs = all_gen_kwargs[0]
220
+
221
+ # Set default values for until and max_new_tokens
222
+ until = [self.tokenizer.decode(self.eot_token_id)]
223
+
224
+ # Update values from gen_kwargs if present
225
+ if "until" in gen_kwargs:
226
+ until = gen_kwargs.pop("until")
227
+ if isinstance(until, str):
228
+ until = [until]
229
+ elif not isinstance(until, list):
230
+ raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
231
+
232
+ if isinstance(contexts, tuple):
233
+ contexts = list(contexts)
234
+
235
+ for i in range(len(contexts)):
236
+ if "<image>" in contexts[i]:
237
+ contexts[i] = contexts[i].replace("<image>", "")
238
+
239
+ # Similar to llava, is visual paths has len 0
240
+ # Then nothing will be executed
241
+ query = []
242
+ if len(visual_paths) == 0:
243
+ for context in contexts:
244
+ query.append({"text": context})
245
+ else:
246
+ for visual_path, context in zip(visual_paths, contexts):
247
+ query.append({"image": visual_path})
248
+ query.append({"text": context})
249
+
250
+ questions = self.tokenizer.from_list_format(query)
251
+ input_ids = self.tokenizer(questions, return_tensors="pt", padding="longest")
252
+
253
+ # preconfigure gen_kwargs with defaults
254
+ if "image_sizes" not in gen_kwargs:
255
+ try:
256
+ gen_kwargs["image_sizes"] = [visuals[0].size]
257
+ except:
258
+ gen_kwargs["image_sizes"] = None
259
+ if "max_new_tokens" not in gen_kwargs:
260
+ gen_kwargs["max_new_tokens"] = 1024
261
+ if "temperature" not in gen_kwargs:
262
+ gen_kwargs["temperature"] = 0
263
+ if "top_p" not in gen_kwargs:
264
+ gen_kwargs["top_p"] = None
265
+ if "num_beams" not in gen_kwargs:
266
+ gen_kwargs["num_beams"] = 1
267
+
268
+ pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eod_id
269
+
270
+ cont = self.model.generate(
271
+ input_ids.input_ids.to(self.device),
272
+ attention_mask=input_ids.attention_mask.to(self.device),
273
+ eos_token_id=self.tokenizer.eod_id,
274
+ pad_token_id=pad_token_id,
275
+ do_sample=True if gen_kwargs["temperature"] > 0 else False,
276
+ temperature=gen_kwargs["temperature"],
277
+ top_p=gen_kwargs["top_p"],
278
+ num_beams=gen_kwargs["num_beams"],
279
+ max_new_tokens=gen_kwargs["max_new_tokens"],
280
+ use_cache=self.use_cache,
281
+ # kwargs=gen_kwargs
282
+ )
283
+
284
+ cont_toks_list = cont.tolist()
285
+ for cont_toks, context in zip(cont_toks_list, contexts):
286
+ # discard context + left-padding toks if using causal decoder-only LMM
287
+ cont_toks = cont_toks[input_ids.input_ids.shape[1] :]
288
+ text_outputs = self.tokenizer.decode(cont_toks, skip_special_tokens=True).strip()
289
+ for term in until:
290
+ if len(term) > 0:
291
+ # ignore '' separator,
292
+ # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
293
+ text_outputs = text_outputs.split(term)[0]
294
+
295
+ res.append(text_outputs)
296
+
297
+ self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
298
+ # remove visuals from tmp
299
+ for visual_path in visual_paths:
300
+ try:
301
+ os.remove(visual_path)
302
+ except:
303
+ pass
304
+ pbar.update(1)
305
+ # reorder this group of results back to original unsorted form
306
+ res = re_ords.get_original(res)
307
+
308
+ pbar.close()
309
+ return res
310
+
311
+ def generate_until_multi_round(self, requests) -> List[str]:
312
+ raise NotImplementedError("TODO: Implement multi-round generation")
LaViDa/eval/lmms_eval/models/qwen_vl_api.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import json
3
+ import os
4
+ import tempfile
5
+ import time
6
+ from copy import deepcopy
7
+ from io import BytesIO
8
+ from typing import List, Tuple, Union
9
+
10
+ import requests as url_requests
11
+ from PIL import Image
12
+ from tqdm import tqdm
13
+
14
+ from lmms_eval import utils
15
+ from lmms_eval.api.instance import Instance
16
+ from lmms_eval.api.model import lmms
17
+ from lmms_eval.api.registry import register_model
18
+
19
+ NUM_SECONDS_TO_SLEEP = 5
20
+ from loguru import logger as eval_logger
21
+
22
+ try:
23
+ import dashscope
24
+ except:
25
+ eval_logger.debug("Can not import Dashscope")
26
+
27
+ API_KEY = os.getenv("DASHSCOPE_API_KEY", "YOUR_API_KEY")
28
+
29
+
30
+ @register_model("qwen-vl-api")
31
+ class Qwen_VL_API(lmms):
32
+ def __init__(
33
+ self,
34
+ model_version: str = "qwen-vl-max",
35
+ image_token: str = "<image>", # Use to separate interleaved image and text
36
+ system_prompt: str = "", # Whether you want some special system prompt here
37
+ tmp_folder: str = "./tmp", # Due to qwen's api restriction,
38
+ continual_mode: bool = False,
39
+ response_persistent_folder: str = None,
40
+ **kwargs,
41
+ ) -> None:
42
+ super().__init__()
43
+ self.continual_mode = continual_mode
44
+
45
+ self.model_version = model_version
46
+ self.image_token = image_token
47
+ self.system_prompt = system_prompt
48
+ self.tmp_folder = tmp_folder
49
+ if self.continual_mode:
50
+ if response_persistent_folder is None:
51
+ raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.")
52
+
53
+ os.makedirs(response_persistent_folder, exist_ok=True)
54
+ self.response_persistent_folder = response_persistent_folder
55
+ self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
56
+
57
+ if os.path.exists(self.response_persistent_file):
58
+ with open(self.response_persistent_file, "r") as f:
59
+ self.response_cache = json.load(f)
60
+ self.cache_mode = "resume"
61
+ else:
62
+ self.response_cache = {}
63
+ self.cache_mode = "start"
64
+
65
+ @property
66
+ def rank(self):
67
+ return self._rank
68
+
69
+ @property
70
+ def world_size(self):
71
+ return self._world_size
72
+
73
+ def save_image_to_temp_file(self, image):
74
+ temp_file = tempfile.NamedTemporaryFile(suffix=".jpeg", delete=True)
75
+ image.save(temp_file.name)
76
+ return temp_file
77
+
78
+ def generate_until(self, requests) -> List[str]:
79
+ res = []
80
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
81
+
82
+ for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
83
+ if self.continual_mode is True and self.cache_mode == "resume":
84
+ doc_uuid = f"{task}___{split}___{doc_id}"
85
+ if doc_uuid in self.response_cache:
86
+ response_text = self.response_cache[doc_uuid]
87
+ if response_text:
88
+ res.append(response_text)
89
+ pbar.update(1)
90
+ continue
91
+ # encode, pad, and truncate contexts for this batch
92
+ visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
93
+ visuals = self.flatten(visuals)
94
+ imgs = []
95
+
96
+ for idx, visual in enumerate(visuals):
97
+ temp_file = self.save_image_to_temp_file(visual)
98
+ imgs.append(temp_file.name)
99
+
100
+ messages = [{"role": "user", "content": []}]
101
+
102
+ if self.image_token not in contexts:
103
+ for img in imgs:
104
+ messages[0]["content"].append({"image": img})
105
+ messages[0]["content"].append({"text": contexts})
106
+ else:
107
+ contexts = contexts.split(self.image_token)
108
+
109
+ for idx, img in enumerate(imgs):
110
+ messages[0]["content"].append({"text": contexts[idx]})
111
+ messages[0]["content"].append({"image": img})
112
+ messages[0]["content"].append({"text": contexts[-1]})
113
+
114
+ if "max_new_tokens" not in gen_kwargs or gen_kwargs["max_new_tokens"] > 1500:
115
+ gen_kwargs["max_new_tokens"] = 1024
116
+ if "temperature" not in gen_kwargs:
117
+ gen_kwargs["temperature"] = 0
118
+ if "top_p" not in gen_kwargs:
119
+ gen_kwargs["top_p"] = None
120
+ if "num_beams" not in gen_kwargs:
121
+ gen_kwargs["num_beams"] = 1
122
+
123
+ for attempt in range(5):
124
+ try:
125
+ response_data = dashscope.MultiModalConversation.call(model=self.model_version, messages=messages, api_key=API_KEY, max_length=gen_kwargs["max_new_tokens"], temperature=gen_kwargs["temperature"])
126
+ break
127
+ except Exception as e:
128
+ eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
129
+ if attempt < 5 - 1: # If we have retries left, sleep and then continue to next attempt
130
+ time.sleep(NUM_SECONDS_TO_SLEEP)
131
+ else: # If this was the last attempt, log and return empty
132
+ eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
133
+ res.append("")
134
+ pbar.update(1)
135
+ continue
136
+ try:
137
+ res.append(response_data["output"]["choices"][0]["message"]["content"][0]["text"].strip())
138
+ except Exception as e:
139
+ eval_logger.error(f"Error {e} happens when parsing input.")
140
+ eval_logger.error(f"{response_data}")
141
+ res.append("")
142
+
143
+ if self.continual_mode is True: # Cache the response
144
+ doc_uuid = f"{task}___{split}___{doc_id}"
145
+ self.response_cache[doc_uuid] = res[-1]
146
+ with open(self.response_persistent_file, "w") as f:
147
+ json.dump(self.response_cache, f)
148
+ pbar.update(1)
149
+
150
+ pbar.close()
151
+
152
+ return res
153
+
154
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
155
+ assert False, "Not supported for claude"
156
+
157
+ def flatten(self, input):
158
+ new_list = []
159
+ for i in input:
160
+ for j in i:
161
+ new_list.append(j)
162
+ return new_list
163
+
164
+ def generate_until_multi_round(self, requests) -> List[str]:
165
+ raise NotImplementedError("TODO: Implement multi-round generation")