ChristinaW commited on
Commit
47a136e
·
verified ·
1 Parent(s): 32162fd

Upload folder using huggingface_hub

Browse files
config.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/work1/amd/xiwenwei/Emu/Emu2/Emu2",
3
+ "architectures": [
4
+ "EmuForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "auto_map": {
9
+ "AutoConfig": "configuration_emu.EmuConfig",
10
+ "AutoModelForCausalLM": "modeling_emu.EmuForCausalLM"
11
+ },
12
+ "bos_token_id": 1,
13
+ "d_model": 1792,
14
+ "eos_token_id": 2,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 6656,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 17920,
19
+ "llama_config_path": "/share/project/quansun/models/llm_models/llama/hf/llama-30b",
20
+ "max_position_embeddings": 2048,
21
+ "model_version": "base",
22
+ "num_attention_heads": 52,
23
+ "num_hidden_layers": 60,
24
+ "num_key_value_heads": 52,
25
+ "pad_token_id": 32000,
26
+ "pretraining_tp": 1,
27
+ "rms_norm_eps": 1e-06,
28
+ "rope_scaling": null,
29
+ "rope_theta": 10000.0,
30
+ "tie_word_embeddings": false,
31
+ "torch_dtype": "bfloat16",
32
+ "transformers_version": "4.44.0",
33
+ "use_cache": true,
34
+ "vision_config": {
35
+ "drop_path_rate": 0,
36
+ "eva_model_name": "eva-clip-E-14-plus",
37
+ "head_width": 112,
38
+ "image_size": 448,
39
+ "intermediate_size": 15360,
40
+ "layer_norm_eps": 1e-06,
41
+ "layers": 64,
42
+ "mlp_ratio": 8.571428571428571,
43
+ "n_query": 64,
44
+ "patch_size": 14,
45
+ "postnorm": true,
46
+ "qkv_bias": true,
47
+ "v_query": 64,
48
+ "width": 1792,
49
+ "xattn": false
50
+ },
51
+ "vocab_size": 32272
52
+ }
configuration_emu.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Literal
2
+ from transformers import PretrainedConfig
3
+
4
+
5
+ class EmuConfig(PretrainedConfig):
6
+ _auto_class = "AutoConfig"
7
+
8
+ def __init__(
9
+ self,
10
+ vocab_size=32000,
11
+ hidden_size=4096,
12
+ intermediate_size=11008,
13
+ num_hidden_layers=32,
14
+ num_attention_heads=32,
15
+ hidden_act='silu',
16
+ max_position_embeddings=2048,
17
+ initializer_range=0.02,
18
+ rms_norm_eps=1e-06,
19
+ model_version: Literal["base", "chat"] = "base",
20
+ pad_token_id=0,
21
+ bos_token_id=1,
22
+ eos_token_id=2,
23
+ tie_word_embeddings=False,
24
+ use_cache=True,
25
+ pretraining_tp=1,
26
+ rope_theta=10000.0,
27
+ rope_scaling=None,
28
+ attention_bias=False,
29
+ attention_dropout=0.0,
30
+ **kwargs,
31
+ ):
32
+ self.hidden_size = hidden_size
33
+ self.intermediate_size = intermediate_size
34
+ self.num_attention_heads = num_attention_heads
35
+ self.max_position_embeddings = max_position_embeddings
36
+ self.rms_norm_eps = rms_norm_eps
37
+ self.initializer_range = initializer_range
38
+ self.vocab_size = vocab_size
39
+ self.num_hidden_layers = num_hidden_layers
40
+ self.hidden_act = hidden_act
41
+ self.model_version = model_version
42
+ self.use_cache = use_cache
43
+ self.pretraining_tp = pretraining_tp
44
+ self.use_cache = use_cache
45
+ self.rope_theta = rope_theta
46
+ self.rope_scaling = rope_scaling
47
+ self._rope_scaling_validation()
48
+ self.attention_bias = attention_bias
49
+ self.attention_dropout = attention_dropout
50
+ super().__init__(
51
+ pad_token_id=pad_token_id,
52
+ bos_token_id=bos_token_id,
53
+ eos_token_id=eos_token_id,
54
+ tie_word_embeddings=tie_word_embeddings,
55
+ **kwargs,
56
+ )
57
+
58
+ def _rope_scaling_validation(self):
59
+ """
60
+ Validate the `rope_scaling` configuration.
61
+ """
62
+ if self.rope_scaling is None:
63
+ return
64
+
65
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
66
+ raise ValueError(
67
+ "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
68
+ f"got {self.rope_scaling}"
69
+ )
70
+ rope_scaling_type = self.rope_scaling.get("type", None)
71
+ rope_scaling_factor = self.rope_scaling.get("factor", None)
72
+ if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
73
+ raise ValueError(
74
+ f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
75
+ )
76
+ if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
77
+ raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")
constants.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ EVA_IMAGE_SIZE = 448
2
+ OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
3
+ OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
4
+
5
+ DEFAULT_IMAGE_FILE_SUFFIX = ['jpg', '0.png', 'png', 'jpeg', 'webp']
6
+ DEFAULT_TEXT_FILE_SUFFIX = ['txt', '0.txt']
7
+
8
+ IGNORE_INDEX = -100
9
+
10
+ # special tokens
11
+ # START
12
+ DEFAULT_PAD_TOKEN = "[PAD]"
13
+ DEFAULT_BOS_TOKEN = '<s>'
14
+ DEFAULT_EOS_TOKEN = '</s>'
15
+ DEFAULT_UNK_TOKEN = "<unk>"
16
+
17
+ DEFAULT_IMG_TOKEN = "[IMG]"
18
+ DEFAULT_IMG_END_TOKEN = "[/IMG]"
19
+ DEFAULT_IMAGE_TOKEN = "<image>"
20
+ DEFAULT_gIMG_TOKEN = "[gIMG]"
21
+ DEFAULT_gIMG_END_TOKEN = "[/gIMG]"
22
+ DEFAULT_EOC_TOKEN = "[EOC]"
23
+ DEFAULT_VIDEO_TOKEN = "[VIDEO]"
24
+
25
+ GRD_SYMBOL = "<grounding>"
26
+ BOP_SYMBOL = "<phrase>"
27
+ EOP_SYMBOL = "</phrase>"
28
+ BOO_SYMBOL = "<object>"
29
+ EOO_SYMBOL = "</object>"
30
+ DOM_SYMBOL = "</delimiter_of_multi_objects/>"
31
+
32
+ REC_SYMBOL = "<REC>"
33
+
34
+ USER_TOKEN = "[USER]"
35
+ ASSISTANT_TOKEN = "[ASSISTANT]"
36
+ # END
37
+
38
+ # special token id
39
+ # START
40
+ IMAGE = 32003
41
+ BOI = 32001
42
+
43
+ # END
44
+
45
+ DEFAULT_IMG_PLACEHOLDER = "[<IMG_PLH>]"
46
+ DEFAULT_VID_PLACEHOLDER = "[<VID_PLH>]"
47
+ FAKE_VIDEO_END_TOKEN = "[/VIDEO]"
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32000,
6
+ "transformers_version": "4.44.0"
7
+ }
model-00001-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff555a578375fecd46ccaadc20f0476923d9b6ef7461813bdf939054c6e499f1
3
+ size 4977365880
model-00002-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10ee1fb3ae4aef1232a518f957ae7482600fe47b366aab38d7b011e262c007e4
3
+ size 4984263544
model-00003-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2badd7166b95f1cd0ebef3006609ac47902ea07e5c0ebcc1d9b016436123d45a
3
+ size 4873397680
model-00004-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e06073fdb9a20b017ffe616903bad73ae8498ffd1923ecd47691b0dce4d60a76
3
+ size 4996080424
model-00005-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ac1eef18f28391d67455e9d969b2a54dd0e7bf2c89a578b3cdfd06859cb516c
3
+ size 4873370832
model-00006-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3e0df7a307afc57ddb6f452385683e74f944d39856f2ae4ab54155ab7d7be27
3
+ size 4934738904
model-00007-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7110d5d479837da635e07b861e7d712d9835703cd88b077cb6ce2d72aa1a97fa
3
+ size 4934712392
model-00008-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a4d4965a80652582e65e54f138585a636594b9415fb9c69811315c635429f49
3
+ size 4873397728
model-00009-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f65859dc68deda36f3168cb4b14f051b5d0a9e9c61602a330440bcce9e33fce
3
+ size 4996080464
model-00010-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0bf5f0b1ed6caa1d053f30f42d48892c04b186664275e66b7ce1ac1f6667e02
3
+ size 4873370832
model-00011-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a102ef2bcc907c1ab7bf5e7f1c296cc6192949665aa01a1a5495367c6550520d
3
+ size 4934738904
model-00012-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4e942071eb0ee5ddd1bc2a3f3439d369bb969ec85137157e168afac27630326
3
+ size 4934712392
model-00013-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd01be0e40a84601f80a8509334c21d996d7d55b0d9a90c7f5aac38bb4327a21
3
+ size 4873397728
model-00014-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25c8880a709f2839776ea93387000e470f3f4f259127b385a067cc282460429e
3
+ size 4996080464
model-00015-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89d80e0438198679c04efe4fe3fe4acd4db8bbbd64f7457a56a9da6ac27d771e
3
+ size 4757728872
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
modeling_emu.py ADDED
@@ -0,0 +1,913 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # from functools import partial
2
+ # from typing import Any, List, Optional, Mapping, Callable
3
+ # from collections import OrderedDict
4
+ # from argparse import Namespace
5
+ # import torch
6
+ # from torch import nn
7
+ # import torch.nn.functional as F
8
+ # import torchvision.transforms as T
9
+ # import PIL
10
+ # import transformers
11
+ # from transformers import PreTrainedModel, PreTrainedTokenizer
12
+
13
+ # from .configuration_emu import EmuConfig
14
+ # from .constants import *
15
+ # from .modeling_llama import LlamaForCausalLM
16
+ # from .visual import EVAVisionTransformer
17
+
18
+
19
+ # class EmuPreTrainedModel(PreTrainedModel):
20
+ # config_class = EmuConfig
21
+ # base_model_prefix = "model"
22
+ # supports_gradient_checkpointing = False
23
+ # _no_split_modules = ["LlamaDecoderLayer", "Block"]
24
+ # _skip_keys_device_placement = "past_key_values"
25
+
26
+ # def _init_weights(self, module):
27
+ # std = self.config.initializer_range
28
+ # if isinstance(module, nn.Linear):
29
+ # module.weight.data.normal_(mean=0.0, std=std)
30
+ # if module.bias is not None:
31
+ # module.bias.data.zero_()
32
+ # elif isinstance(module, nn.Embedding):
33
+ # module.weight.data.normal_(mean=0.0, std=std)
34
+ # if module.padding_idx is not None:
35
+ # module.weight.data[module.padding_idx].zero_()
36
+
37
+ # class EmuForClsAndRegression(EmuPreTrainedModel):
38
+
39
+ # def __init__(self, config):
40
+ # super(EmuForClsAndRegression, self).__init__(config)
41
+
42
+ # self.lm = LlamaForCausalLM(config=config)
43
+
44
+ # self.lm.model.embed_tokens.padding_idx = config.pad_token_id
45
+
46
+ # def get_num_layers(self):
47
+ # return len(self.lm.model.layers)
48
+
49
+ # class EmuModel(EmuPreTrainedModel):
50
+
51
+ # def __init__(self, config):
52
+ # super().__init__(config)
53
+
54
+ # vision_config = Namespace(**config.vision_config)
55
+
56
+ # self.visual = EVAVisionTransformer(
57
+ # img_size=vision_config.image_size,
58
+ # patch_size=vision_config.patch_size,
59
+ # embed_dim=vision_config.width,
60
+ # depth=vision_config.layers,
61
+ # num_heads=vision_config.width // vision_config.head_width,
62
+ # mlp_ratio=vision_config.mlp_ratio,
63
+ # qkv_bias=vision_config.qkv_bias,
64
+ # drop_path_rate=vision_config.drop_path_rate,
65
+ # norm_layer=partial(nn.LayerNorm, eps=vision_config.layer_norm_eps),
66
+ # xattn=vision_config.xattn,
67
+ # postnorm=vision_config.postnorm,
68
+ # )
69
+
70
+ # self.decoder = EmuForClsAndRegression(config)
71
+
72
+ # self.gradient_checkpointing = False
73
+
74
+ # self.n_query = vision_config.n_query
75
+ # self.v_query = vision_config.v_query
76
+
77
+ # @property
78
+ # def device(self):
79
+ # return next(iter(self.parameters())).device
80
+
81
+ # @property
82
+ # def dtype(self):
83
+ # return next(iter(self.parameters())).dtype
84
+
85
+ # @torch.no_grad()
86
+ # def encode_image(self, image: torch.Tensor, *, n_query=None):
87
+ # n_query = n_query if n_query is not None else self.n_query
88
+
89
+ # image_embeds = self.visual(image)
90
+ # image_embeds = image_embeds[:, 1:, :]
91
+ # b, n, c = image_embeds.shape
92
+ # sqrt_n = int(n**0.5)
93
+ # image_embeds = image_embeds.permute(0, 2, 1).view(b, c, sqrt_n, sqrt_n)
94
+
95
+ # stride = int(sqrt_n // (n_query ** 0.5))
96
+ # image_embeds = F.avg_pool2d(image_embeds, kernel_size=(stride, stride), stride=stride)
97
+ # image_embeds = image_embeds.view(b, c, -1).permute(0, 2, 1).contiguous()
98
+ # return image_embeds
99
+
100
+
101
+ # class EmuForCausalLM(EmuPreTrainedModel):
102
+ # _auto_class = "AutoModelForCausalLM"
103
+
104
+ # def __init__(self, config):
105
+ # super().__init__(config)
106
+
107
+ # self.config = config
108
+ # self.model = EmuModel(config)
109
+ # # LM to EVA
110
+ # self.project_down = nn.Linear(config.hidden_size, config.d_model, bias=False)
111
+ # # EVA to LM
112
+ # self.project_up = nn.Linear(config.d_model, config.hidden_size, bias=False)
113
+
114
+ # self.n_query = self.model.n_query
115
+ # self.v_query = self.model.v_query
116
+
117
+ # self.image_placeholder = DEFAULT_IMG_TOKEN + DEFAULT_IMAGE_TOKEN * self.n_query + DEFAULT_IMG_END_TOKEN
118
+ # # temporarily borrow [gIMG] as the video frame feature placeholder.
119
+ # self.video_placeholder = DEFAULT_IMG_TOKEN + DEFAULT_gIMG_TOKEN * self.v_query + DEFAULT_IMG_END_TOKEN
120
+
121
+ # # @property
122
+ # # def device(self):
123
+ # # return next(iter(self.parameters())).device
124
+
125
+ # # @property
126
+ # # def dtype(self):
127
+ # # return next(iter(self.parameters())).dtype
128
+ # def device(self, module=None):
129
+ # if module is None:
130
+ # return next(self.parameters()).device
131
+ # return next(module.parameters()).device
132
+
133
+ # def dtype(self, module):
134
+ # if module is None:
135
+ # return next(self.parameters()).dtype
136
+ # return next(module.parameters()).dtype
137
+
138
+
139
+ # @torch.no_grad()
140
+ # def generate(
141
+ # self,
142
+ # input_ids,
143
+ # attention_mask,
144
+ # image: Optional[torch.Tensor] = None,
145
+ # video: Optional[torch.Tensor] = None,
146
+ # num_beams=5,
147
+ # max_new_tokens=10,
148
+ # min_len=1,
149
+ # do_sample=False,
150
+ # penalty_alpha=None,
151
+ # top_p=None,
152
+ # top_k=None,
153
+ # temperature=None,
154
+ # length_penalty=-1,
155
+ # repetition_penalty=1.0,
156
+ # **kwargs
157
+ # ):
158
+
159
+ # text_embeds = self.model.decoder.lm.model.embed_tokens(input_ids).to("cuda")
160
+ # if image is not None:
161
+ # prompt_image_embeds = self.model.encode_image(image, n_query=self.n_query)
162
+ # _, _, c = prompt_image_embeds.shape
163
+ # prompt_image_embeds = prompt_image_embeds.view(-1, c)
164
+ # prompt_image_embeds = self.project_up(prompt_image_embeds)
165
+ # image_idx = (input_ids == IMAGE)
166
+ # text_embeds[image_idx] = prompt_image_embeds.to(text_embeds.device)
167
+
168
+ # if video is not None:
169
+ # prompt_video_embeds = self.model.encode_image(video, n_query=self.v_query)
170
+ # _, _, c = prompt_video_embeds.shape
171
+ # prompt_video_embeds = prompt_video_embeds.view(-1, c)
172
+ # prompt_video_embeds = self.project_up(prompt_video_embeds)
173
+ # video_idx = (input_ids == VIDEO)
174
+ # text_embeds[video_idx] = prompt_video_embeds.to(text_embeds.device)
175
+
176
+ # outputs = self.model.decoder.lm.generate(
177
+ # inputs_embeds=text_embeds,
178
+ # attention_mask=attention_mask,
179
+ # do_sample=do_sample,
180
+ # num_beams=num_beams,
181
+ # max_new_tokens=max_new_tokens,
182
+ # min_length=min_len,
183
+ # length_penalty=length_penalty,
184
+ # repetition_penalty=repetition_penalty,
185
+ # penalty_alpha=penalty_alpha,
186
+ # top_k=top_k,
187
+ # top_p=top_p,
188
+ # temperature=temperature,
189
+ # **kwargs,
190
+ # )
191
+
192
+ # return outputs
193
+
194
+ # def prepare_image_input(self, images):
195
+ # image_size: int = self.config.vision_config['image_size']
196
+ # transform = T.Compose(
197
+ # [
198
+ # T.Resize(
199
+ # (image_size, image_size), interpolation=T.InterpolationMode.BICUBIC
200
+ # ),
201
+ # T.ToTensor(),
202
+ # T.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD),
203
+ # ]
204
+ # )
205
+ # images = [transform(image) for image in images]
206
+ # return torch.stack(images, 0)
207
+
208
+ # def _prepare_chat_template(self, text, system_msg=""):
209
+ # text = [
210
+ # system_msg + USER_TOKEN + ": " + t + ASSISTANT_TOKEN +":"
211
+ # for t in text
212
+ # ]
213
+ # return text
214
+
215
+ # def prepare_text_input(
216
+ # self,
217
+ # text: List[str],
218
+ # tokenizer: PreTrainedTokenizer,
219
+ # image_placeholder: str = DEFAULT_IMG_PLACEHOLDER,
220
+ # video_placeholder: str = DEFAULT_VID_PLACEHOLDER,
221
+ # ):
222
+ # text = [
223
+ # t.replace(image_placeholder, self.image_placeholder).replace(video_placeholder, self.video_placeholder)
224
+ # for t in text
225
+ # ]
226
+ # input_ids = tokenizer(text, padding="longest", return_tensors="pt")
227
+ # return input_ids
228
+
229
+
230
+ # def build_input_ids(
231
+ # self,
232
+ # text: List[str],
233
+ # tokenizer: PreTrainedTokenizer,
234
+ # image: Optional[List["PIL.Image"]] = None,
235
+ # video: Optional[List["PIL.Image"]] = None,
236
+ # system_msg: str = "",
237
+ # to_cuda: bool = True
238
+ # ):
239
+
240
+ # if self.config.model_version == "chat":
241
+ # text = self._prepare_chat_template(text, system_msg)
242
+
243
+ # if image is not None:
244
+ # image = self.prepare_image_input(image)
245
+ # if video is not None:
246
+ # video = self.prepare_image_input(video)
247
+ # inputs = self.prepare_text_input(text, tokenizer)
248
+ # input_ids = inputs.input_ids
249
+ # attention_mask = inputs.attention_mask
250
+
251
+ # if to_cuda:
252
+ # input_ids = input_ids.to("cuda")
253
+ # attention_mask = attention_mask.to("cuda")
254
+ # if image is not None:
255
+ # image = image.to("cuda")
256
+ # if video is not None:
257
+ # video = video.to("cuda")
258
+
259
+
260
+
261
+ # return {
262
+ # 'input_ids': input_ids,
263
+ # 'attention_mask': attention_mask,
264
+ # 'image': image,
265
+ # 'video': video
266
+ # }
267
+
268
+ # def __call__(
269
+ # self,
270
+ # text: List[str],
271
+ # tokenizer: PreTrainedTokenizer,
272
+ # image: Optional[torch.Tensor] = None,
273
+ # placeholder: str = DEFAULT_IMG_PLACEHOLDER,
274
+ # lm_loss_weight: float = 1.0,
275
+ # img_loss_weight: float = 1.0,
276
+ # ):
277
+ # IMAGE, BOI = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_TOKEN, DEFAULT_IMG_TOKEN])
278
+ # if image is not None:
279
+ # batch_size = len(image)
280
+ # # Now image is a list of images in [(5,3,448,448), (...)]
281
+ # flattened_images = torch.cat(image, dim=0)
282
+ # # [bs*5, 3, 448, 448]
283
+ # prompt_image_embeds = self.model.encode_image(flattened_images)
284
+ # _, _, c = prompt_image_embeds.shape
285
+ # label = prompt_image_embeds
286
+ # label = label.view(batch_size, -1, self.n_query, c)
287
+ # label = label[:, -1, :, :]#.reshape(-1, c)
288
+
289
+ # prompt_image_embeds = prompt_image_embeds.view(-1, c)
290
+ # prompt_image_embeds = self.project_up(prompt_image_embeds)
291
+
292
+ # text = [t.replace(placeholder, self.image_placeholder) for t in text]
293
+
294
+ # inputs = tokenizer(text, padding="longest", return_tensors="pt")
295
+ # device = self.device(self.model.decoder.lm.model.embed_tokens)
296
+ # attention_mask = inputs.attention_mask.to(device)
297
+ # input_ids = inputs.input_ids.to(device) # B x N
298
+ # text_embeds = self.model.decoder.lm.model.embed_tokens(input_ids)
299
+ # image_idx = (input_ids == IMAGE)
300
+ # text_embeds[image_idx] = prompt_image_embeds.to(text_embeds.device)
301
+
302
+ # lm_labels = input_ids.clone()
303
+ # # boi_idx = (input_ids == BOI)
304
+ # lm_labels[image_idx] = IGNORE_INDEX # ignore image tokens
305
+ # # lm_labels[boi_idx] = IGNORE_INDEX # also ignore any BOI tokens
306
+
307
+ # outputs = self.model.decoder.lm(
308
+ # inputs_embeds=text_embeds,
309
+ # attention_mask=attention_mask,
310
+ # labels=lm_labels,
311
+ # output_hidden_states=True,
312
+ # return_dict=True,
313
+ # )
314
+ # lm_loss = outputs.loss
315
+
316
+ # hidden_states = outputs.hidden_states[-1]
317
+ # hidden_states = self.project_down(hidden_states)
318
+
319
+ # # Compute last image embedding positions
320
+ # boi_idx = torch.where(input_ids == BOI)
321
+ # last_img_boi_idx = []
322
+ # for bi in range(batch_size):
323
+ # last_img_boi_idx.append(boi_idx[1][boi_idx[0] == bi][-1].item())
324
+ # # Position of BOI of the input sequence will be the first img embedding position of the output sequence as forward will generate new token at the end of the sequence
325
+ # preds = []
326
+ # targets = []
327
+ # for i, idx in enumerate(last_img_boi_idx):
328
+ # avail = hidden_states.size(1) - idx
329
+ # m = min(self.n_query, avail)
330
+ # preds.append(hidden_states[i, idx:idx+m, :].contiguous()) #self.n_query, :].contiguous())
331
+ # targets.append(label[i, :m, :].contiguous()) #, idx-1:idx+self.n_query, :].contiguous())
332
+
333
+ # preds = torch.stack(preds).view(-1, hidden_states.shape[-1])
334
+ # targets = torch.stack(targets).view(-1, hidden_states.shape[-1])
335
+
336
+ # # print(" label.shape:", label.shape) # should be [batch_size, n_query, C]
337
+ # # print(" len(last_img_boi_idx):", len(last_img_boi_idx))
338
+ # # print(" preds_list len:", len(preds)) # number of rows in preds
339
+ # # print(" preds.shape:", preds.shape)
340
+ # # print(" targets.shape:", targets.shape)
341
+ # loss_fn = torch.nn.MSELoss()
342
+ # img_loss = loss_fn(preds, targets) #label)
343
+ # total_loss = lm_loss_weight * lm_loss + img_loss_weight * img_loss
344
+
345
+ # # print(f"Image loss: {img_loss}, Language loss: {lm_loss}")
346
+
347
+ # return total_loss
348
+
349
+
350
+ # @torch.no_grad()
351
+ # def generate_image(
352
+ # self,
353
+ # text: List[str],
354
+ # tokenizer: PreTrainedTokenizer,
355
+ # image: Optional[torch.Tensor] = None,
356
+ # placeholder: str = DEFAULT_IMG_PLACEHOLDER,
357
+ # ):
358
+ # IMAGE, BOI = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_TOKEN, DEFAULT_IMG_TOKEN])
359
+ # if image is not None:
360
+ # prompt_image_embeds = self.model.encode_image(image)
361
+ # _, _, c = prompt_image_embeds.shape
362
+ # prompt_image_embeds = prompt_image_embeds.view(-1, c)
363
+ # prompt_image_embeds = self.project_up(prompt_image_embeds)
364
+
365
+ # text = [t.replace(placeholder, self.image_placeholder) for t in text]
366
+
367
+ # target_image_embeds = None
368
+ # for num_img_token in range(self.n_query):
369
+ # if num_img_token == 0:
370
+ # text = [f"{t}{DEFAULT_IMG_TOKEN}" for t in text]
371
+ # else:
372
+ # text = [f"{t}{DEFAULT_IMAGE_TOKEN}" for t in text]
373
+
374
+ # inputs = tokenizer(text, padding="longest", return_tensors="pt")
375
+ # device = self.device(self.model.decoder.lm.model.embed_tokens)
376
+ # attention_mask = inputs.attention_mask.to(device)
377
+ # input_ids = inputs.input_ids.to(device) # B x N
378
+
379
+ # text_embeds = self.model.decoder.lm.model.embed_tokens(input_ids)
380
+
381
+ # image_idx = (input_ids == IMAGE)
382
+ # cumsum_idx = torch.flip(torch.cumsum(torch.flip(image_idx, dims=[1]), dim=1), dims=[1])
383
+ # if image is not None:
384
+ # prompt_idx = torch.logical_and(image_idx, cumsum_idx > num_img_token)
385
+ # text_embeds[prompt_idx] = prompt_image_embeds.to(text_embeds.device)
386
+
387
+ # if target_image_embeds is not None:
388
+ # target_idx = torch.logical_and(image_idx, torch.logical_and(cumsum_idx > 0, cumsum_idx <= num_img_token))
389
+ # text_embeds[target_idx] = self.project_up(target_image_embeds).to(text_embeds.device)
390
+
391
+ # outputs = self.model.decoder.lm.model(
392
+ # inputs_embeds=text_embeds,
393
+ # attention_mask=attention_mask,
394
+ # output_hidden_states=True,
395
+ # return_dict=True,
396
+ # )
397
+
398
+ # image_idx = (input_ids == IMAGE) + (input_ids == BOI)
399
+ # cumsum_idx = torch.flip(torch.cumsum(torch.flip(image_idx, dims=[1]), dim=1), dims=[1])
400
+ # target_idx = torch.logical_and(image_idx, torch.logical_and(cumsum_idx > 0, cumsum_idx <= num_img_token+1))
401
+
402
+ # hidden_states = outputs.hidden_states[-1]
403
+ # target_image_embeds = hidden_states[target_idx.to(hidden_states.device)]
404
+ # target_image_embeds = target_image_embeds.view(-1, target_image_embeds.shape[-1])
405
+ # target_image_embeds = self.project_down(target_image_embeds)
406
+
407
+ # _, C = target_image_embeds.shape
408
+ # B = hidden_states.shape[0]
409
+ # target_image_embeds = target_image_embeds.view(B, -1, C)
410
+
411
+ # return target_image_embeds
412
+
413
+ from functools import partial
414
+ from typing import Any, List, Optional, Mapping, Callable
415
+ from collections import OrderedDict
416
+ from argparse import Namespace
417
+ import torch
418
+ from torch import nn
419
+ import torch.nn.functional as F
420
+ import torchvision.transforms as T
421
+ import PIL
422
+ import transformers
423
+ from transformers import PreTrainedModel, PreTrainedTokenizer
424
+
425
+ from .configuration_emu import EmuConfig
426
+ from .constants import *
427
+ from .modeling_llama import LlamaForCausalLM
428
+ from .visual import EVAVisionTransformer
429
+
430
+
431
+ class EmuPreTrainedModel(PreTrainedModel):
432
+ config_class = EmuConfig
433
+ base_model_prefix = "model"
434
+ supports_gradient_checkpointing = False
435
+ _no_split_modules = ["LlamaDecoderLayer", "Block"]
436
+ _skip_keys_device_placement = "past_key_values"
437
+
438
+ def _init_weights(self, module):
439
+ std = self.config.initializer_range
440
+ if isinstance(module, nn.Linear):
441
+ module.weight.data.normal_(mean=0.0, std=std)
442
+ if module.bias is not None:
443
+ module.bias.data.zero_()
444
+ elif isinstance(module, nn.Embedding):
445
+ module.weight.data.normal_(mean=0.0, std=std)
446
+ if module.padding_idx is not None:
447
+ module.weight.data[module.padding_idx].zero_()
448
+
449
+ class EmuForClsAndRegression(EmuPreTrainedModel):
450
+
451
+ def __init__(self, config):
452
+ super(EmuForClsAndRegression, self).__init__(config)
453
+
454
+ self.lm = LlamaForCausalLM(config=config)
455
+
456
+ self.lm.model.embed_tokens.padding_idx = config.pad_token_id
457
+
458
+ def get_num_layers(self):
459
+ return len(self.lm.model.layers)
460
+
461
+ class EmuModel(EmuPreTrainedModel):
462
+
463
+ def __init__(self, config):
464
+ super().__init__(config)
465
+
466
+ vision_config = Namespace(**config.vision_config)
467
+
468
+ self.visual = EVAVisionTransformer(
469
+ img_size=vision_config.image_size,
470
+ patch_size=vision_config.patch_size,
471
+ embed_dim=vision_config.width,
472
+ depth=vision_config.layers,
473
+ num_heads=vision_config.width // vision_config.head_width,
474
+ mlp_ratio=vision_config.mlp_ratio,
475
+ qkv_bias=vision_config.qkv_bias,
476
+ drop_path_rate=vision_config.drop_path_rate,
477
+ norm_layer=partial(nn.LayerNorm, eps=vision_config.layer_norm_eps),
478
+ xattn=vision_config.xattn,
479
+ postnorm=vision_config.postnorm,
480
+ )
481
+
482
+ self.decoder = EmuForClsAndRegression(config)
483
+
484
+ self.gradient_checkpointing = False
485
+
486
+ self.n_query = vision_config.n_query
487
+ self.v_query = vision_config.v_query
488
+
489
+ @property
490
+ def device(self):
491
+ return next(iter(self.parameters())).device
492
+
493
+ @property
494
+ def dtype(self):
495
+ return next(iter(self.parameters())).dtype
496
+
497
+ @torch.no_grad()
498
+ def encode_image(self, image: torch.Tensor, *, n_query=None):
499
+ n_query = n_query if n_query is not None else self.n_query
500
+
501
+ image_embeds = self.visual(image)
502
+ image_embeds = image_embeds[:, 1:, :]
503
+ b, n, c = image_embeds.shape
504
+ sqrt_n = int(n**0.5)
505
+ image_embeds = image_embeds.permute(0, 2, 1).view(b, c, sqrt_n, sqrt_n)
506
+
507
+ stride = int(sqrt_n // (n_query ** 0.5))
508
+ image_embeds = F.avg_pool2d(image_embeds, kernel_size=(stride, stride), stride=stride)
509
+ image_embeds = image_embeds.view(b, c, -1).permute(0, 2, 1).contiguous()
510
+ return image_embeds
511
+
512
+
513
+ class EmuForCausalLM(EmuPreTrainedModel):
514
+ _auto_class = "AutoModelForCausalLM"
515
+
516
+ def __init__(self, config):
517
+ super().__init__(config)
518
+
519
+ self.config = config
520
+ self.model = EmuModel(config)
521
+ # LM to EVA
522
+ self.project_down = nn.Linear(config.hidden_size, config.d_model, bias=False)
523
+ # EVA to LM
524
+ self.project_up = nn.Linear(config.d_model, config.hidden_size, bias=False)
525
+
526
+ self.n_query = self.model.n_query
527
+ self.v_query = self.model.v_query
528
+
529
+ self.image_placeholder = DEFAULT_IMG_TOKEN + DEFAULT_IMAGE_TOKEN * self.n_query + DEFAULT_IMG_END_TOKEN
530
+ # temporarily borrow [gIMG] as the video frame feature placeholder.
531
+ self.video_placeholder = DEFAULT_IMG_TOKEN + DEFAULT_gIMG_TOKEN * self.v_query + DEFAULT_IMG_END_TOKEN
532
+
533
+ # @property
534
+ # def device(self):
535
+ # return next(iter(self.parameters())).device
536
+
537
+ # @property
538
+ # def dtype(self):
539
+ # return next(iter(self.parameters())).dtype
540
+ def device(self, module=None):
541
+ if module is None:
542
+ return next(self.parameters()).device
543
+ return next(module.parameters()).device
544
+
545
+ def dtype(self, module=None):
546
+ if module is None:
547
+ return next(self.parameters()).dtype
548
+ return next(module.parameters()).dtype
549
+
550
+
551
+ @torch.no_grad()
552
+ def generate(
553
+ self,
554
+ input_ids,
555
+ attention_mask,
556
+ image: Optional[torch.Tensor] = None,
557
+ video: Optional[torch.Tensor] = None,
558
+ num_beams=5,
559
+ max_new_tokens=10,
560
+ min_len=1,
561
+ do_sample=False,
562
+ penalty_alpha=None,
563
+ top_p=None,
564
+ top_k=None,
565
+ temperature=None,
566
+ length_penalty=-1,
567
+ repetition_penalty=1.0,
568
+ lambda_txt: float = 1.0, # <<< NEW
569
+ lambda_img: float = 1.0, # <<< NEW
570
+ **kwargs
571
+ ):
572
+
573
+ text_embeds = self.model.decoder.lm.model.embed_tokens(input_ids).to("cuda")
574
+ if image is not None:
575
+ prompt_image_embeds = self.model.encode_image(image, n_query=self.n_query)
576
+ _, _, c = prompt_image_embeds.shape
577
+ prompt_image_embeds = prompt_image_embeds.view(-1, c)
578
+ prompt_image_embeds = self.project_up(prompt_image_embeds)
579
+ image_idx = (input_ids == IMAGE)
580
+ text_embeds[image_idx] = prompt_image_embeds.to(text_embeds.device)
581
+
582
+ if video is not None:
583
+ prompt_video_embeds = self.model.encode_image(video, n_query=self.v_query)
584
+ _, _, c = prompt_video_embeds.shape
585
+ prompt_video_embeds = prompt_video_embeds.view(-1, c)
586
+ prompt_video_embeds = self.project_up(prompt_video_embeds)
587
+ video_idx = (input_ids == VIDEO)
588
+ text_embeds[video_idx] = prompt_video_embeds.to(text_embeds.device)
589
+
590
+ # modified: build masks for initial context
591
+ B, T = input_ids.shape
592
+ img_token_mask = torch.zeros((B, T), dtype=text_embeds.dtype, device=text_embeds.device)
593
+ txt_token_mask = torch.ones((B, T), dtype=text_embeds.dtype, device=text_embeds.device)
594
+
595
+ if image is not None:
596
+ boi_idx = torch.where(input_ids == BOI)
597
+ for b in range(B):
598
+ idxs = boi_idx[1][boi_idx[0] == b]
599
+ if idxs.numel() > 0:
600
+ j = idxs[-1].item()
601
+ m = min(self.n_query, T - j)
602
+ img_token_mask[b, j:j+m] = 1.0
603
+ txt_token_mask[b, j:j+m] = 0.0
604
+
605
+ outputs = self.model.decoder.lm.generate(
606
+ inputs_embeds=text_embeds,
607
+ attention_mask=attention_mask,
608
+ do_sample=do_sample,
609
+ num_beams=num_beams,
610
+ max_new_tokens=max_new_tokens,
611
+ min_length=min_len,
612
+ length_penalty=length_penalty,
613
+ repetition_penalty=repetition_penalty,
614
+ penalty_alpha=penalty_alpha,
615
+ top_k=top_k,
616
+ top_p=top_p,
617
+ temperature=temperature,
618
+ lambda_txt=lambda_txt, # modified
619
+ lambda_img=lambda_img,
620
+ # img_token_mask=img_token_mask,
621
+ # txt_token_mask=txt_token_mask,
622
+ **kwargs,
623
+ )
624
+
625
+ return outputs
626
+
627
+ def prepare_image_input(self, images):
628
+ image_size: int = self.config.vision_config['image_size']
629
+ transform = T.Compose(
630
+ [
631
+ T.Resize(
632
+ (image_size, image_size), interpolation=T.InterpolationMode.BICUBIC
633
+ ),
634
+ T.ToTensor(),
635
+ T.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD),
636
+ ]
637
+ )
638
+ images = [transform(image) for image in images]
639
+ return torch.stack(images, 0)
640
+
641
+ def _prepare_chat_template(self, text, system_msg=""):
642
+ text = [
643
+ system_msg + USER_TOKEN + ": " + t + ASSISTANT_TOKEN +":"
644
+ for t in text
645
+ ]
646
+ return text
647
+
648
+ def prepare_text_input(
649
+ self,
650
+ text: List[str],
651
+ tokenizer: PreTrainedTokenizer,
652
+ image_placeholder: str = DEFAULT_IMG_PLACEHOLDER,
653
+ video_placeholder: str = DEFAULT_VID_PLACEHOLDER,
654
+ ):
655
+ text = [
656
+ t.replace(image_placeholder, self.image_placeholder).replace(video_placeholder, self.video_placeholder)
657
+ for t in text
658
+ ]
659
+ input_ids = tokenizer(text, padding="longest", return_tensors="pt")
660
+ return input_ids
661
+
662
+
663
+ def build_input_ids(
664
+ self,
665
+ text: List[str],
666
+ tokenizer: PreTrainedTokenizer,
667
+ image: Optional[List["PIL.Image"]] = None,
668
+ video: Optional[List["PIL.Image"]] = None,
669
+ system_msg: str = "",
670
+ to_cuda: bool = True
671
+ ):
672
+
673
+ if self.config.model_version == "chat":
674
+ text = self._prepare_chat_template(text, system_msg)
675
+
676
+ if image is not None:
677
+ image = self.prepare_image_input(image)
678
+ if video is not None:
679
+ video = self.prepare_image_input(video)
680
+ inputs = self.prepare_text_input(text, tokenizer)
681
+ input_ids = inputs.input_ids
682
+ attention_mask = inputs.attention_mask
683
+
684
+ if to_cuda:
685
+ input_ids = input_ids.to("cuda")
686
+ attention_mask = attention_mask.to("cuda")
687
+ if image is not None:
688
+ image = image.to("cuda")
689
+ if video is not None:
690
+ video = video.to("cuda")
691
+
692
+
693
+
694
+ return {
695
+ 'input_ids': input_ids,
696
+ 'attention_mask': attention_mask,
697
+ 'image': image,
698
+ 'video': video
699
+ }
700
+
701
+ def __call__(
702
+ self,
703
+ text: List[str],
704
+ tokenizer: PreTrainedTokenizer,
705
+ image: Optional[torch.Tensor] = None,
706
+ placeholder: str = DEFAULT_IMG_PLACEHOLDER,
707
+ lm_loss_weight: float = 1.0,
708
+ img_loss_weight: float = 1.0,
709
+ return_dict=False,
710
+ lambda_txt: float = 1.0,
711
+ lambda_img: float = 1.0,
712
+ lambda_orth: float = 1e-4,
713
+ ):
714
+ IMAGE, BOI = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_TOKEN, DEFAULT_IMG_TOKEN])
715
+ if image is not None:
716
+ batch_size = len(image)
717
+ # Now image is a list of images in [(5,3,448,448), (...)]
718
+ flattened_images = torch.cat(image, dim=0)
719
+ # [bs*5, 3, 448, 448]
720
+ prompt_image_embeds = self.model.encode_image(flattened_images)
721
+ _, _, c = prompt_image_embeds.shape
722
+ label = prompt_image_embeds
723
+ label = label.view(batch_size, -1, self.n_query, c)
724
+ label = label[:, -1, :, :]#.reshape(-1, c)
725
+
726
+ prompt_image_embeds = prompt_image_embeds.view(-1, c)
727
+ prompt_image_embeds = self.project_up(prompt_image_embeds)
728
+
729
+ text = [t.replace(placeholder, self.image_placeholder) for t in text]
730
+
731
+ inputs = tokenizer(text, padding="longest", return_tensors="pt")
732
+ device = self.device(self.model.decoder.lm.model.embed_tokens)
733
+ attention_mask = inputs.attention_mask.to(device)
734
+ input_ids = inputs.input_ids.to(device) # B x N
735
+ text_embeds = self.model.decoder.lm.model.embed_tokens(input_ids)
736
+ image_idx = (input_ids == IMAGE)
737
+ text_embeds[image_idx] = prompt_image_embeds.to(text_embeds.device)
738
+
739
+ # modified: modality masks (build BEFORE LM forward)
740
+ B, N = input_ids.shape
741
+ # float masks so they can multiply activations; keep dtype/device consistent
742
+ img_token_mask = torch.zeros((B, N), dtype=text_embeds.dtype, device=text_embeds.device)
743
+ last_img_boi_idx = [None] * B
744
+ if image is not None:
745
+ boi_idx = torch.where(input_ids == BOI)
746
+ for bi in range(B):
747
+ idxs = boi_idx[1][boi_idx[0] == bi]
748
+ if idxs.numel() > 0:
749
+ j = idxs[-1].item()
750
+ last_img_boi_idx[bi] = j
751
+ m = min(self.n_query, N - j)
752
+ img_token_mask[bi, j:j+m] = 1.0 # only gate the n_query positions
753
+ txt_token_mask = 1.0 - img_token_mask
754
+
755
+ lm_labels = input_ids.clone()
756
+ # boi_idx = (input_ids == BOI)
757
+ lm_labels[image_idx] = IGNORE_INDEX # ignore image tokens
758
+ # lm_labels[boi_idx] = IGNORE_INDEX # also ignore any BOI tokens
759
+
760
+ outputs = self.model.decoder.lm(
761
+ inputs_embeds=text_embeds,
762
+ attention_mask=attention_mask,
763
+ labels=lm_labels,
764
+ output_hidden_states=True,
765
+ return_dict=True,
766
+ lambda_txt=lambda_txt,
767
+ lambda_img=lambda_img,
768
+ # img_token_mask=img_token_mask, # modified
769
+ # txt_token_mask=txt_token_mask, # modified
770
+ )
771
+ lm_loss = outputs.loss
772
+
773
+ hidden_states = outputs.hidden_states[-1]
774
+ hidden_states = self.project_down(hidden_states)
775
+
776
+ # =================== OLD =======================================================
777
+ # # Compute last image embedding positions
778
+ # boi_idx = torch.where(input_ids == BOI)
779
+ # last_img_boi_idx = []
780
+ # for bi in range(batch_size):
781
+ # last_img_boi_idx.append(boi_idx[1][boi_idx[0] == bi][-1].item())
782
+ # # Position of BOI of the input sequence will be the first img embedding position of the output sequence as forward will generate new token at the end of the sequence
783
+ # preds = []
784
+ # targets = []
785
+ # for i, idx in enumerate(last_img_boi_idx):
786
+ # avail = hidden_states.size(1) - idx
787
+ # m = min(self.n_query, avail)
788
+ # preds.append(hidden_states[i, idx:idx+m, :].contiguous()) #self.n_query, :].contiguous())
789
+ # targets.append(label[i, :m, :].contiguous()) #, idx-1:idx+self.n_query, :].contiguous())
790
+
791
+ # preds = torch.stack(preds).view(-1, hidden_states.shape[-1])
792
+ # targets = torch.stack(targets).view(-1, hidden_states.shape[-1])
793
+
794
+ # # print(" label.shape:", label.shape) # should be [batch_size, n_query, C]
795
+ # # print(" len(last_img_boi_idx):", len(last_img_boi_idx))
796
+ # # print(" preds_list len:", len(preds)) # number of rows in preds
797
+ # # print(" preds.shape:", preds.shape)
798
+ # # print(" targets.shape:", targets.shape)
799
+ # loss_fn = torch.nn.MSELoss()
800
+ # img_loss = loss_fn(preds, targets) #label)
801
+ # total_loss = lm_loss_weight * lm_loss + img_loss_weight * img_loss
802
+
803
+ # # print(f"Image loss: {img_loss}, Language loss: {lm_loss}")
804
+ # =================== OLD ======================================
805
+
806
+ # If you use image loss, reuse last_img_boi_idx above (we computed it earlier)
807
+ preds, targets = [], []
808
+ if image is not None:
809
+ for i, j in enumerate(last_img_boi_idx):
810
+ if j is None: # no BOI for this sample
811
+ continue
812
+ avail = hidden_states.size(1) - j
813
+ m = min(self.n_query, avail)
814
+ preds.append(hidden_states[i, j:j+m, :].contiguous())
815
+ targets.append(label[i, :m, :].contiguous())
816
+
817
+ preds = torch.stack(preds).view(-1, hidden_states.shape[-1])
818
+ targets = torch.stack(targets).view(-1, hidden_states.shape[-1])
819
+ img_loss = torch.nn.functional.mse_loss(preds, targets)
820
+ total_loss = lm_loss_weight * lm_loss + img_loss_weight * img_loss
821
+ else:
822
+ img_loss = torch.tensor(0.0, dtype=lm_loss.dtype, device=lm_loss.device)
823
+ total_loss = lm_loss
824
+
825
+ # ########################## Orthogonal Regularization ##########################
826
+ # orth_txt_loss, orth_img_loss = 0., 0.
827
+ # for name, param in self.model.named_parameters():
828
+ # if "lora_A" in name:
829
+ # for name_, param_ in self.model.named_parameters():
830
+ # if "loratxt_A" in name_ and name.split("lora_A")[0] == name_.split("loratxt_A")[0]:
831
+ # orth_txt_loss += torch.abs(torch.mm(param, param_.T)).sum() # [r * dim] * [dim * r]
832
+ # break # target modules have been matched
833
+ # if "lora_A" in name:
834
+ # for name_, param_ in self.model.named_parameters():
835
+ # if "loraimg_A" in name_ and name.split("lora_A")[0] == name_.split("loraimg_A")[0]:
836
+ # orth_img_loss += torch.abs(torch.mm(param, param_.T)).sum() # [r * dim] * [dim * r]
837
+ # break
838
+ # print(f"Orthogonal loss: {orthogonal_loss}")
839
+ # orthogonal_loss = orthogonal_loss * lambda_orth
840
+ # print(f"Orthogonal loss after scaling: {orthogonal_loss}")
841
+ # scale = lm_loss.detach() / (orth_txt_loss.detach() + 1e-8)
842
+ # lm_loss = lm_loss + scale * orth_txt_loss
843
+ # scale = img_loss.detach() / (orth_img_loss.detach() + 1e-8)
844
+ # img_loss = img_loss + scale * orth_img_loss
845
+
846
+ if not return_dict:
847
+ return total_loss
848
+ else:
849
+ return total_loss, lm_loss, img_loss
850
+
851
+
852
+ @torch.no_grad()
853
+ def generate_image(
854
+ self,
855
+ text: List[str],
856
+ tokenizer: PreTrainedTokenizer,
857
+ image: Optional[torch.Tensor] = None,
858
+ placeholder: str = DEFAULT_IMG_PLACEHOLDER,
859
+ ):
860
+ IMAGE, BOI = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_TOKEN, DEFAULT_IMG_TOKEN])
861
+ if image is not None:
862
+ prompt_image_embeds = self.model.encode_image(image)
863
+ _, _, c = prompt_image_embeds.shape
864
+ prompt_image_embeds = prompt_image_embeds.view(-1, c)
865
+ prompt_image_embeds = self.project_up(prompt_image_embeds)
866
+
867
+ text = [t.replace(placeholder, self.image_placeholder) for t in text]
868
+
869
+ target_image_embeds = None
870
+ for num_img_token in range(self.n_query):
871
+ if num_img_token == 0:
872
+ text = [f"{t}{DEFAULT_IMG_TOKEN}" for t in text]
873
+ else:
874
+ text = [f"{t}{DEFAULT_IMAGE_TOKEN}" for t in text]
875
+
876
+ inputs = tokenizer(text, padding="longest", return_tensors="pt")
877
+ device = self.device(self.model.decoder.lm.model.embed_tokens)
878
+ attention_mask = inputs.attention_mask.to(device)
879
+ input_ids = inputs.input_ids.to(device) # B x N
880
+
881
+ text_embeds = self.model.decoder.lm.model.embed_tokens(input_ids)
882
+
883
+ image_idx = (input_ids == IMAGE)
884
+ cumsum_idx = torch.flip(torch.cumsum(torch.flip(image_idx, dims=[1]), dim=1), dims=[1])
885
+ if image is not None:
886
+ prompt_idx = torch.logical_and(image_idx, cumsum_idx > num_img_token)
887
+ text_embeds[prompt_idx] = prompt_image_embeds.to(text_embeds.device)
888
+
889
+ if target_image_embeds is not None:
890
+ target_idx = torch.logical_and(image_idx, torch.logical_and(cumsum_idx > 0, cumsum_idx <= num_img_token))
891
+ text_embeds[target_idx] = self.project_up(target_image_embeds).to(text_embeds.device)
892
+
893
+ outputs = self.model.decoder.lm.model(
894
+ inputs_embeds=text_embeds,
895
+ attention_mask=attention_mask,
896
+ output_hidden_states=True,
897
+ return_dict=True,
898
+ )
899
+
900
+ image_idx = (input_ids == IMAGE) + (input_ids == BOI)
901
+ cumsum_idx = torch.flip(torch.cumsum(torch.flip(image_idx, dims=[1]), dim=1), dims=[1])
902
+ target_idx = torch.logical_and(image_idx, torch.logical_and(cumsum_idx > 0, cumsum_idx <= num_img_token+1))
903
+
904
+ hidden_states = outputs.hidden_states[-1]
905
+ target_image_embeds = hidden_states[target_idx.to(hidden_states.device)]
906
+ target_image_embeds = target_image_embeds.view(-1, target_image_embeds.shape[-1])
907
+ target_image_embeds = self.project_down(target_image_embeds)
908
+
909
+ _, C = target_image_embeds.shape
910
+ B = hidden_states.shape[0]
911
+ target_image_embeds = target_image_embeds.view(B, -1, C)
912
+
913
+ return target_image_embeds
modeling_llama.py ADDED
The diff for this file is too large to render. See raw diff
 
visual.py ADDED
@@ -0,0 +1,450 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # --------------------------------------------------------
2
+ # Adapted from https://github.com/microsoft/unilm/tree/master/beit
3
+ # --------------------------------------------------------
4
+
5
+ import os
6
+ from functools import partial
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+ import torch.nn.functional as F
11
+ from torch.utils.checkpoint import checkpoint
12
+
13
+ try:
14
+ from timm.models.layers import drop_path, to_2tuple
15
+ except:
16
+ from timm.layers import drop_path, to_2tuple
17
+
18
+ try:
19
+ import xformers.ops as xops
20
+ except ImportError:
21
+ xops = None
22
+ print("Please 'pip install xformers'")
23
+
24
+
25
+ class PatchDropout(nn.Module):
26
+ """
27
+ https://arxiv.org/abs/2212.00794
28
+ """
29
+
30
+ def __init__(self, prob, exclude_first_token=True):
31
+ super().__init__()
32
+ assert 0 <= prob < 1.
33
+ self.prob = prob
34
+ self.exclude_first_token = exclude_first_token # exclude CLS token
35
+ print(f"os.getenv('RoPE')={os.getenv('RoPE')}")
36
+
37
+ def forward(self, x):
38
+ if not self.training or self.prob == 0.:
39
+ return x
40
+
41
+ if self.exclude_first_token:
42
+ cls_tokens, x = x[:, :1], x[:, 1:]
43
+ else:
44
+ cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
45
+
46
+ batch = x.size()[0]
47
+ num_tokens = x.size()[1]
48
+
49
+ batch_indices = torch.arange(batch)
50
+ batch_indices = batch_indices[..., None]
51
+
52
+ keep_prob = 1 - self.prob
53
+ num_patches_keep = max(1, int(num_tokens * keep_prob))
54
+
55
+ rand = torch.randn(batch, num_tokens)
56
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
57
+
58
+ x = x[batch_indices, patch_indices_keep]
59
+
60
+ if self.exclude_first_token:
61
+ x = torch.cat((cls_tokens, x), dim=1)
62
+
63
+ if self.training and os.getenv('RoPE') == '1':
64
+ return x, patch_indices_keep
65
+
66
+ return x
67
+
68
+ class DropPath(nn.Module):
69
+ """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
70
+ """
71
+ def __init__(self, drop_prob=None):
72
+ super(DropPath, self).__init__()
73
+ self.drop_prob = drop_prob
74
+
75
+ def forward(self, x):
76
+ return drop_path(x, self.drop_prob, self.training)
77
+
78
+ def extra_repr(self) -> str:
79
+ return 'p={}'.format(self.drop_prob)
80
+
81
+
82
+ class Mlp(nn.Module):
83
+ def __init__(
84
+ self,
85
+ in_features,
86
+ hidden_features=None,
87
+ out_features=None,
88
+ act_layer=nn.GELU,
89
+ norm_layer=nn.LayerNorm,
90
+ drop=0.,
91
+ subln=False,
92
+
93
+ ):
94
+ super().__init__()
95
+ out_features = out_features or in_features
96
+ hidden_features = hidden_features or in_features
97
+ self.fc1 = nn.Linear(in_features, hidden_features)
98
+ self.act = act_layer()
99
+
100
+ self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity()
101
+
102
+ self.fc2 = nn.Linear(hidden_features, out_features)
103
+ self.drop = nn.Dropout(drop)
104
+
105
+ def forward(self, x):
106
+ x = self.fc1(x)
107
+ x = self.act(x)
108
+ # x = self.drop(x)
109
+ # commit this for the orignal BERT implement
110
+ x = self.ffn_ln(x)
111
+
112
+ x = self.fc2(x)
113
+ x = self.drop(x)
114
+ return x
115
+
116
+ class SwiGLU(nn.Module):
117
+ def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.,
118
+ norm_layer=nn.LayerNorm, subln=False):
119
+ super().__init__()
120
+ out_features = out_features or in_features
121
+ hidden_features = hidden_features or in_features
122
+
123
+ self.w1 = nn.Linear(in_features, hidden_features)
124
+ self.w2 = nn.Linear(in_features, hidden_features)
125
+
126
+ self.act = act_layer()
127
+ self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity()
128
+ self.w3 = nn.Linear(hidden_features, out_features)
129
+
130
+ self.drop = nn.Dropout(drop)
131
+
132
+ def forward(self, x):
133
+ x1 = self.w1(x)
134
+ x2 = self.w2(x)
135
+ hidden = self.act(x1) * x2
136
+ x = self.ffn_ln(hidden)
137
+ x = self.w3(x)
138
+ x = self.drop(x)
139
+ return x
140
+
141
+ class Attention(nn.Module):
142
+ def __init__(
143
+ self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.,
144
+ proj_drop=0., window_size=None, attn_head_dim=None, xattn=False, rope=None, subln=False, norm_layer=nn.LayerNorm):
145
+ super().__init__()
146
+ self.num_heads = num_heads
147
+ head_dim = dim // num_heads
148
+ if attn_head_dim is not None:
149
+ head_dim = attn_head_dim
150
+ all_head_dim = head_dim * self.num_heads
151
+ self.scale = qk_scale or head_dim ** -0.5
152
+
153
+ self.subln = subln
154
+ if self.subln:
155
+ self.q_proj = nn.Linear(dim, all_head_dim, bias=False)
156
+ self.k_proj = nn.Linear(dim, all_head_dim, bias=False)
157
+ self.v_proj = nn.Linear(dim, all_head_dim, bias=False)
158
+ else:
159
+ if qkv_bias:
160
+ self.qkv = nn.Linear(dim, all_head_dim * 3, bias=True)
161
+ else:
162
+ self.qkv = nn.Linear(dim, all_head_dim * 3, bias=False)
163
+
164
+ # if qkv_bias:
165
+ # self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
166
+ # self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
167
+ # else:
168
+ # self.q_bias = None
169
+ # self.v_bias = None
170
+
171
+ self.window_size = None
172
+ self.relative_position_bias_table = None
173
+ self.relative_position_index = None
174
+
175
+ self.attn_drop = nn.Dropout(attn_drop)
176
+ self.inner_attn_ln = norm_layer(all_head_dim) if subln else nn.Identity()
177
+ # self.proj = nn.Linear(all_head_dim, all_head_dim)
178
+ self.proj = nn.Linear(all_head_dim, dim)
179
+ self.proj_drop = nn.Dropout(proj_drop)
180
+ self.xattn = xattn
181
+ self.xattn_drop = attn_drop
182
+
183
+ self.rope = rope
184
+
185
+ def forward(self, x, rel_pos_bias=None, attn_mask=None):
186
+ B, N, C = x.shape
187
+ if self.subln:
188
+ q = F.linear(input=x, weight=self.q_proj.weight, bias=self.q_bias)
189
+ k = F.linear(input=x, weight=self.k_proj.weight, bias=None)
190
+ v = F.linear(input=x, weight=self.v_proj.weight, bias=self.v_bias)
191
+
192
+ q = q.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3) # B, num_heads, N, C
193
+ k = k.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
194
+ v = v.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
195
+ else:
196
+
197
+ # qkv_bias = None
198
+ # if self.q_bias is not None:
199
+ # qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))
200
+
201
+ # qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)
202
+
203
+ qkv = self.qkv(x)
204
+
205
+ qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) # 3, B, num_heads, N, C
206
+ q, k, v = qkv[0], qkv[1], qkv[2]
207
+
208
+ if self.rope:
209
+ q_t = q[:, :, 1:, :]
210
+ ro_q_t = self.rope(q_t)
211
+ q = torch.cat((q[:, :, :1, :], ro_q_t), -2).type_as(v)
212
+
213
+ k_t = k[:, :, 1:, :]
214
+ ro_k_t = self.rope(k_t)
215
+ k = torch.cat((k[:, :, :1, :], ro_k_t), -2).type_as(v)
216
+
217
+ if self.xattn:
218
+ q = q.permute(0, 2, 1, 3) # B, num_heads, N, C -> B, N, num_heads, C
219
+ k = k.permute(0, 2, 1, 3)
220
+ v = v.permute(0, 2, 1, 3)
221
+
222
+ x = xops.memory_efficient_attention(
223
+ q, k, v,
224
+ p=self.xattn_drop,
225
+ scale=self.scale,
226
+ )
227
+ x = x.reshape(B, N, -1)
228
+ x = self.inner_attn_ln(x)
229
+ x = self.proj(x)
230
+ x = self.proj_drop(x)
231
+ else:
232
+ q = q * self.scale
233
+ attn = (q @ k.transpose(-2, -1))
234
+
235
+ if self.relative_position_bias_table is not None:
236
+ relative_position_bias = \
237
+ self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
238
+ self.window_size[0] * self.window_size[1] + 1,
239
+ self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH
240
+ relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww
241
+ attn = attn + relative_position_bias.unsqueeze(0).type_as(attn)
242
+
243
+ if rel_pos_bias is not None:
244
+ attn = attn + rel_pos_bias.type_as(attn)
245
+
246
+ if attn_mask is not None:
247
+ attn_mask = attn_mask.bool()
248
+ attn = attn.masked_fill(~attn_mask[:, None, None, :], float("-inf"))
249
+
250
+ attn = attn.softmax(dim=-1)
251
+ attn = self.attn_drop(attn)
252
+
253
+ x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
254
+ x = self.inner_attn_ln(x)
255
+ x = self.proj(x)
256
+ x = self.proj_drop(x)
257
+ return x
258
+
259
+
260
+ class Block(nn.Module):
261
+
262
+ def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
263
+ drop_path=0., init_values=None, act_layer=nn.GELU, norm_layer=nn.LayerNorm,
264
+ window_size=None, attn_head_dim=None, xattn=False, rope=None, postnorm=False,
265
+ subln=False, naiveswiglu=False):
266
+ super().__init__()
267
+ self.norm1 = norm_layer(dim)
268
+ self.attn = Attention(
269
+ dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,
270
+ attn_drop=attn_drop, proj_drop=drop, window_size=window_size, attn_head_dim=attn_head_dim,
271
+ xattn=xattn, rope=rope, subln=subln, norm_layer=norm_layer)
272
+ # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
273
+ self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
274
+ self.norm2 = norm_layer(dim)
275
+ mlp_hidden_dim = int(dim * mlp_ratio)
276
+
277
+ if naiveswiglu:
278
+ self.mlp = SwiGLU(
279
+ in_features=dim,
280
+ hidden_features=mlp_hidden_dim,
281
+ subln=subln,
282
+ norm_layer=norm_layer,
283
+ )
284
+ else:
285
+ self.mlp = Mlp(
286
+ in_features=dim,
287
+ hidden_features=mlp_hidden_dim,
288
+ act_layer=act_layer,
289
+ subln=subln,
290
+ drop=drop
291
+ )
292
+
293
+ if init_values is not None and init_values > 0:
294
+ self.gamma_1 = nn.Parameter(init_values * torch.ones((dim)),requires_grad=True)
295
+ self.gamma_2 = nn.Parameter(init_values * torch.ones((dim)),requires_grad=True)
296
+ else:
297
+ self.gamma_1, self.gamma_2 = None, None
298
+
299
+ self.postnorm = postnorm
300
+
301
+ def forward(self, x, rel_pos_bias=None, attn_mask=None):
302
+ if self.gamma_1 is None:
303
+ if self.postnorm:
304
+ x = x + self.drop_path(self.norm1(self.attn(x, rel_pos_bias=rel_pos_bias, attn_mask=attn_mask)))
305
+ x = x + self.drop_path(self.norm2(self.mlp(x)))
306
+ else:
307
+ x = x + self.drop_path(self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias, attn_mask=attn_mask))
308
+ x = x + self.drop_path(self.mlp(self.norm2(x)))
309
+ else:
310
+ if self.postnorm:
311
+ x = x + self.drop_path(self.gamma_1 * self.norm1(self.attn(x, rel_pos_bias=rel_pos_bias, attn_mask=attn_mask)))
312
+ x = x + self.drop_path(self.gamma_2 * self.norm2(self.mlp(x)))
313
+ else:
314
+ x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias, attn_mask=attn_mask))
315
+ x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x)))
316
+ return x
317
+
318
+
319
+ class PatchEmbed(nn.Module):
320
+ """ Image to Patch Embedding
321
+ """
322
+ def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
323
+ super().__init__()
324
+ img_size = to_2tuple(img_size)
325
+ patch_size = to_2tuple(patch_size)
326
+ num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0])
327
+ self.patch_shape = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
328
+ self.img_size = img_size
329
+ self.patch_size = patch_size
330
+ self.num_patches = num_patches
331
+
332
+ self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
333
+
334
+ def forward(self, x, **kwargs):
335
+ B, C, H, W = x.shape
336
+ # FIXME look at relaxing size constraints
337
+ assert H == self.img_size[0] and W == self.img_size[1], \
338
+ f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
339
+ x = self.proj(x).flatten(2).transpose(1, 2)
340
+ return x
341
+
342
+
343
+ class EVAVisionTransformer(nn.Module):
344
+ """ Vision Transformer with support for patch or hybrid CNN input stage
345
+ """
346
+ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
347
+ num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,
348
+ drop_path_rate=0., norm_layer=nn.LayerNorm, init_values=None, patch_dropout=0.,
349
+ use_abs_pos_emb=True, use_rel_pos_bias=False, use_shared_rel_pos_bias=False, rope=False,
350
+ use_mean_pooling=True, init_scale=0.001, grad_checkpointing=False, xattn=False, postnorm=False,
351
+ pt_hw_seq_len=16, intp_freq=False, naiveswiglu=False, subln=False,
352
+ ):
353
+ super().__init__()
354
+ self.image_size = img_size
355
+ # self.num_classes = num_classes
356
+ self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models
357
+
358
+ self.patch_embed = PatchEmbed(
359
+ img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
360
+ num_patches = self.patch_embed.num_patches
361
+
362
+ self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
363
+ if use_abs_pos_emb:
364
+ self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
365
+ else:
366
+ self.pos_embed = None
367
+ self.pos_drop = nn.Dropout(p=drop_rate)
368
+
369
+ self.rel_pos_bias = None
370
+ self.rope = None
371
+
372
+ self.naiveswiglu = naiveswiglu
373
+
374
+ dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
375
+ self.use_rel_pos_bias = use_rel_pos_bias
376
+ self.blocks = nn.ModuleList([
377
+ Block(
378
+ dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
379
+ drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer,
380
+ init_values=init_values, window_size=self.patch_embed.patch_shape if use_rel_pos_bias else None,
381
+ xattn=xattn, rope=self.rope, postnorm=postnorm, subln=subln, naiveswiglu=naiveswiglu)
382
+ for i in range(depth)])
383
+
384
+ # setting a patch_dropout of 0. would mean it is disabled and this function would be the identity fn
385
+ self.patch_dropout = PatchDropout(patch_dropout) if patch_dropout > 0. else nn.Identity()
386
+
387
+ self.grad_checkpointing = grad_checkpointing
388
+
389
+
390
+ def get_num_layers(self):
391
+ return len(self.blocks)
392
+
393
+ def lock(self, unlocked_groups=0, freeze_bn_stats=False):
394
+ assert unlocked_groups == 0, 'partial locking not currently supported for this model'
395
+ for param in self.parameters():
396
+ param.requires_grad = False
397
+
398
+ @torch.jit.ignore
399
+ def set_grad_checkpointing(self, enable=True):
400
+ self.grad_checkpointing = enable
401
+
402
+ @torch.jit.ignore
403
+ def no_weight_decay(self):
404
+ return {'pos_embed', 'cls_token'}
405
+
406
+
407
+ def forward_features(self, x):
408
+ x = self.patch_embed(x)
409
+ batch_size, seq_len, _ = x.size()
410
+
411
+ cls_tokens = self.cls_token.expand(batch_size, -1, -1) # stole cls_tokens impl from Phil Wang, thanks
412
+ x = torch.cat((cls_tokens, x), dim=1)
413
+ if self.pos_embed is not None:
414
+ x = x + self.pos_embed
415
+ x = self.pos_drop(x)
416
+
417
+ # a patch_dropout of 0. would mean it is disabled and this function would do nothing but return what was passed in
418
+ if os.getenv('RoPE') == '1':
419
+ if self.training and not isinstance(self.patch_dropout, nn.Identity):
420
+ x, patch_indices_keep = self.patch_dropout(x)
421
+ self.rope.forward = partial(self.rope.forward, patch_indices_keep=patch_indices_keep)
422
+ else:
423
+ self.rope.forward = partial(self.rope.forward, patch_indices_keep=None)
424
+ x = self.patch_dropout(x)
425
+ else:
426
+ x = self.patch_dropout(x)
427
+
428
+ rel_pos_bias = None
429
+
430
+ for blk in self.blocks:
431
+ if self.grad_checkpointing:
432
+ x = checkpoint(blk, x, (rel_pos_bias,))
433
+ else:
434
+ x = blk(x, rel_pos_bias=rel_pos_bias)
435
+
436
+ return x
437
+
438
+ def forward(self, x):
439
+
440
+ """
441
+ :return:
442
+ forward_features function returns raw features of ViT,
443
+ forward with return_all_features returns normalized features of ViT
444
+ :param x:
445
+ :param return_all_features:
446
+ """
447
+
448
+ features = self.forward_features(x) # [B, n_patch, C]
449
+
450
+ return features