Transformers
nebulette commited on
Commit
069dce3
·
verified ·
1 Parent(s): 0e28d5f

Upload 4 files

Browse files
Files changed (4) hide show
  1. __init__.py +0 -0
  2. model.py +290 -0
  3. pipeline.py +234 -0
  4. vae.py +380 -0
__init__.py ADDED
File without changes
model.py ADDED
@@ -0,0 +1,290 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from safetensors.torch import load_model
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+
7
+ SPRINT_NUM_F = 2
8
+ SPRINT_NUM_H = 2
9
+ TEXT_EMBED_DIM = 640
10
+
11
+ def adaln(
12
+ x: torch.Tensor,
13
+ scale: torch.Tensor,
14
+ shift: torch.Tensor,
15
+ eps: float = 1e-6
16
+ ):
17
+ normalized = F.layer_norm(
18
+ x,
19
+ normalized_shape=(x.shape[-1],),
20
+ weight=None,
21
+ bias=None,
22
+ eps=eps,
23
+ )
24
+
25
+ return normalized * (1 + scale) + shift
26
+
27
+ def rope_2d(head_dim, height, width, device):
28
+ """Split-half 2D RoPE over centered token coordinates (x frequencies, then y), as (1, N, 1, head_dim // 2, 2, 2) rotations."""
29
+ axis_dim = head_dim // 2
30
+ inv_freq = 1.0 / (10000.0 ** (torch.arange(0, axis_dim, 2, dtype=torch.float32, device=device) / axis_dim))
31
+ y = torch.arange(height, dtype=torch.float32, device=device) - (height - 1) / 2
32
+ x = torch.arange(width, dtype=torch.float32, device=device) - (width - 1) / 2
33
+ y, x = torch.meshgrid(y, x, indexing="ij")
34
+ angles = torch.cat([torch.outer(x.flatten(), inv_freq), torch.outer(y.flatten(), inv_freq)], dim=-1)
35
+ cos, sin = torch.cos(angles), torch.sin(angles)
36
+ return torch.stack([cos, -sin, sin, cos], dim=-1).view(1, height * width, 1, axis_dim, 2, 2)
37
+
38
+ def modulate(x, shift, scale):
39
+ return torch.addcmul(shift, x, 1 + scale)
40
+
41
+ def rope_split_half(x, pe):
42
+ """Differentiable ck.apply_rope_split_half1: pair k is (x[k], x[k + head_dim // 2])."""
43
+ x_ = x.reshape(*x.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2).to(pe.dtype)
44
+ return (pe[..., 0] * x_[..., 0] + pe[..., 1] * x_[..., 1]).movedim(-1, -2).reshape(x.shape).type_as(x)
45
+
46
+ def norm_rope(q, k, q_norm, k_norm, pe):
47
+ return rope_split_half(q_norm(q), pe), rope_split_half(k_norm(k), pe)
48
+
49
+ class Embed(nn.Module):
50
+ def __init__(self, in_dim, hidden_size, norm=False, dtype=None, device=None):
51
+ super().__init__()
52
+ self.proj = nn.Linear(in_dim, hidden_size, bias=True, dtype=dtype, device=device)
53
+ self.norm = nn.RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device) if norm else nn.Identity()
54
+
55
+ def forward(self, x):
56
+ return self.norm(self.proj(x))
57
+
58
+ class FeedForward(nn.Module):
59
+ def __init__(self, dim, hidden_dim, dtype=None, device=None):
60
+ super().__init__()
61
+ self.w12 = nn.Linear(dim, hidden_dim * 2, bias=False, dtype=dtype, device=device)
62
+ self.w3 = nn.Linear(hidden_dim, dim, bias=False, dtype=dtype, device=device)
63
+
64
+ def forward(self, x):
65
+ x1, x2 = self.w12(x).chunk(2, dim=-1)
66
+ return self.w3(F.silu(x1) * x2)
67
+
68
+ class Attention(nn.Module):
69
+ """Self-attention over image tokens with 2D RoPE, optionally joint with (non-rotated) text keys and values."""
70
+
71
+ def __init__(self, dim, num_heads, cross_attention, dtype=None, device=None):
72
+ super().__init__()
73
+ self.num_heads = num_heads
74
+ self.head_dim = dim // num_heads
75
+ self.qkv_x = nn.Linear(dim, dim * 3, bias=False, dtype=dtype, device=device)
76
+ self.kv_y = nn.Linear(dim, dim * 2, bias=False, dtype=dtype, device=device) if cross_attention else None
77
+ self.q_norm = nn.RMSNorm(self.head_dim, eps=1e-6, dtype=dtype, device=device)
78
+ self.k_norm = nn.RMSNorm(self.head_dim, eps=1e-6, dtype=dtype, device=device)
79
+ self.proj = nn.Linear(dim, dim, bias=True, dtype=dtype, device=device)
80
+
81
+ def forward(self, x, txt, pe, txt_bias, transformer_options={}):
82
+ b, n, c = x.shape
83
+ q, k, v = self.qkv_x(x).view(b, n, 3, self.num_heads, self.head_dim).unbind(2)
84
+ q, k = norm_rope(q, k, self.q_norm, self.k_norm, pe)
85
+ mask = None
86
+ if self.kv_y is not None:
87
+ ky, vy = self.kv_y(txt).view(b, -1, 2, self.num_heads, self.head_dim).unbind(2)
88
+ k = torch.cat([k, self.k_norm(ky)], dim=1)
89
+ v = torch.cat([v, vy], dim=1)
90
+ # Image keys are always visible; text keys carry log(emphasis weight), -inf for padding.
91
+ mask = F.pad(txt_bias, (n, 0))[:, None, None, :]
92
+ # x = optimized_attention(q.reshape(b, n, c), k.reshape(b, -1, c), v.reshape(b, -1, c), self.num_heads, mask=mask, transformer_options=transformer_options)
93
+ q = q.transpose(1, 2)
94
+ k = k.transpose(1, 2)
95
+ v = v.transpose(1, 2)
96
+ # Mask from ldm/modules/attention, attention_pytorch() function.
97
+ if mask is not None:
98
+ # add a batch dimension if there isn't already one
99
+ if mask.ndim == 2:
100
+ mask = mask.unsqueeze(0)
101
+ # add a heads dimension if there isn't already one
102
+ if mask.ndim == 3:
103
+ mask = mask.unsqueeze(1)
104
+ x = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
105
+ x = x.transpose(1, 2).reshape(b, n, c)
106
+
107
+ return self.proj(x)
108
+
109
+ class DiTBlock(nn.Module):
110
+ """Image block; its adaLN modulation is computed by the caller (adaLN-single)."""
111
+
112
+ def __init__(self, hidden_size, num_heads, mlp_hidden, cross_attention, dtype=None, device=None):
113
+ super().__init__()
114
+ self.norm1 = nn.RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device)
115
+ self.attn = Attention(hidden_size, num_heads, cross_attention, dtype=dtype, device=device)
116
+ self.norm2 = nn.RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device)
117
+ self.mlp = FeedForward(hidden_size, mlp_hidden, dtype=dtype, device=device)
118
+
119
+ def forward(self, x, txt, pe, mod, txt_bias, transformer_options={}):
120
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1)
121
+ x = torch.addcmul(x, gate_msa, self.attn(modulate(self.norm1(x), shift_msa, scale_msa), txt, pe, txt_bias, transformer_options=transformer_options))
122
+ return torch.addcmul(x, gate_mlp, self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp)))
123
+
124
+ class TextRefineAttention(nn.Module):
125
+ def __init__(self, dim, num_heads, dtype=None, device=None):
126
+ super().__init__()
127
+ self.num_heads = num_heads
128
+ self.head_dim = dim // num_heads
129
+ self.qkv = nn.Linear(dim, dim * 3, bias=False, dtype=dtype, device=device)
130
+ self.q_norm = nn.RMSNorm(self.head_dim, eps=1e-6, dtype=dtype, device=device)
131
+ self.k_norm = nn.RMSNorm(self.head_dim, eps=1e-6, dtype=dtype, device=device)
132
+ self.proj = nn.Linear(dim, dim, bias=True, dtype=dtype, device=device)
133
+
134
+ def forward(self, x, mask, transformer_options={}):
135
+ b, n, c = x.shape
136
+ q, k, v = self.qkv(x).view(b, n, 3, self.num_heads, self.head_dim).unbind(2)
137
+ q, k = self.q_norm(q), self.k_norm(k)
138
+ # SDPA expects (B, H, N, D)
139
+ q = q.transpose(1, 2)
140
+ k = k.transpose(1, 2)
141
+ v = v.transpose(1, 2)
142
+ # x = optimized_attention(self.q_norm(q).reshape(b, n, c), self.k_norm(k).reshape(b, n, c), v.reshape(b, n, c), self.num_heads, mask=mask, transformer_options=transformer_options)
143
+ x = F.scaled_dot_product_attention(q, k, v)
144
+ x = x.transpose(1, 2).reshape(b, n, c)
145
+ return self.proj(x)
146
+
147
+ class TextRefineBlock(nn.Module):
148
+ def __init__(self, hidden_size, num_heads, mlp_hidden, dtype=None, device=None):
149
+ super().__init__()
150
+ self.norm1 = nn.RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device)
151
+ self.attn = TextRefineAttention(hidden_size, num_heads, dtype=dtype, device=device)
152
+ self.norm2 = nn.RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device)
153
+ self.mlp = FeedForward(hidden_size, mlp_hidden, dtype=dtype, device=device)
154
+ self.adaLN_modulation = nn.Sequential(nn.Linear(hidden_size, 6 * hidden_size, bias=True, dtype=dtype, device=device))
155
+
156
+ def forward(self, x, c, mask, keep, transformer_options={}):
157
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=-1)
158
+ x = torch.addcmul(x, gate_msa, self.attn(modulate(self.norm1(x), shift_msa, scale_msa), mask, transformer_options=transformer_options))
159
+ x = torch.addcmul(x, gate_mlp, self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp)))
160
+ return x * keep
161
+
162
+ class FinalLayer(nn.Module):
163
+ def __init__(self, hidden_size, out_channels, dtype=None, device=None):
164
+ super().__init__()
165
+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
166
+ self.adaLN_modulation = nn.Linear(hidden_size, 2 * hidden_size, bias=True, dtype=dtype, device=device)
167
+ self.linear = nn.Linear(hidden_size, out_channels, bias=True, dtype=dtype, device=device)
168
+
169
+ def forward(self, x, c):
170
+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1)
171
+ return self.linear(adaln(x, scale, shift, self.norm_final.eps))
172
+
173
+ class TimestepEmbedder(nn.Module):
174
+ """
175
+ Embeds scalar timesteps into vector representations.
176
+ """
177
+
178
+ def __init__(self, hidden_size, frequency_embedding_size=256, output_size=None, dtype=None, device=None, max_period=10000):
179
+ super().__init__()
180
+ if output_size is None:
181
+ output_size = hidden_size
182
+ self.mlp = nn.Sequential(
183
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True, dtype=dtype, device=device),
184
+ nn.SiLU(),
185
+ nn.Linear(hidden_size, output_size, bias=True, dtype=dtype, device=device),
186
+ )
187
+ self.frequency_embedding_size = frequency_embedding_size
188
+ self.max_period = max_period
189
+
190
+ def forward(self, t, dtype, **kwargs):
191
+ t_freq = self.timestep_embedding(t).to(dtype)
192
+ t_emb = self.mlp(t_freq)
193
+ return t_emb
194
+
195
+ def timestep_embedding(self, timesteps, repeat_only=False):
196
+ """
197
+ Create sinusoidal timestep embeddings.
198
+ :param timesteps: a 1-D Tensor of N indices, one per batch element.
199
+ These may be fractional.
200
+ :return: an [N x dim] Tensor of positional embeddings.
201
+ """
202
+ dim = self.frequency_embedding_size
203
+ if not repeat_only:
204
+ half = dim // 2
205
+ freqs = torch.exp(
206
+ -math.log(self.max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device=timesteps.device) / half
207
+ )
208
+ args = timesteps[:, None].float() * freqs[None]
209
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
210
+ if dim % 2:
211
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
212
+ else:
213
+ # embedding = repeat(timesteps, 'b -> b d', d=dim)
214
+ embedding = timesteps[:, None].expand(-1, dim)
215
+ return embedding
216
+
217
+ class Transformer2DModel(nn.Module):
218
+ def __init__(self, in_channels=64, hidden_size=1536, num_heads=16, num_blocks=18, num_text_blocks=2, mlp_hidden=4096, dtype=None, device=None):
219
+ super().__init__()
220
+ self.dtype = dtype
221
+ self.in_channels = in_channels
222
+ self.num_heads = num_heads
223
+ self.head_dim = hidden_size // num_heads
224
+ self.s_embedder = Embed(in_channels, hidden_size, dtype=dtype, device=device)
225
+ self.t_embedder = TimestepEmbedder(hidden_size, dtype=dtype, device=device)
226
+ self.y_embedder = Embed(TEXT_EMBED_DIM, hidden_size, norm=True, dtype=dtype, device=device)
227
+ self.shared_encoder_adaLN = nn.Sequential(nn.Linear(hidden_size, 6 * hidden_size, bias=True, dtype=dtype, device=device))
228
+ self.encoder_adaLN_offsets = nn.ParameterList([nn.Parameter(torch.empty(6 * hidden_size, dtype=dtype, device=device)) for _ in range(num_blocks)])
229
+ self.y_pool_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
230
+ self.sprint_out_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
231
+ self.blocks = nn.ModuleList([DiTBlock(hidden_size, num_heads, mlp_hidden, i % 2 == 0, dtype=dtype, device=device) for i in range(num_blocks)])
232
+ self.text_refine_blocks = nn.ModuleList([TextRefineBlock(hidden_size, num_heads, mlp_hidden, dtype=dtype, device=device) for _ in range(num_text_blocks)])
233
+ self.final_layer = FinalLayer(hidden_size, in_channels, dtype=dtype, device=device)
234
+
235
+ def run_block(self, i, x, txt, pe, mod, txt_bias, transformer_options):
236
+ offset = self.encoder_adaLN_offsets[i].to(dtype=mod.dtype)
237
+ return self.blocks[i](x, txt, pe, mod + offset, txt_bias, transformer_options=transformer_options)
238
+
239
+ def sparse_path(self, x, txt, pe, mod, txt_bias, transformer_options):
240
+ g = x
241
+ for i in range(SPRINT_NUM_F, len(self.blocks) - SPRINT_NUM_H):
242
+ g = self.run_block(i, g, txt, pe, mod, txt_bias, transformer_options)
243
+ return x + self.sprint_out_proj(g - x)
244
+
245
+ @torch.no_grad()
246
+ def forward(self, x, timestep, context, token_weights, sparse_skip=None, transformer_options={}, **kwargs):
247
+ """sparse_skip: optional per-row flags; flagged rows skip the sparse middle blocks."""
248
+ b, _, h, w = x.shape
249
+ keep = token_weights > 0
250
+ txt_bias = torch.log(token_weights.clamp(min=1e-4)).masked_fill(~keep, float("-inf"))
251
+ # Refine attention is a hard mask; a row with no text key keeps its first one and is zeroed after.
252
+ refine_keep = keep.clone()
253
+ refine_keep[:, 0] |= ~keep.any(dim=1)
254
+ refine_mask = torch.zeros_like(txt_bias).masked_fill(~refine_keep, float("-inf"))[:, None, None, :]
255
+ keep = keep.unsqueeze(-1).to(x.dtype)
256
+
257
+ t = self.t_embedder(timestep * 1000.0, x.dtype).unsqueeze(1)
258
+ txt = self.y_embedder(context)
259
+ time_condition = F.silu(t)
260
+ for block in self.text_refine_blocks:
261
+ txt = block(txt, time_condition, refine_mask, keep, transformer_options=transformer_options)
262
+
263
+ weights = token_weights.unsqueeze(-1)
264
+ pooled = (txt * weights).sum(dim=1) / weights.sum(dim=1).clamp(min=1.0)
265
+ condition = F.silu(t + self.y_pool_proj(pooled).unsqueeze(1))
266
+ mod = self.shared_encoder_adaLN(condition)
267
+
268
+ pe = rope_2d(self.head_dim, h, w, x.device)
269
+ s = self.s_embedder(x.flatten(2).transpose(1, 2))
270
+ for i in range(SPRINT_NUM_F):
271
+ s = self.run_block(i, s, txt, pe, mod, txt_bias, transformer_options)
272
+
273
+ if sparse_skip is None or not any(sparse_skip):
274
+ s = self.sparse_path(s, txt, pe, mod, txt_bias, transformer_options)
275
+ elif not all(sparse_skip):
276
+ rows = [i for i, skip in enumerate(sparse_skip) if not skip]
277
+ s[rows] = self.sparse_path(s[rows], txt[rows], pe, mod[rows], txt_bias[rows], transformer_options)
278
+
279
+ for i in range(len(self.blocks) - SPRINT_NUM_H, len(self.blocks)):
280
+ s = self.run_block(i, s, txt, pe, mod, txt_bias, transformer_options)
281
+
282
+ x0 = self.final_layer(s, condition).transpose(1, 2).reshape(b, self.in_channels, h, w)
283
+ return (x - x0) / timestep.view(-1, 1, 1, 1)
284
+
285
+ @staticmethod
286
+ def from_safetensors(path, device=None):
287
+ model = Transformer2DModel()
288
+ load_model(model, path)
289
+
290
+ return model.to(device)
pipeline.py ADDED
@@ -0,0 +1,234 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .model import Transformer2DModel
2
+ from .vae import VAE
3
+ from PIL import Image
4
+ import torch
5
+ from transformers import AutoTokenizer, Gemma3ForCausalLM
6
+
7
+ NUM_STEPS = 50
8
+ CFG_SCALE = 4.0
9
+ SHIFT = 3.0
10
+
11
+ HEIGHT = 1024
12
+ WIDTH = 1024
13
+
14
+ class Pipeline:
15
+ def __init__(
16
+ self,
17
+ transformer: str,
18
+ text_encoder: str,
19
+ vae: str,
20
+ device=None
21
+ ):
22
+ self.transformer = Transformer2DModel.from_safetensors(transformer)
23
+ self.transformer.eval()
24
+ self.tokenizer = AutoTokenizer.from_pretrained(text_encoder)
25
+ self.text_encoder = Gemma3ForCausalLM.from_pretrained(text_encoder)
26
+ self.text_encoder.eval()
27
+ self.vae = VAE.from_safetensors(vae)
28
+ self.vae.eval()
29
+ self.transformer.to(device)
30
+ self.text_encoder.to(device)
31
+ self.vae.to(device)
32
+
33
+ @staticmethod
34
+ def get_timestep_schedule(
35
+ num_steps: int,
36
+ shift: float = 3.0,
37
+ device: str = "cuda",
38
+ ):
39
+ """
40
+ Creates a timestep schedule from 1 -> 0.
41
+
42
+ The common flow-matching shift is:
43
+
44
+ t_shifted = s*t / (1 + (s - 1)*t)
45
+
46
+ where s is the shift value.
47
+ """
48
+ t = torch.linspace(
49
+ 1.0,
50
+ 0.0,
51
+ num_steps + 1,
52
+ device=device,
53
+ dtype=torch.float32,
54
+ )
55
+
56
+ if shift != 1.0:
57
+ t = shift * t / (1.0 + (shift - 1.0) * t)
58
+
59
+ return t
60
+
61
+ def encode_text(
62
+ self,
63
+ text: str,
64
+ device: str,
65
+ dtype: torch.dtype
66
+ ):
67
+ tokens = self.tokenizer(
68
+ text,
69
+ return_tensors="pt",
70
+ padding=False,
71
+ truncation=True,
72
+ )
73
+
74
+ input_ids = tokens.input_ids.to(device)
75
+ attention_mask = tokens.attention_mask.to(device)
76
+
77
+ with torch.inference_mode():
78
+ outputs = self.text_encoder(
79
+ input_ids=input_ids,
80
+ attention_mask=attention_mask,
81
+ output_hidden_states=True,
82
+ return_dict=True,
83
+ )
84
+
85
+ # Gemma hidden_states[-1] is the output of the final layer.
86
+ # Use the penultimate layer with final normalization.
87
+ hidden = outputs.hidden_states[-2]
88
+
89
+ # Depending on the exact Gemma implementation, the final norm may
90
+ # be exposed as model.norm or model.language_model.norm.
91
+ if hasattr(self.text_encoder, "model") and hasattr(self.text_encoder.model, "norm"):
92
+ hidden = self.text_encoder.model.norm(hidden)
93
+ elif hasattr(self.text_encoder, "language_model") and hasattr(
94
+ self.text_encoder.language_model, "norm"
95
+ ):
96
+ hidden = self.text_encoder.language_model.norm(hidden)
97
+
98
+ hidden = hidden.to(device=device, dtype=dtype)
99
+ token_weights = attention_mask.to(device=device, dtype=dtype)
100
+
101
+ return hidden, token_weights
102
+
103
+ def predict_velocity(
104
+ model,
105
+ latents: torch.Tensor,
106
+ timestep: torch.Tensor,
107
+ cond_context: torch.Tensor,
108
+ cond_weights: torch.Tensor,
109
+ uncond_context: torch.Tensor,
110
+ uncond_weights: torch.Tensor,
111
+ cfg_scale: float,
112
+ ):
113
+ """
114
+ Performs classifier-free guidance.
115
+
116
+ The unconditioned pass uses sparse_skip=True, matching the
117
+ path-drop behavior used during training.
118
+ """
119
+ batch_size = latents.shape[0]
120
+ t = timestep.expand(batch_size)
121
+
122
+ with torch.autocast(device_type="cuda", dtype=latents.dtype):
123
+ velocity_uncond = model(
124
+ latents,
125
+ t,
126
+ uncond_context,
127
+ token_weights=uncond_weights,
128
+ sparse_skip=[True] * batch_size,
129
+ )
130
+
131
+ velocity_cond = model(
132
+ latents,
133
+ t,
134
+ cond_context,
135
+ token_weights=cond_weights,
136
+ sparse_skip=[False] * batch_size,
137
+ )
138
+
139
+ # CFG interpolation.
140
+ velocity = velocity_uncond + cfg_scale * (
141
+ velocity_cond - velocity_uncond
142
+ )
143
+
144
+ return velocity.float()
145
+
146
+ @torch.inference_mode()
147
+ def generate(
148
+ self,
149
+ positive: str,
150
+ negative: str = '',
151
+ height: int = 256,
152
+ width: int = 256,
153
+ num_steps: int = 50,
154
+ cfg_scale: float = 4.0,
155
+ shift: float = 3.0,
156
+ seed: int = 0,
157
+ ):
158
+ torch.manual_seed(seed)
159
+ if torch.cuda.is_available():
160
+ torch.cuda.manual_seed_all(seed)
161
+
162
+ device = next(self.transformer.parameters()).device
163
+ dtype = next(self.transformer.parameters()).dtype
164
+
165
+ # Encode conditional and unconditional prompts.
166
+ cond_context, cond_weights = self.encode_text(
167
+ positive,
168
+ device,
169
+ dtype,
170
+ )
171
+
172
+ uncond_context, uncond_weights = self.encode_text(
173
+ negative,
174
+ device,
175
+ dtype,
176
+ )
177
+
178
+ latent_channels = 64
179
+ latent_height = height // 8
180
+ latent_width = width // 8
181
+
182
+ latents = torch.randn(
183
+ 1,
184
+ latent_channels,
185
+ latent_height,
186
+ latent_width,
187
+ device=device,
188
+ dtype=dtype,
189
+ )
190
+
191
+ timesteps = Pipeline.get_timestep_schedule(
192
+ num_steps=num_steps,
193
+ shift=shift,
194
+ device=device,
195
+ )
196
+
197
+ for step in range(num_steps):
198
+ t_cur = timesteps[step]
199
+ t_next = timesteps[step + 1]
200
+
201
+ dt = t_next - t_cur
202
+
203
+ timestep = t_cur.reshape(1)
204
+
205
+ velocity = Pipeline.predict_velocity(
206
+ model=self.transformer,
207
+ latents=latents,
208
+ timestep=timestep,
209
+ cond_context=cond_context,
210
+ cond_weights=cond_weights,
211
+ uncond_context=uncond_context,
212
+ uncond_weights=uncond_weights,
213
+ cfg_scale=cfg_scale,
214
+ )
215
+
216
+ # Euler integration from t=1 toward t=0.
217
+ latents = latents.float() + dt * velocity
218
+ latents = latents.to(dtype)
219
+
220
+ # Decode the final x prediction.
221
+ #
222
+ # Adjust this call if your VAE uses decode(latents) with a different
223
+ # latent scaling convention.
224
+ image = self.vae.decode(latents.float())
225
+
226
+ # Convert from BCHW float [0, 1] to a PIL image.
227
+ if isinstance(image, (tuple, list)):
228
+ image = image[0]
229
+
230
+ image = image.float().clamp(0.0, 1.0)
231
+ image = image[0].permute(1, 2, 0)
232
+ image = (image * 255.0).round().to(torch.uint8).cpu().numpy()
233
+
234
+ return Image.fromarray(image)
vae.py ADDED
@@ -0,0 +1,380 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import numpy as np
3
+ from safetensors.torch import load_file
4
+ import timm
5
+ import torch
6
+ import torch.nn as nn
7
+ import torch.nn.functional as F
8
+ from transformers import Dinov2Model
9
+ from typing import Optional, Tuple
10
+ from timm.models.vision_transformer import VisionTransformer
11
+
12
+
13
+ DINO_MODEL_NAME = "vit_base_patch14_dinov2.lvd142m"
14
+
15
+ DINO_MEAN = (0.485, 0.456, 0.406)
16
+ DINO_STD = (0.229, 0.224, 0.225)
17
+
18
+ DINO_PATCH_SIZE = 14
19
+ DINO_GRID = 37
20
+ LATENT_DOWNSAMPLE_FACTOR = 16
21
+ ENCODER_LAYERS = 6
22
+
23
+ class ResnetBlock(nn.Module):
24
+ def __init__(self, in_ch: int, out_ch: int) -> None:
25
+ super().__init__()
26
+ self.norm1 = nn.GroupNorm(32, in_ch, eps=1e-6, affine=True)
27
+ self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1)
28
+ self.norm2 = nn.GroupNorm(32, out_ch, eps=1e-6, affine=True)
29
+ self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1)
30
+ self.nin_shortcut = (
31
+ nn.Conv2d(in_ch, out_ch, 1) if in_ch != out_ch else nn.Identity()
32
+ )
33
+
34
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
35
+ h = self.conv1(F.silu(self.norm1(x)))
36
+ h = self.conv2(F.silu(self.norm2(h)))
37
+ return self.nin_shortcut(x) + h
38
+
39
+ class AttnBlock(nn.Module):
40
+ def __init__(self, channels: int) -> None:
41
+ super().__init__()
42
+ self.norm = nn.GroupNorm(32, channels, eps=1e-6, affine=True)
43
+ self.q = nn.Conv2d(channels, channels, kernel_size=1)
44
+ self.k = nn.Conv2d(channels, channels, kernel_size=1)
45
+ self.v = nn.Conv2d(channels, channels, kernel_size=1)
46
+ self.proj_out = nn.Conv2d(channels, channels, 1)
47
+
48
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
49
+ batch, channels, height, width = x.shape
50
+ normalized = self.norm(x)
51
+ # (B, C, H, W) -> (B, H*W, C)
52
+ q = self.q(normalized).flatten(2).transpose(1, 2)
53
+ k = self.k(normalized).flatten(2).transpose(1, 2)
54
+ v = self.v(normalized).flatten(2).transpose(1, 2)
55
+ h = F.scaled_dot_product_attention(q, k, v)
56
+ h = h.transpose(1, 2).reshape(batch, channels, height, width)
57
+ return x + self.proj_out(h)
58
+
59
+ class Upsample(nn.Module):
60
+ def __init__(self, channels: int, scale_factor: float) -> None:
61
+ super().__init__()
62
+ self.conv = nn.Conv2d(channels, channels, 3, padding=1)
63
+ self.scale_factor = scale_factor
64
+
65
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
66
+ return self.conv(F.interpolate(x, scale_factor=self.scale_factor, mode="nearest"))
67
+
68
+ class Decoder(nn.Module):
69
+ # ldm/modules/diffusionmodules/model.py
70
+ def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
71
+ attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
72
+ resolution, z_channels, tanh_out=False, use_linear_attn=False,
73
+ attn_op=AttnBlock,
74
+ conv3d=False,
75
+ time_compress=None,
76
+ **ignorekwargs):
77
+ super().__init__()
78
+ self.ch = ch
79
+ self.temb_ch = 0
80
+ self.num_resolutions = len(ch_mult)
81
+ self.num_res_blocks = num_res_blocks
82
+ self.resolution = resolution
83
+ self.in_channels = in_channels
84
+ self.tanh_out = tanh_out
85
+ self.carried = False
86
+
87
+ # compute block_in and curr_res at lowest res
88
+ block_in = ch*ch_mult[self.num_resolutions-1]
89
+ curr_res = resolution // 2**(self.num_resolutions-1)
90
+ self.z_shape = (1,z_channels,curr_res,curr_res)
91
+
92
+ # z to block_in
93
+ self.conv_in = nn.Conv2d(
94
+ z_channels,
95
+ block_in,
96
+ kernel_size=3,
97
+ stride=1,
98
+ padding=1)
99
+
100
+ # middle
101
+ self.mid = MidBlock(block_in)
102
+
103
+ # upsampling
104
+ self.up = nn.ModuleList()
105
+ for i_level in reversed(range(self.num_resolutions)):
106
+ block = nn.ModuleList()
107
+ attn = nn.ModuleList()
108
+ block_out = ch*ch_mult[i_level]
109
+ for i_block in range(self.num_res_blocks+1):
110
+ block.append(ResnetBlock(block_in, block_out))
111
+ block_in = block_out
112
+ if curr_res in attn_resolutions:
113
+ attn.append(AttnBlock(block_in))
114
+ up = nn.Module()
115
+ up.block = block
116
+ up.attn = attn
117
+ if i_level != 0:
118
+ scale_factor = 2.0
119
+ if time_compress is not None:
120
+ if i_level > math.log2(time_compress):
121
+ scale_factor = (1.0, 2.0, 2.0)
122
+
123
+ up.upsample = Upsample(block_in, scale_factor=scale_factor)
124
+ curr_res = curr_res * 2
125
+ self.up.insert(0, up) # prepend to get consistent order
126
+
127
+ self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
128
+ self.conv_out = nn.Conv2d(
129
+ block_in,
130
+ out_ch,
131
+ kernel_size=3,
132
+ stride=1,
133
+ padding=1)
134
+
135
+ def forward(self, z, **kwargs) -> list[torch.Tensor]:
136
+ # timestep embedding
137
+ temb = None
138
+ h = self.conv_in(z)
139
+ h = self.mid(h)
140
+
141
+ if self.carried:
142
+ h = torch.split(h, 2, dim=2)
143
+ else:
144
+ h = [ h ]
145
+ out = []
146
+ conv_carry_in = None
147
+ for i, h1 in enumerate(h):
148
+ conv_carry_out = []
149
+ if i == len(h) - 1:
150
+ conv_carry_out = None
151
+ for i_level in reversed(range(self.num_resolutions)):
152
+ for i_block in range(self.num_res_blocks+1):
153
+ h1 = self.up[i_level].block[i_block](h1)
154
+ if len(self.up[i_level].attn) > 0:
155
+ assert i == 0 #carried should not happen if attn exists
156
+ h1 = self.up[i_level].attn[i_block](h1)
157
+ if i_level != 0:
158
+ h1 = self.up[i_level].upsample(h1)
159
+
160
+ h1 = self.norm_out(h1)
161
+ h1 = F.silu(h1)
162
+ h1 = self.conv_out(h1)
163
+ if self.tanh_out:
164
+ h1 = torch.tanh(h1)
165
+ out.append(h1)
166
+ conv_carry_in = conv_carry_out
167
+
168
+ return out
169
+
170
+ class MidBlock(nn.Module):
171
+ def __init__(self, block_in):
172
+ super().__init__()
173
+ self.block_1 = ResnetBlock(block_in, block_in)
174
+ self.attn_1 = AttnBlock(block_in)
175
+ self.block_2 = ResnetBlock(block_in, block_in)
176
+
177
+ def forward(self, x):
178
+ x = self.block_1(x)
179
+ x = self.attn_1(x)
180
+ x = self.block_2(x)
181
+
182
+ return x
183
+
184
+ class VAE(nn.Module):
185
+ def __init__(self):
186
+ super().__init__()
187
+ self.latent_channels = 64
188
+ self._dino = [timm.create_model(
189
+ DINO_MODEL_NAME,
190
+ pretrained=True,
191
+ num_classes=0,
192
+ dynamic_img_size=True,
193
+ dynamic_img_pad=True
194
+ )]
195
+ # The call_dino() works without changing the output_fmt.
196
+ # self.dino.patch_embed.output_fmt = 'NCHW'
197
+ self.dino.eval()
198
+ # self._dino[0].patch_embed.strict_img_size = False
199
+ self.embed_dim = self.dino.embed_dim
200
+
201
+ # The original model uses the final six DINO transformer blocks.
202
+ self.feature_norms = nn.ModuleList([
203
+ nn.LayerNorm(self.embed_dim, eps=1e-6)
204
+ for _ in range(ENCODER_LAYERS)
205
+ ])
206
+ self.encoder_projection = nn.Conv2d(
207
+ self.embed_dim * ENCODER_LAYERS,
208
+ self.latent_channels // 2,
209
+ kernel_size=1,
210
+ )
211
+ self.semantic_projection = nn.Conv2d(
212
+ self.embed_dim,
213
+ self.latent_channels // 2,
214
+ kernel_size=1,
215
+ )
216
+ self.semantic_patch_embeddings = Dinov2PatchEmbeddings(self.embed_dim)
217
+
218
+ # Replace this with a PyTorch implementation.
219
+ self.decoder = Decoder(
220
+ ch=128,
221
+ in_channels=3,
222
+ out_ch=3,
223
+ ch_mult=(1, 1, 2, 2, 4),
224
+ num_res_blocks=2,
225
+ attn_resolutions=(16,),
226
+ resolution=256,
227
+ z_channels=self.latent_channels,
228
+ tanh_out=True,
229
+ )
230
+ self.register_buffer(
231
+ "latent_mean",
232
+ torch.empty(1, self.latent_channels, 1, 1),
233
+ )
234
+ self.register_buffer(
235
+ "latent_std",
236
+ torch.empty(1, self.latent_channels, 1, 1),
237
+ )
238
+ self.register_buffer(
239
+ "dino_mean",
240
+ torch.tensor(DINO_MEAN).view(1, 3, 1, 1),
241
+ persistent=False
242
+ )
243
+ self.register_buffer(
244
+ "dino_std",
245
+ torch.tensor(DINO_STD).view(1, 3, 1, 1),
246
+ persistent=False
247
+ )
248
+
249
+ @property
250
+ def dino(self) -> VisionTransformer:
251
+ return self._dino[0]
252
+
253
+ def call_dino(self, pixel_values, patch_embeddings=None):
254
+ # This was probably from Dinov2Embeddings.interpolate_pos_encoding() function.
255
+ if patch_embeddings is None:
256
+ x = self.dino.patch_embed(pixel_values)
257
+ else:
258
+ x = patch_embeddings(pixel_values)
259
+ if x.shape[-1] == self.dino.embed_dim:
260
+ x = x.flatten(1, 2)
261
+ else:
262
+ x = x.flatten(2).transpose(1, 2) # NCHW -> NLC
263
+ pos = self.dino.pos_embed.to(x.dtype)
264
+ patch_pos = pos[:, 1:].reshape(1, DINO_GRID, DINO_GRID, -1).permute(0, 3, 1, 2)
265
+ grid = (pixel_values.shape[-2] // DINO_PATCH_SIZE, pixel_values.shape[-1] // DINO_PATCH_SIZE)
266
+ patch_pos = F.interpolate(patch_pos, size=grid, mode="bicubic", antialias=True).flatten(2).transpose(1, 2)
267
+ pos = torch.cat([pos[:, :1], patch_pos], dim=1).to(x.dtype)
268
+ B, _, _ = x.shape
269
+ cls = self.dino.cls_token.expand(B, 1, -1).to(x.dtype)
270
+ x = torch.cat([cls, x], dim=1) + pos
271
+ hidden_states = []
272
+ for layer in self.dino.blocks:
273
+ x = layer(x)
274
+ hidden_states.append(x)
275
+
276
+ return hidden_states
277
+
278
+ def encode(self, x: torch.Tensor) -> torch.Tensor:
279
+ """
280
+ Input:
281
+ x: [-1, 1], shape [B, 3, H, W]
282
+
283
+ Output:
284
+ Normalized latent tensor.
285
+ """
286
+ batch_size, _, height, width = x.shape
287
+ grid_h = max(1, (height + LATENT_DOWNSAMPLE_FACTOR // 2) // LATENT_DOWNSAMPLE_FACTOR)
288
+ grid_w = max(1, (width + LATENT_DOWNSAMPLE_FACTOR // 2) // LATENT_DOWNSAMPLE_FACTOR)
289
+ target_height = grid_h * DINO_PATCH_SIZE
290
+ target_width = grid_w * DINO_PATCH_SIZE
291
+ x = F.interpolate(
292
+ x,
293
+ size=(target_height, target_width),
294
+ mode="bicubic",
295
+ align_corners=False,
296
+ antialias=True,
297
+ )
298
+
299
+ # Convert [-1, 1] to [0, 1], then apply ImageNet normalization.
300
+ # x = (x + 1.0) * 0.5
301
+ x = (x - self.dino_mean) / self.dino_std
302
+
303
+ print(self.dino.patch_embed.output_fmt)
304
+ hidden_states = self.call_dino(x)
305
+
306
+ # Match:
307
+ # dino_hidden_states(...)[-ENCODER_LAYERS:]
308
+ features = hidden_states[-ENCODER_LAYERS:]
309
+ normalized_features = []
310
+ for feature, norm in zip(features, self.feature_norms):
311
+ # Remove CLS token.
312
+ feature = feature[:, 1:]
313
+ feature = norm(feature)
314
+ normalized_features.append(feature)
315
+ features = torch.cat(normalized_features, dim=-1) # (B, patches, C * ENCODER_LAYERS)
316
+ features = (
317
+ features.transpose(1, 2)
318
+ .reshape(batch_size, -1, grid_h, grid_w)
319
+ ) # (B, C * ENCODER_LAYERS, grid_h, grid_w)
320
+
321
+ # Semantic branch uses the final transformer block followed by
322
+ # the model's final LayerNorm.
323
+ assert not torch.equal(self.dino.patch_embed.proj.weight, self.semantic_patch_embeddings.projection.weight)
324
+ semantic = self.call_dino(x, self.semantic_patch_embeddings)[-1][:, 1:]
325
+ semantic = self.dino.norm(semantic)
326
+ semantic = (
327
+ semantic.transpose(1, 2)
328
+ .reshape(batch_size, -1, grid_h, grid_w,)
329
+ )
330
+
331
+ z = torch.cat([self.encoder_projection(features), self.semantic_projection(semantic)], dim=1)
332
+
333
+ return (z - self.latent_mean) / self.latent_std
334
+
335
+ def decode(self, z: torch.Tensor) -> torch.Tensor:
336
+ z = z * self.latent_std + self.latent_mean
337
+
338
+ return self.decoder(z)
339
+
340
+ @torch.no_grad()
341
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
342
+ return self.decode(self.encode(x))
343
+
344
+ @staticmethod
345
+ def from_safetensors(path, device=None):
346
+ model = VAE()
347
+ sd = load_file(path)
348
+ for key in list(sd.keys()):
349
+ if key.startswith('encoder.'):
350
+ del sd[key]
351
+ model.load_state_dict(sd)
352
+ del sd
353
+
354
+ return model.to(device)
355
+
356
+ def to_numpy_array(self, x: torch.Tensor):
357
+ # Clip the tanh output.
358
+ x = (x.detach().clamp(-1, 1) + 1) / 2 * 255.0
359
+ if len(x.shape) == 4:
360
+ x = x.squeeze(0)
361
+ x = x.cpu().movedim(0, 2) # (CHW) -> (HWC)
362
+
363
+ return x.numpy().astype(np.uint8)
364
+
365
+ def to_tensor(self, image):
366
+ pixels = torch.from_numpy(np.array(image)).float()
367
+ pixels = pixels.permute(2, 0, 1)
368
+ # Dino requires [0, 1] values.
369
+ pixels = (pixels / 255.0) # * 2.0 - 1.0
370
+
371
+ return pixels.unsqueeze(0)
372
+
373
+ class Dinov2PatchEmbeddings(nn.Module):
374
+ def __init__(self, hidden_dim: int):
375
+ super().__init__()
376
+ self.projection = nn.Conv2d(3, hidden_dim, kernel_size=14, stride=14)
377
+
378
+ def forward(self, x):
379
+ # See transformers.models.dinov2.modeling_dinov2 for implementation.
380
+ return self.projection(x)