BlueSourceJY commited on
Commit
ed2db84
·
verified ·
1 Parent(s): 7672ea4

Upload models/lightningdit_rot_head_vis.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. models/lightningdit_rot_head_vis.py +628 -0
models/lightningdit_rot_head_vis.py ADDED
@@ -0,0 +1,628 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Lightning DiT's codes are built from original DiT & SiT.
3
+ (https://github.com/facebookresearch/DiT; https://github.com/willisma/SiT)
4
+ It demonstrates that a advanced DiT together with advanced diffusion skills
5
+ could also achieve a very promising result with 1.35 FID on ImageNet 256 generation.
6
+
7
+ Enjoy everyone, DiT strikes back!
8
+
9
+ by Maple (Jingfeng Yao) from HUST-VL
10
+ """
11
+
12
+ import os
13
+ import math
14
+ import numpy as np
15
+
16
+ import torch
17
+ import torch.nn as nn
18
+ import torch.nn.functional as F
19
+ from torch.utils.checkpoint import checkpoint
20
+
21
+ from timm.models.vision_transformer import PatchEmbed, Mlp
22
+ from models.swiglu_ffn import SwiGLUFFN
23
+ from models.pos_embed import VisionRotaryEmbeddingFast
24
+ from models.rmsnorm import RMSNorm
25
+ from visualize_attention import visualize_attention_matrix
26
+ from attn_mask_rot import get_attention_mask
27
+
28
+ def rot_by_head(q,k,v,num_rot):
29
+ # q, k, v: (B, num_heads, N, head_dim)
30
+ # q1, q2, q3, q4 = torch.chunk(q, 4, dim=1)
31
+ # k1, k2, k3, k4 = torch.chunk(k, 4, dim=1)
32
+ # v1, v2, v3, v4 = torch.chunk(v, 4, dim=1)
33
+ q_list = torch.chunk(q, num_rot, dim=1)
34
+ k_list = torch.chunk(k, num_rot, dim=1)
35
+ v_list = torch.chunk(v, num_rot, dim=1)
36
+ q_rot_list = []
37
+ k_rot_list = []
38
+ v_rot_list = []
39
+ c = int(4 / num_rot)
40
+ for i, (q_i, k_i, v_i) in enumerate(zip(q_list, k_list, v_list)):
41
+ if i != 0:
42
+ B, num_heads_i, N, head_dim_i = q_i.shape
43
+ H = W = int(N ** 0.5)
44
+ q_i = q_i.reshape(B, num_heads_i, H, W, head_dim_i)
45
+ k_i = k_i.reshape(B, num_heads_i, H, W, head_dim_i)
46
+ v_i = v_i.reshape(B, num_heads_i, H, W, head_dim_i)
47
+ k = c*i
48
+ q_i_rot = torch.rot90(q_i, k=k, dims=(3, 2))
49
+ k_i_rot = torch.rot90(k_i, k=k, dims=(3, 2))
50
+ v_i_rot = torch.rot90(v_i, k=k, dims=(3, 2))
51
+ q_rot_list.append(q_i_rot.reshape(B, num_heads_i, N, head_dim_i))
52
+ k_rot_list.append(k_i_rot.reshape(B, num_heads_i, N, head_dim_i))
53
+ v_rot_list.append(v_i_rot.reshape(B, num_heads_i, N, head_dim_i))
54
+ else:
55
+ q_rot_list.append(q_i)
56
+ k_rot_list.append(k_i)
57
+ v_rot_list.append(v_i)
58
+ q_rot = torch.cat(q_rot_list, dim=1)
59
+ k_rot = torch.cat(k_rot_list, dim=1)
60
+ v_rot = torch.cat(v_rot_list, dim=1)
61
+
62
+ return q_rot, k_rot, v_rot
63
+
64
+ def recover(x_rot, num_rot):
65
+ x_rot_list = torch.chunk(x_rot, num_rot, dim=1)
66
+ x_list = []
67
+ c = int(4 / num_rot)
68
+ for i, x_i in enumerate(x_rot_list):
69
+ if i != 0:
70
+ B, num_heads_i, N, head_dim_i = x_i.shape
71
+ H = W = int(N ** 0.5)
72
+ x_i = x_i.reshape(B, num_heads_i, H, W, head_dim_i)
73
+ k = -c*i
74
+ x_i_recov = torch.rot90(x_i, k=k, dims=(3, 2))
75
+ x_list.append(x_i_recov.reshape(B, num_heads_i, N, head_dim_i))
76
+ else:
77
+ x_list.append(x_i)
78
+
79
+ return torch.cat(x_list, dim=1)
80
+
81
+ @torch.compile
82
+ def modulate(x, shift, scale):
83
+ if shift is None:
84
+ return x * (1 + scale.unsqueeze(1))
85
+ return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
86
+
87
+ class Attention(nn.Module):
88
+ """
89
+ Attention module of LightningDiT.
90
+ """
91
+ def __init__(
92
+ self,
93
+ dim: int,
94
+ num_heads: int = 8,
95
+ qkv_bias: bool = False,
96
+ qk_norm: bool = False,
97
+ attn_drop: float = 0.,
98
+ proj_drop: float = 0.,
99
+ norm_layer: nn.Module = nn.LayerNorm,
100
+ fused_attn: bool = True,
101
+ use_rmsnorm: bool = False,
102
+ num_rot: int = 4
103
+ ) -> None:
104
+ super().__init__()
105
+ assert dim % num_heads == 0, 'dim should be divisible by num_heads'
106
+
107
+ self.num_heads = num_heads
108
+ self.head_dim = dim // num_heads
109
+ self.scale = self.head_dim ** -0.5
110
+ self.fused_attn = fused_attn
111
+
112
+ if use_rmsnorm:
113
+ norm_layer = RMSNorm
114
+
115
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
116
+ self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
117
+ self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
118
+ self.attn_drop = nn.Dropout(attn_drop)
119
+ self.proj = nn.Linear(dim, dim)
120
+ self.proj_drop = nn.Dropout(proj_drop)
121
+
122
+ self.num_rot = num_rot
123
+
124
+ def forward(self, x: torch.Tensor, rope=None) -> torch.Tensor:
125
+ B, N, C = x.shape
126
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
127
+ q, k, v = qkv.unbind(0)
128
+ q, k = self.q_norm(q), self.k_norm(k)
129
+
130
+ if rope is not None:
131
+ q = rope(q)
132
+ k = rope(k) #(B, self.num_heads, N, self.head_dim)
133
+
134
+ # divide into groups by heads
135
+ assert self.num_rot == 4
136
+ q_rot, k_rot, v_rot = rot_by_head(q, k, v, self.num_rot) #(B, self.num_heads, N, self.head_dim)
137
+
138
+ if self.fused_attn:
139
+ x = F.scaled_dot_product_attention(
140
+ q_rot, k_rot, v_rot,
141
+ is_causal=True,
142
+ dropout_p=self.attn_drop.p if self.training else 0.,
143
+ )
144
+ else:
145
+ q = q * self.scale
146
+ attn = q @ k.transpose(-2, -1)
147
+ attn = attn.softmax(dim=-1)
148
+ attn = self.attn_drop(attn)
149
+ x = attn @ v
150
+
151
+ x = recover(x, self.num_rot) #(B, self.num_heads, N, self.head_dim)
152
+
153
+ # #------Store attention map for visualization------#
154
+ # q_vis = q * self.scale
155
+ # attn_vis = q_vis @ k.transpose(-2, -1)
156
+ # # attn_vis = attn_vis.softmax(dim=-1)
157
+ # mask_0 = get_attention_mask(16, 0, device=attn_vis.device) # Assuming N=256 for 16x16 patches
158
+ # mask_0 = mask_0.unsqueeze(0).unsqueeze(0).repeat(1,4,1,1) # (1, 4, N, N)
159
+ # mask_90 = get_attention_mask(16, 90, device=attn_vis.device) # Assuming N=256 for 16x16 patches
160
+ # mask_90 = mask_90.unsqueeze(0).unsqueeze(0).repeat(1,4,1,1) # (1, 4, N, N)
161
+ # mask_180 = get_attention_mask(16, 180, device=attn_vis.device) # Assuming N=256 for 16x16 patches
162
+ # mask_180 = mask_180.unsqueeze(0).unsqueeze(0).repeat(1,4,1,1) # (1, 4, N, N)
163
+ # mask_270 = get_attention_mask(16, 270, device=attn_vis.device) # Assuming N=256 for 16x16 patches
164
+ # mask_270 = mask_270.unsqueeze(0).unsqueeze(0).repeat(1,4,1,1) # (1, 4, N, N)
165
+ # mask = torch.cat([mask_0, mask_90, mask_180, mask_270], dim=1) # (1, num_heads=16, N, N)
166
+
167
+ # attn_vis = attn_vis + mask # Apply the same mask used in forward pass to the attention weights for visualization
168
+
169
+ # # print("**************")
170
+ # # print("q.shape:", q_vis.shape) # (B, num_heads=16, N=256, head_dim=72)
171
+ # # print("k.shape:", k.shape) # (B, num_heads=16, N=256, head_dim=72)
172
+ # # print("attn_vis.shape:", attn_vis.shape) # (B, num_heads=16, N=256, N=256)
173
+ # # print("**************")
174
+
175
+ # # Save attention weights as class attribute for visualization
176
+ # attn_vis = attn_vis.softmax(dim=-1)
177
+ # self.attn_weights = attn_vis.detach()
178
+ # #-------------------------------------------------#
179
+
180
+
181
+ x = x.transpose(1, 2).reshape(B, N, C)
182
+ x = self.proj(x)
183
+ x = self.proj_drop(x)
184
+ return x
185
+
186
+
187
+ class TimestepEmbedder(nn.Module):
188
+ """
189
+ Embeds scalar timesteps into vector representations.
190
+ Same as DiT.
191
+ """
192
+ def __init__(self, hidden_size: int, frequency_embedding_size: int = 256) -> None:
193
+ super().__init__()
194
+ self.frequency_embedding_size = frequency_embedding_size
195
+ self.mlp = nn.Sequential(
196
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),
197
+ nn.SiLU(),
198
+ nn.Linear(hidden_size, hidden_size, bias=True),
199
+ )
200
+
201
+ @staticmethod
202
+ def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor:
203
+ """
204
+ Create sinusoidal timestep embeddings.
205
+ Args:
206
+ t: A 1-D Tensor of N indices, one per batch element. These may be fractional.
207
+ dim: The dimension of the output.
208
+ max_period: Controls the minimum frequency of the embeddings.
209
+ Returns:
210
+ An (N, D) Tensor of positional embeddings.
211
+ """
212
+ # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
213
+ half = dim // 2
214
+ freqs = torch.exp(
215
+ -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
216
+ ).to(device=t.device)
217
+
218
+ args = t[:, None].float() * freqs[None]
219
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
220
+
221
+ if dim % 2:
222
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
223
+
224
+ return embedding
225
+
226
+ @torch.compile
227
+ def forward(self, t: torch.Tensor) -> torch.Tensor:
228
+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
229
+ t_emb = self.mlp(t_freq)
230
+ return t_emb
231
+
232
+
233
+ class LabelEmbedder(nn.Module):
234
+ """
235
+ Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
236
+ Same as DiT.
237
+ """
238
+ def __init__(self, num_classes, hidden_size, dropout_prob):
239
+ super().__init__()
240
+ use_cfg_embedding = dropout_prob > 0
241
+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)
242
+ self.num_classes = num_classes
243
+ self.dropout_prob = dropout_prob
244
+
245
+ def token_drop(self, labels, force_drop_ids=None):
246
+ """
247
+ Drops labels to enable classifier-free guidance.
248
+ """
249
+ if force_drop_ids is None:
250
+ drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob
251
+ else:
252
+ drop_ids = force_drop_ids == 1
253
+ labels = torch.where(drop_ids, self.num_classes, labels)
254
+ return labels
255
+
256
+ @torch.compile
257
+ def forward(self, labels, train, force_drop_ids=None):
258
+ use_dropout = self.dropout_prob > 0
259
+ if (train and use_dropout) or (force_drop_ids is not None):
260
+ labels = self.token_drop(labels, force_drop_ids)
261
+ embeddings = self.embedding_table(labels)
262
+ return embeddings
263
+
264
+ class LightningDiTBlock(nn.Module):
265
+ """
266
+ Lightning DiT Block. We add features including:
267
+ - ROPE
268
+ - QKNorm
269
+ - RMSNorm
270
+ - SwiGLU
271
+ - No shift AdaLN.
272
+ Not all of them are used in the final model, please refer to the paper for more details.
273
+ """
274
+ def __init__(
275
+ self,
276
+ hidden_size,
277
+ num_heads,
278
+ mlp_ratio=4.0,
279
+ use_qknorm=False,
280
+ use_swiglu=False,
281
+ use_rmsnorm=False,
282
+ wo_shift=False,
283
+ num_rot=4,
284
+ **block_kwargs
285
+ ):
286
+ super().__init__()
287
+
288
+ # Initialize normalization layers
289
+ if not use_rmsnorm:
290
+ self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
291
+ self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
292
+ else:
293
+ self.norm1 = RMSNorm(hidden_size)
294
+ self.norm2 = RMSNorm(hidden_size)
295
+
296
+ # Initialize attention layer
297
+ self.attn = Attention(
298
+ hidden_size,
299
+ num_heads=num_heads,
300
+ qkv_bias=True,
301
+ qk_norm=use_qknorm,
302
+ use_rmsnorm=use_rmsnorm,
303
+ num_rot=num_rot,
304
+ **block_kwargs
305
+ )
306
+
307
+ # Initialize MLP layer
308
+ mlp_hidden_dim = int(hidden_size * mlp_ratio)
309
+ approx_gelu = lambda: nn.GELU(approximate="tanh")
310
+ if use_swiglu:
311
+ # here we did not use SwiGLU from xformers because it is not compatible with torch.compile for now.
312
+ self.mlp = SwiGLUFFN(hidden_size, int(2/3 * mlp_hidden_dim))
313
+ else:
314
+ self.mlp = Mlp(
315
+ in_features=hidden_size,
316
+ hidden_features=mlp_hidden_dim,
317
+ act_layer=approx_gelu,
318
+ drop=0
319
+ )
320
+
321
+ # Initialize AdaLN modulation
322
+ if wo_shift:
323
+ self.adaLN_modulation = nn.Sequential(
324
+ nn.SiLU(),
325
+ nn.Linear(hidden_size, 4 * hidden_size, bias=True)
326
+ )
327
+ else:
328
+ self.adaLN_modulation = nn.Sequential(
329
+ nn.SiLU(),
330
+ nn.Linear(hidden_size, 6 * hidden_size, bias=True)
331
+ )
332
+ self.wo_shift = wo_shift
333
+
334
+ @torch.compile
335
+ def forward(self, x, c, feat_rope=None):
336
+ if self.wo_shift:
337
+ scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(4, dim=1)
338
+ shift_msa = None
339
+ shift_mlp = None
340
+ else:
341
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1)
342
+
343
+ x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x), shift_msa, scale_msa), rope=feat_rope)
344
+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp))
345
+ return x
346
+
347
+ class FinalLayer(nn.Module):
348
+ """
349
+ The final layer of LightningDiT.
350
+ """
351
+ def __init__(self, hidden_size, patch_size, out_channels, use_rmsnorm=False):
352
+ super().__init__()
353
+ if not use_rmsnorm:
354
+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
355
+ else:
356
+ self.norm_final = RMSNorm(hidden_size)
357
+ self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
358
+ self.adaLN_modulation = nn.Sequential(
359
+ nn.SiLU(),
360
+ nn.Linear(hidden_size, 2 * hidden_size, bias=True)
361
+ )
362
+ @torch.compile
363
+ def forward(self, x, c):
364
+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
365
+ x = modulate(self.norm_final(x), shift, scale)
366
+ x = self.linear(x)
367
+ return x
368
+
369
+
370
+ class LightningDiT(nn.Module):
371
+ """
372
+ Diffusion model with a Transformer backbone.
373
+ """
374
+ def __init__(
375
+ self,
376
+ input_size=32,
377
+ patch_size=2,
378
+ in_channels=32,
379
+ hidden_size=1152,
380
+ depth=28,
381
+ num_heads=16,
382
+ mlp_ratio=4.0,
383
+ class_dropout_prob=0.1,
384
+ num_classes=1000,
385
+ learn_sigma=False,
386
+ use_qknorm=False,
387
+ use_swiglu=False,
388
+ use_rope=False,
389
+ use_rmsnorm=False,
390
+ wo_shift=False,
391
+ num_rot=4,
392
+ use_checkpoint=False,
393
+ ):
394
+ super().__init__()
395
+ self.learn_sigma = learn_sigma
396
+ self.in_channels = in_channels
397
+ self.out_channels = in_channels if not learn_sigma else in_channels * 2
398
+ self.patch_size = patch_size
399
+ self.num_heads = num_heads
400
+ self.use_rope = use_rope
401
+ self.use_rmsnorm = use_rmsnorm
402
+ self.depth = depth
403
+ self.hidden_size = hidden_size
404
+ self.use_checkpoint = use_checkpoint
405
+ self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size, bias=True)
406
+ self.t_embedder = TimestepEmbedder(hidden_size)
407
+ self.y_embedder = LabelEmbedder(num_classes, hidden_size, class_dropout_prob)
408
+ num_patches = self.x_embedder.num_patches
409
+ # Will use fixed sin-cos embedding:
410
+ self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, hidden_size), requires_grad=False)
411
+
412
+ # use rotary position encoding, borrow from EVA
413
+ if self.use_rope:
414
+ half_head_dim = hidden_size // num_heads // 2
415
+ hw_seq_len = input_size // patch_size
416
+ self.feat_rope = VisionRotaryEmbeddingFast(
417
+ dim=half_head_dim,
418
+ pt_seq_len=hw_seq_len,
419
+ )
420
+ else:
421
+ self.feat_rope = None
422
+
423
+ self.blocks = nn.ModuleList([
424
+ LightningDiTBlock(hidden_size,
425
+ num_heads,
426
+ mlp_ratio=mlp_ratio,
427
+ use_qknorm=use_qknorm,
428
+ use_swiglu=use_swiglu,
429
+ use_rmsnorm=use_rmsnorm,
430
+ wo_shift=wo_shift,
431
+ num_rot=num_rot,
432
+ ) for _ in range(depth)
433
+ ])
434
+ self.final_layer = FinalLayer(hidden_size, patch_size, self.out_channels, use_rmsnorm=use_rmsnorm)
435
+ self.initialize_weights()
436
+
437
+ def initialize_weights(self):
438
+ # Initialize transformer layers:
439
+ def _basic_init(module):
440
+ if isinstance(module, nn.Linear):
441
+ torch.nn.init.xavier_uniform_(module.weight)
442
+ if module.bias is not None:
443
+ nn.init.constant_(module.bias, 0)
444
+ self.apply(_basic_init)
445
+
446
+ # Initialize (and freeze) pos_embed by sin-cos embedding:
447
+ pos_embed = get_2d_sincos_pos_embed(self.pos_embed.shape[-1], int(self.x_embedder.num_patches ** 0.5))
448
+ self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
449
+
450
+ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
451
+ w = self.x_embedder.proj.weight.data
452
+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
453
+ nn.init.constant_(self.x_embedder.proj.bias, 0)
454
+
455
+ # Initialize label embedding table:
456
+ nn.init.normal_(self.y_embedder.embedding_table.weight, std=0.02)
457
+
458
+ # Initialize timestep embedding MLP:
459
+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
460
+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
461
+
462
+ # Zero-out adaLN modulation layers in LightningDiT blocks:
463
+ for block in self.blocks:
464
+ nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
465
+ nn.init.constant_(block.adaLN_modulation[-1].bias, 0)
466
+
467
+ # Zero-out output layers:
468
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
469
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)
470
+ nn.init.constant_(self.final_layer.linear.weight, 0)
471
+ nn.init.constant_(self.final_layer.linear.bias, 0)
472
+
473
+ def unpatchify(self, x):
474
+ """
475
+ x: (N, T, patch_size**2 * C)
476
+ imgs: (N, H, W, C)
477
+ """
478
+ c = self.out_channels
479
+ p = self.x_embedder.patch_size[0]
480
+ h = w = int(x.shape[1] ** 0.5)
481
+ assert h * w == x.shape[1]
482
+
483
+ x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
484
+ x = torch.einsum('nhwpqc->nchpwq', x)
485
+ imgs = x.reshape(shape=(x.shape[0], c, h * p, h * p))
486
+ return imgs
487
+
488
+ def forward(self, x, t=None, y=None):
489
+ """
490
+ Forward pass of LightningDiT.
491
+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
492
+ t: (N,) tensor of diffusion timesteps
493
+ y: (N,) tensor of class labels
494
+ use_checkpoint: boolean to toggle checkpointing
495
+ """
496
+
497
+ use_checkpoint = self.use_checkpoint
498
+ # print("x.shape:", x.shape) # (N, C, H, W)
499
+ x = self.x_embedder(x) + self.pos_embed # (N, T, D), where T = H * W / patch_size ** 2
500
+ t = self.t_embedder(t) # (N, D)
501
+ y = self.y_embedder(y, self.training) # (N, D)
502
+ c = t + y # (N, D)
503
+
504
+ for block in self.blocks:
505
+ if use_checkpoint:
506
+ x = checkpoint(block, x, c, self.feat_rope, use_reentrant=True)
507
+ else:
508
+ x = block(x, c, self.feat_rope)
509
+
510
+ x = self.final_layer(x, c) # (N, T, patch_size ** 2 * out_channels)
511
+ x = self.unpatchify(x) # (N, out_channels, H, W)
512
+
513
+ if self.learn_sigma:
514
+ x, _ = x.chunk(2, dim=1)
515
+ return x
516
+
517
+ def forward_with_cfg(self, x, t, y, cfg_scale, cfg_interval=None, cfg_interval_start=None):
518
+ """
519
+ Forward pass of LightningDiT, but also batches the unconditional forward pass for classifier-free guidance.
520
+ """
521
+ # https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynb
522
+ half = x[: len(x) // 2]
523
+ combined = torch.cat([half, half], dim=0)
524
+ model_out = self.forward(combined, t, y)
525
+ # For exact reproducibility reasons, we apply classifier-free guidance on only
526
+ # three channels by default. The standard approach to cfg applies it to all channels.
527
+ # This can be done by uncommenting the following line and commenting-out the line following that.
528
+ # eps, rest = model_out[:, :self.in_channels], model_out[:, self.in_channels:]
529
+ eps, rest = model_out[:, :3], model_out[:, 3:]
530
+ cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0)
531
+ half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps)
532
+
533
+ if cfg_interval is True:
534
+ timestep = t[0]
535
+ if timestep < cfg_interval_start:
536
+ half_eps = cond_eps
537
+
538
+ eps = torch.cat([half_eps, half_eps], dim=0)
539
+ return torch.cat([eps, rest], dim=1)
540
+
541
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0):
542
+ """
543
+ grid_size: int of the grid height and width
544
+ return:
545
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
546
+ """
547
+ grid_h = np.arange(grid_size, dtype=np.float32)
548
+ grid_w = np.arange(grid_size, dtype=np.float32)
549
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
550
+ grid = np.stack(grid, axis=0)
551
+
552
+ grid = grid.reshape([2, 1, grid_size, grid_size])
553
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
554
+ if cls_token and extra_tokens > 0:
555
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
556
+ return pos_embed
557
+
558
+
559
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
560
+ assert embed_dim % 2 == 0
561
+
562
+ # use half of dimensions to encode grid_h
563
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
564
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
565
+
566
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
567
+ return emb
568
+
569
+
570
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
571
+ """
572
+ embed_dim: output dimension for each position
573
+ pos: a list of positions to be encoded: size (M,)
574
+ out: (M, D)
575
+ """
576
+ assert embed_dim % 2 == 0
577
+ omega = np.arange(embed_dim // 2, dtype=np.float64)
578
+ omega /= embed_dim / 2.
579
+ omega = 1. / 10000**omega # (D/2,)
580
+
581
+ pos = pos.reshape(-1) # (M,)
582
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
583
+
584
+ emb_sin = np.sin(out) # (M, D/2)
585
+ emb_cos = np.cos(out) # (M, D/2)
586
+
587
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
588
+ return emb
589
+
590
+
591
+ #################################################################################
592
+ # LightningDiT Configs #
593
+ #################################################################################
594
+
595
+ def LightningDiT_XL_1(**kwargs):
596
+ return LightningDiT(depth=28, hidden_size=1152, patch_size=1, num_heads=16, **kwargs)
597
+
598
+ def LightningDiT_XL_2(**kwargs):
599
+ return LightningDiT(depth=28, hidden_size=1152, patch_size=2, num_heads=16, **kwargs)
600
+
601
+ def LightningDiT_L_2(**kwargs):
602
+ return LightningDiT(depth=24, hidden_size=1024, patch_size=2, num_heads=16, **kwargs)
603
+
604
+ def LightningDiT_B_1(**kwargs):
605
+ return LightningDiT(depth=12, hidden_size=768, patch_size=1, num_heads=12, **kwargs)
606
+
607
+ def LightningDiT_B_2(**kwargs):
608
+ return LightningDiT(depth=12, hidden_size=768, patch_size=2, num_heads=12, **kwargs)
609
+
610
+ def LightningDiT_1p0B_1(**kwargs):
611
+ return LightningDiT(depth=24, hidden_size=1536, patch_size=1, num_heads=24, **kwargs)
612
+
613
+ def LightningDiT_1p0B_2(**kwargs):
614
+ return LightningDiT(depth=24, hidden_size=1536, patch_size=2, num_heads=24, **kwargs)
615
+
616
+ def LightningDiT_1p6B_1(**kwargs):
617
+ return LightningDiT(depth=28, hidden_size=1792, patch_size=1, num_heads=28, **kwargs)
618
+
619
+ def LightningDiT_1p6B_2(**kwargs):
620
+ return LightningDiT(depth=28, hidden_size=1792, patch_size=2, num_heads=28, **kwargs)
621
+
622
+ LightningDiT_models = {
623
+ 'LightningDiT-B/1': LightningDiT_B_1, 'LightningDiT-B/2': LightningDiT_B_2,
624
+ 'LightningDiT-L/2': LightningDiT_L_2,
625
+ 'LightningDiT-XL/1': LightningDiT_XL_1, 'LightningDiT-XL/2': LightningDiT_XL_2,
626
+ 'LightningDiT-1p0B/1': LightningDiT_1p0B_1, 'LightningDiT-1p0B/2': LightningDiT_1p0B_2,
627
+ 'LightningDiT-1p6B/1': LightningDiT_1p6B_1, 'LightningDiT-1p6B/2': LightningDiT_1p6B_2,
628
+ }