BlueSourceJY commited on
Commit
15e3097
·
verified ·
1 Parent(s): 4136ef6

Upload models/lightningdit_conv_general.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. models/lightningdit_conv_general.py +817 -0
models/lightningdit_conv_general.py ADDED
@@ -0,0 +1,817 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Lightning DiT's codes are built from original DiT & SiT.
3
+ (https://github.com/facebookresearch/DiT; https://github.com/willisma/SiT)
4
+ It demonstrates that a advanced DiT together with advanced diffusion skills
5
+ could also achieve a very promising result with 1.35 FID on ImageNet 256 generation.
6
+
7
+ Enjoy everyone, DiT strikes back!
8
+
9
+ by Maple (Jingfeng Yao) from HUST-VL
10
+ """
11
+
12
+ import os
13
+ import math
14
+ import numpy as np
15
+
16
+ import torch
17
+ import torch.nn as nn
18
+ import torch.nn.functional as F
19
+ from torch.utils.checkpoint import checkpoint
20
+
21
+ from timm.models.vision_transformer import PatchEmbed, Mlp
22
+ from models.swiglu_ffn import SwiGLUFFN
23
+ from models.pos_embed import VisionRotaryEmbeddingFast
24
+ from models.rmsnorm import RMSNorm
25
+ from visualize_attention import visualize_attention_matrix
26
+
27
+
28
+ def ensure_contiguous(tensor):
29
+ """确保张量内存布局连续,避免DDP梯度布局警告"""
30
+ if not tensor.is_contiguous():
31
+ return tensor.contiguous()
32
+ return tensor
33
+
34
+
35
+ @torch.compile
36
+ def modulate(x, shift, scale):
37
+ if shift is None:
38
+ return x * (1 + scale.unsqueeze(1))
39
+ return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
40
+
41
+ class Attention(nn.Module):
42
+ """
43
+ Attention module of LightningDiT.
44
+ """
45
+ def __init__(
46
+ self,
47
+ dim: int,
48
+ num_heads: int = 8,
49
+ qkv_bias: bool = False,
50
+ qk_norm: bool = False,
51
+ attn_drop: float = 0.,
52
+ proj_drop: float = 0.,
53
+ norm_layer: nn.Module = nn.LayerNorm,
54
+ fused_attn: bool = True,
55
+ use_rmsnorm: bool = False,
56
+ ) -> None:
57
+ super().__init__()
58
+ assert dim % num_heads == 0, 'dim should be divisible by num_heads'
59
+
60
+ self.num_heads = num_heads
61
+ self.head_dim = dim // num_heads
62
+ self.scale = self.head_dim ** -0.5
63
+ self.fused_attn = fused_attn
64
+
65
+ if use_rmsnorm:
66
+ norm_layer = RMSNorm
67
+
68
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
69
+ self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
70
+ self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
71
+ self.attn_drop = nn.Dropout(attn_drop)
72
+ self.proj = nn.Linear(dim, dim)
73
+ self.proj_drop = nn.Dropout(proj_drop)
74
+
75
+ def forward(self, x: torch.Tensor, rope=None) -> torch.Tensor:
76
+ B, N, C = x.shape
77
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
78
+ q, k, v = qkv.unbind(0)
79
+ q, k = self.q_norm(q), self.k_norm(k)
80
+
81
+ if rope is not None:
82
+ q = rope(q)
83
+ k = rope(k) #(B, self.num_heads, N, self.head_dim)
84
+
85
+ if self.fused_attn:
86
+ x = F.scaled_dot_product_attention(
87
+ q, k, v,
88
+ dropout_p=self.attn_drop.p if self.training else 0.,
89
+ )
90
+ else:
91
+ q = q * self.scale
92
+ attn = q @ k.transpose(-2, -1)
93
+ attn = attn.softmax(dim=-1)
94
+ attn = self.attn_drop(attn)
95
+ x = attn @ v
96
+
97
+ #------Store attention map for visualization------#
98
+ # q_vis = q * self.scale
99
+ # attn_vis = q_vis @ k.transpose(-2, -1)
100
+ # attn_vis = attn_vis.softmax(dim=-1)
101
+
102
+ # # Save attention weights as class attribute for visualization
103
+ # self.attn_weights = attn_vis.detach()
104
+ #-------------------------------------------------#
105
+
106
+
107
+ x = x.transpose(1, 2).reshape(B, N, C)
108
+ x = self.proj(x)
109
+ x = self.proj_drop(x)
110
+ return x
111
+
112
+ class AttentionConv(nn.Module):
113
+ """
114
+ Attention module of LightningDiT with convolutional downsampling for k and v.
115
+ """
116
+ def __init__(
117
+ self,
118
+ dim: int,
119
+ num_heads: int = 8,
120
+ qkv_bias: bool = False,
121
+ qk_norm: bool = False,
122
+ attn_drop: float = 0.,
123
+ proj_drop: float = 0.,
124
+ norm_layer: nn.Module = nn.LayerNorm,
125
+ fused_attn: bool = True,
126
+ use_rmsnorm: bool = False,
127
+ stride: int = 2, # 卷积步幅 s
128
+ kernel_size: int = 2, # 卷积核大小 k
129
+ ) -> None:
130
+ super().__init__()
131
+ assert dim % num_heads == 0, 'dim should be divisible by num_heads'
132
+
133
+ self.dim = dim
134
+ self.num_heads = num_heads
135
+ self.head_dim = dim // num_heads
136
+ self.scale = self.head_dim ** -0.5
137
+ self.fused_attn = fused_attn
138
+ self.downsample_factor = stride
139
+
140
+ if use_rmsnorm:
141
+ norm_layer = RMSNorm
142
+
143
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
144
+ self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
145
+ self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
146
+ self.attn_drop = nn.Dropout(attn_drop)
147
+ self.proj = nn.Linear(dim, dim)
148
+ self.proj_drop = nn.Dropout(proj_drop)
149
+
150
+ # 为 k 和 v 添加不同的卷积下采样层
151
+ self.k_conv = nn.Conv2d(
152
+ self.head_dim, self.head_dim,
153
+ kernel_size=kernel_size,
154
+ stride=stride,
155
+ groups=self.head_dim, # 深度可分离卷积,每个通道独立卷积
156
+ bias=False, # 避免偏置带来的内存布局问题
157
+ padding=int((kernel_size - stride) / 2)
158
+ )
159
+ self.v_conv = nn.Conv2d(
160
+ self.head_dim, self.head_dim,
161
+ kernel_size=kernel_size,
162
+ stride=stride,
163
+ groups=self.head_dim,
164
+ bias=False,
165
+ padding=int((kernel_size - stride) / 2)
166
+ )
167
+ print("kernel_size:", kernel_size, f"stride:{stride}")
168
+
169
+ # 确保卷积权重初始化后是连续的
170
+ with torch.no_grad():
171
+ self.k_conv.weight.data = self.k_conv.weight.data.contiguous()
172
+ self.v_conv.weight.data = self.v_conv.weight.data.contiguous()
173
+
174
+ def forward(self, x: torch.Tensor, rope=None) -> torch.Tensor:
175
+ B, N, C = x.shape
176
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
177
+ q, k, v = qkv.unbind(0)
178
+ q, k = self.q_norm(q), self.k_norm(k)
179
+
180
+ if rope is not None:
181
+ q = rope(q)
182
+ k = rope(k) #(B, self.num_heads, N, self.head_dim)
183
+
184
+ # 对 k 和 v 进行卷积下采样
185
+ # 假设 N = H * W,需要将序列重塑为 2D 形状进行卷积
186
+ H = W = int(N ** 0.5) # 假设是正方形的 patch grid
187
+ assert H * W == N, f"序列长度 {N} 必须是完全平方数"
188
+
189
+ # 重塑 k 和 v 为 (B*num_heads, head_dim, H, W) 格式进行卷积
190
+ k_reshaped = k.permute(0, 1, 3, 2).reshape(B*self.num_heads, self.head_dim, H, W).contiguous()
191
+ v_reshaped = v.permute(0, 1, 3, 2).reshape(B*self.num_heads, self.head_dim, H, W).contiguous()
192
+ # print("\n")
193
+ # print("k_reshaped.shape:", k_reshaped.shape)
194
+ # print("v_reshaped.shape:", v_reshaped.shape) #(8000,72,16,16)
195
+
196
+ # 应用卷积下采样
197
+ k_downsampled = self.k_conv(k_reshaped).contiguous() # (B*num_heads, head_dim, H//f, W//f)
198
+ v_downsampled = self.v_conv(v_reshaped).contiguous() # (B*num_heads, head_dim, H//f, W//f)
199
+ # print("k_downsampled.shape:", k_downsampled.shape)
200
+ # print("v_downsampled.shape:", v_downsampled.shape) #(8000,72,8,8)
201
+ # print("\n")
202
+
203
+
204
+ '''
205
+ kernel_size=stride=2
206
+ k_reshaped.shape: torch.Size([384, 64, 16, 16])
207
+ v_reshaped.shape: torch.Size([384, 64, 16, 16])
208
+ k_downsampled.shape: torch.Size([384, 64, 8, 8])
209
+ v_downsampled.shape: torch.Size([384, 64, 8, 8])
210
+ '''
211
+
212
+ '''
213
+ kernel_size=4, stride=2
214
+ k_reshaped.shape: torch.Size([384, 64, 16, 16])
215
+ v_reshaped.shape: torch.Size([384, 64, 16, 16])
216
+ k_downsampled.shape: torch.Size([384, 64, 7, 7])
217
+ v_downsampled.shape: torch.Size([384, 64, 7, 7])
218
+ '''
219
+
220
+ # 重新整理为注意力计算所需的形状
221
+ H_new, W_new = H // self.downsample_factor, W // self.downsample_factor
222
+ N_new = H_new * W_new
223
+
224
+ k = k_downsampled.reshape(B, self.num_heads, self.head_dim, N_new).permute(0, 1, 3, 2).contiguous()
225
+ v = v_downsampled.reshape(B, self.num_heads, self.head_dim, N_new).permute(0, 1, 3, 2).contiguous()
226
+
227
+ if self.fused_attn:
228
+ x = F.scaled_dot_product_attention(
229
+ q, k, v,
230
+ dropout_p=self.attn_drop.p if self.training else 0.,
231
+ )
232
+ else:
233
+ q = q * self.scale
234
+ attn = q @ k.transpose(-2, -1)
235
+ attn = attn.softmax(dim=-1)
236
+ attn = self.attn_drop(attn)
237
+ x = attn @ v
238
+
239
+ #------Store attention map for visualization------#
240
+ # q_vis = q * self.scale
241
+ # attn_vis = q_vis @ k.transpose(-2, -1)
242
+ # attn_vis = attn_vis.softmax(dim=-1)
243
+
244
+ # # Save attention weights as class attribute for visualization
245
+ # self.attn_weights = attn_vis.detach()
246
+ #-------------------------------------------------#
247
+
248
+ # 注意:这里 x 的序列长度是 N_new,需要通过插值恢复到原始长度 N
249
+ # current_seq_len = x.shape[2] # 当前序列长度(应该是 N_new)
250
+
251
+ # if current_seq_len != N: # 如果序列长度发生了变化
252
+ # # 将 x 重塑为 (B*num_heads, head_dim, H_new, W_new) 然后上采样
253
+ # x_reshaped = x.reshape(B * self.num_heads, self.head_dim, H_new, W_new).contiguous()
254
+ # # 使用双线性插值上采样回原始尺寸
255
+ # x_upsampled = F.interpolate(x_reshaped, size=(H, W), mode='bilinear', align_corners=False)
256
+ # x = x_upsampled.reshape(B, self.num_heads, N, self.head_dim).contiguous()
257
+
258
+ x = x.transpose(1, 2).reshape(B, N, C).contiguous()
259
+ x = self.proj(x)
260
+ x = self.proj_drop(x)
261
+ return x
262
+
263
+ class TimestepEmbedder(nn.Module):
264
+ """
265
+ Embeds scalar timesteps into vector representations.
266
+ Same as DiT.
267
+ """
268
+ def __init__(self, hidden_size: int, frequency_embedding_size: int = 256) -> None:
269
+ super().__init__()
270
+ self.frequency_embedding_size = frequency_embedding_size
271
+ self.mlp = nn.Sequential(
272
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),
273
+ nn.SiLU(),
274
+ nn.Linear(hidden_size, hidden_size, bias=True),
275
+ )
276
+
277
+ @staticmethod
278
+ def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor:
279
+ """
280
+ Create sinusoidal timestep embeddings.
281
+ Args:
282
+ t: A 1-D Tensor of N indices, one per batch element. These may be fractional.
283
+ dim: The dimension of the output.
284
+ max_period: Controls the minimum frequency of the embeddings.
285
+ Returns:
286
+ An (N, D) Tensor of positional embeddings.
287
+ """
288
+ # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
289
+ half = dim // 2
290
+ freqs = torch.exp(
291
+ -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
292
+ ).to(device=t.device)
293
+
294
+ args = t[:, None].float() * freqs[None]
295
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
296
+
297
+ if dim % 2:
298
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
299
+
300
+ return embedding
301
+
302
+ @torch.compile
303
+ def forward(self, t: torch.Tensor) -> torch.Tensor:
304
+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
305
+ t_emb = self.mlp(t_freq)
306
+ return t_emb
307
+
308
+
309
+ class LabelEmbedder(nn.Module):
310
+ """
311
+ Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
312
+ Same as DiT.
313
+ """
314
+ def __init__(self, num_classes, hidden_size, dropout_prob):
315
+ super().__init__()
316
+ use_cfg_embedding = dropout_prob > 0
317
+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)
318
+ self.num_classes = num_classes
319
+ self.dropout_prob = dropout_prob
320
+
321
+ def token_drop(self, labels, force_drop_ids=None):
322
+ """
323
+ Drops labels to enable classifier-free guidance.
324
+ """
325
+ if force_drop_ids is None:
326
+ drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob
327
+ else:
328
+ drop_ids = force_drop_ids == 1
329
+ labels = torch.where(drop_ids, self.num_classes, labels)
330
+ return labels
331
+
332
+ @torch.compile
333
+ def forward(self, labels, train, force_drop_ids=None):
334
+ use_dropout = self.dropout_prob > 0
335
+ if (train and use_dropout) or (force_drop_ids is not None):
336
+ labels = self.token_drop(labels, force_drop_ids)
337
+ embeddings = self.embedding_table(labels)
338
+ return embeddings
339
+
340
+ class LightningDiTBlock(nn.Module):
341
+ """
342
+ Lightning DiT Block. We add features including:
343
+ - ROPE
344
+ - QKNorm
345
+ - RMSNorm
346
+ - SwiGLU
347
+ - No shift AdaLN.
348
+ Not all of them are used in the final model, please refer to the paper for more details.
349
+ """
350
+ def __init__(
351
+ self,
352
+ hidden_size,
353
+ num_heads,
354
+ mlp_ratio=4.0,
355
+ use_qknorm=False,
356
+ use_swiglu=False,
357
+ use_rmsnorm=False,
358
+ wo_shift=False,
359
+ **block_kwargs
360
+ ):
361
+ super().__init__()
362
+
363
+ # Initialize normalization layers
364
+ if not use_rmsnorm:
365
+ self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
366
+ self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
367
+ else:
368
+ self.norm1 = RMSNorm(hidden_size)
369
+ self.norm2 = RMSNorm(hidden_size)
370
+
371
+ # Initialize attention layer
372
+ self.attn = Attention(
373
+ hidden_size,
374
+ num_heads=num_heads,
375
+ qkv_bias=True,
376
+ qk_norm=use_qknorm,
377
+ use_rmsnorm=use_rmsnorm,
378
+ **block_kwargs
379
+ )
380
+
381
+ # Initialize MLP layer
382
+ mlp_hidden_dim = int(hidden_size * mlp_ratio)
383
+ approx_gelu = lambda: nn.GELU(approximate="tanh")
384
+ if use_swiglu:
385
+ # here we did not use SwiGLU from xformers because it is not compatible with torch.compile for now.
386
+ self.mlp = SwiGLUFFN(hidden_size, int(2/3 * mlp_hidden_dim))
387
+ else:
388
+ self.mlp = Mlp(
389
+ in_features=hidden_size,
390
+ hidden_features=mlp_hidden_dim,
391
+ act_layer=approx_gelu,
392
+ drop=0
393
+ )
394
+
395
+ # Initialize AdaLN modulation
396
+ if wo_shift:
397
+ self.adaLN_modulation = nn.Sequential(
398
+ nn.SiLU(),
399
+ nn.Linear(hidden_size, 4 * hidden_size, bias=True)
400
+ )
401
+ else:
402
+ self.adaLN_modulation = nn.Sequential(
403
+ nn.SiLU(),
404
+ nn.Linear(hidden_size, 6 * hidden_size, bias=True)
405
+ )
406
+ self.wo_shift = wo_shift
407
+
408
+ @torch.compile
409
+ def forward(self, x, c, feat_rope=None):
410
+ if self.wo_shift:
411
+ scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(4, dim=1)
412
+ shift_msa = None
413
+ shift_mlp = None
414
+ else:
415
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1)
416
+
417
+ x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x), shift_msa, scale_msa), rope=feat_rope)
418
+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp))
419
+ return x
420
+
421
+ class LightningDiTBlock_Conv(nn.Module):
422
+ """
423
+ Lightning DiT Block with kv convolution. We add features including:
424
+ - ROPE
425
+ - QKNorm
426
+ - RMSNorm
427
+ - SwiGLU
428
+ - No shift AdaLN.
429
+ Not all of them are used in the final model, please refer to the paper for more details.
430
+ """
431
+ def __init__(
432
+ self,
433
+ hidden_size,
434
+ num_heads,
435
+ mlp_ratio=4.0,
436
+ use_qknorm=False,
437
+ use_swiglu=False,
438
+ use_rmsnorm=False,
439
+ wo_shift=False,
440
+ stride=2,
441
+ kernel_size=2,
442
+ **block_kwargs
443
+ ):
444
+ super().__init__()
445
+
446
+ # Initialize normalization layers
447
+ if not use_rmsnorm:
448
+ self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
449
+ self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
450
+ else:
451
+ self.norm1 = RMSNorm(hidden_size)
452
+ self.norm2 = RMSNorm(hidden_size)
453
+
454
+ # Initialize attention layer
455
+ self.attn = AttentionConv(
456
+ hidden_size,
457
+ num_heads=num_heads,
458
+ qkv_bias=True,
459
+ qk_norm=use_qknorm,
460
+ use_rmsnorm=use_rmsnorm,
461
+ stride=stride,
462
+ kernel_size=kernel_size,
463
+ **block_kwargs
464
+ )
465
+
466
+ # Initialize MLP layer
467
+ mlp_hidden_dim = int(hidden_size * mlp_ratio)
468
+ approx_gelu = lambda: nn.GELU(approximate="tanh")
469
+ if use_swiglu:
470
+ # here we did not use SwiGLU from xformers because it is not compatible with torch.compile for now.
471
+ self.mlp = SwiGLUFFN(hidden_size, int(2/3 * mlp_hidden_dim))
472
+ else:
473
+ self.mlp = Mlp(
474
+ in_features=hidden_size,
475
+ hidden_features=mlp_hidden_dim,
476
+ act_layer=approx_gelu,
477
+ drop=0
478
+ )
479
+
480
+ # Initialize AdaLN modulation
481
+ if wo_shift:
482
+ self.adaLN_modulation = nn.Sequential(
483
+ nn.SiLU(),
484
+ nn.Linear(hidden_size, 4 * hidden_size, bias=True)
485
+ )
486
+ else:
487
+ self.adaLN_modulation = nn.Sequential(
488
+ nn.SiLU(),
489
+ nn.Linear(hidden_size, 6 * hidden_size, bias=True)
490
+ )
491
+ self.wo_shift = wo_shift
492
+
493
+ @torch.compile
494
+ def forward(self, x, c, feat_rope=None):
495
+ if self.wo_shift:
496
+ scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(4, dim=1)
497
+ shift_msa = None
498
+ shift_mlp = None
499
+ else:
500
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1)
501
+
502
+ x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x), shift_msa, scale_msa), rope=feat_rope)
503
+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp))
504
+ return x
505
+
506
+ class FinalLayer(nn.Module):
507
+ """
508
+ The final layer of LightningDiT.
509
+ """
510
+ def __init__(self, hidden_size, patch_size, out_channels, use_rmsnorm=False):
511
+ super().__init__()
512
+ if not use_rmsnorm:
513
+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
514
+ else:
515
+ self.norm_final = RMSNorm(hidden_size)
516
+ self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
517
+ self.adaLN_modulation = nn.Sequential(
518
+ nn.SiLU(),
519
+ nn.Linear(hidden_size, 2 * hidden_size, bias=True)
520
+ )
521
+ @torch.compile
522
+ def forward(self, x, c):
523
+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
524
+ x = modulate(self.norm_final(x), shift, scale)
525
+ x = self.linear(x)
526
+ return x
527
+
528
+
529
+ class LightningDiT_Conv(nn.Module):
530
+ """
531
+ Diffusion model with a Transformer backbone.
532
+ """
533
+ def __init__(
534
+ self,
535
+ input_size=32,
536
+ patch_size=2,
537
+ in_channels=32,
538
+ hidden_size=1152,
539
+ depth=28,
540
+ num_heads=16,
541
+ mlp_ratio=4.0,
542
+ class_dropout_prob=0.1,
543
+ num_classes=1000,
544
+ learn_sigma=False,
545
+ use_qknorm=False,
546
+ use_swiglu=False,
547
+ use_rope=False,
548
+ use_rmsnorm=False,
549
+ wo_shift=False,
550
+ use_checkpoint=False,
551
+ stride_list=[],
552
+ kernel_size_list=[]
553
+ ):
554
+ super().__init__()
555
+ self.learn_sigma = learn_sigma
556
+ self.in_channels = in_channels
557
+ self.out_channels = in_channels if not learn_sigma else in_channels * 2
558
+ self.patch_size = patch_size
559
+ self.num_heads = num_heads
560
+ self.use_rope = use_rope
561
+ self.use_rmsnorm = use_rmsnorm
562
+ self.depth = depth
563
+ self.hidden_size = hidden_size
564
+ self.use_checkpoint = use_checkpoint
565
+ self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size, bias=True)
566
+ self.t_embedder = TimestepEmbedder(hidden_size)
567
+ self.y_embedder = LabelEmbedder(num_classes, hidden_size, class_dropout_prob)
568
+ num_patches = self.x_embedder.num_patches
569
+ # Will use fixed sin-cos embedding:
570
+ self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, hidden_size), requires_grad=False)
571
+
572
+ # use rotary position encoding, borrow from EVA
573
+ if self.use_rope:
574
+ half_head_dim = hidden_size // num_heads // 2
575
+ hw_seq_len = input_size // patch_size
576
+ self.feat_rope = VisionRotaryEmbeddingFast(
577
+ dim=half_head_dim,
578
+ pt_seq_len=hw_seq_len,
579
+ )
580
+ else:
581
+ self.feat_rope = None
582
+
583
+ self.blocks = nn.ModuleList()
584
+ self.stride_list = stride_list
585
+ self.kernel_size_list = kernel_size_list
586
+
587
+ assert len(stride_list) == len(kernel_size_list), "stride_list and kernel_size_list must have the same length"
588
+ assert len(stride_list) == depth
589
+
590
+ print("Initializing LightningDiT with the following block configurations:")
591
+ print("---------------------------------------")
592
+ print("stride_list:", stride_list)
593
+ print("kernel_size_list:", kernel_size_list)
594
+ print("---------------------------------------")
595
+
596
+ for i in range(depth):
597
+ stride = stride_list[i]
598
+ kernel_size = kernel_size_list[i]
599
+ if stride == 1 and kernel_size == 1:
600
+ block = LightningDiTBlock(hidden_size,
601
+ num_heads,
602
+ mlp_ratio=mlp_ratio,
603
+ use_qknorm=use_qknorm,
604
+ use_swiglu=use_swiglu,
605
+ use_rmsnorm=use_rmsnorm,
606
+ wo_shift=wo_shift,
607
+ )
608
+ print(f"idx={i}: add full block")
609
+ else:
610
+ block = LightningDiTBlock_Conv(hidden_size,
611
+ num_heads,
612
+ mlp_ratio=mlp_ratio,
613
+ use_qknorm=use_qknorm,
614
+ use_swiglu=use_swiglu,
615
+ use_rmsnorm=use_rmsnorm,
616
+ wo_shift=wo_shift,
617
+ stride=stride,
618
+ kernel_size=kernel_size,
619
+ )
620
+ print(f"idx={i}: stride={stride}, kernel_size={kernel_size}")
621
+ self.blocks.append(block)
622
+
623
+ self.final_layer = FinalLayer(hidden_size, patch_size, self.out_channels, use_rmsnorm=use_rmsnorm)
624
+ self.initialize_weights()
625
+
626
+ def initialize_weights(self):
627
+ # Initialize transformer layers:
628
+ def _basic_init(module):
629
+ if isinstance(module, nn.Linear):
630
+ torch.nn.init.xavier_uniform_(module.weight)
631
+ if module.bias is not None:
632
+ nn.init.constant_(module.bias, 0)
633
+ self.apply(_basic_init)
634
+
635
+ # Initialize (and freeze) pos_embed by sin-cos embedding:
636
+ pos_embed = get_2d_sincos_pos_embed(self.pos_embed.shape[-1], int(self.x_embedder.num_patches ** 0.5))
637
+ self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
638
+
639
+ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
640
+ w = self.x_embedder.proj.weight.data
641
+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
642
+ nn.init.constant_(self.x_embedder.proj.bias, 0)
643
+
644
+ # Initialize label embedding table:
645
+ nn.init.normal_(self.y_embedder.embedding_table.weight, std=0.02)
646
+
647
+ # Initialize timestep embedding MLP:
648
+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
649
+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
650
+
651
+ # Zero-out adaLN modulation layers in LightningDiT blocks:
652
+ for block in self.blocks:
653
+ nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
654
+ nn.init.constant_(block.adaLN_modulation[-1].bias, 0)
655
+
656
+ # Zero-out output layers:
657
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
658
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)
659
+ nn.init.constant_(self.final_layer.linear.weight, 0)
660
+ nn.init.constant_(self.final_layer.linear.bias, 0)
661
+
662
+ def unpatchify(self, x):
663
+ """
664
+ x: (N, T, patch_size**2 * C)
665
+ imgs: (N, H, W, C)
666
+ """
667
+ c = self.out_channels
668
+ p = self.x_embedder.patch_size[0]
669
+ h = w = int(x.shape[1] ** 0.5)
670
+ assert h * w == x.shape[1]
671
+
672
+ x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
673
+ x = torch.einsum('nhwpqc->nchpwq', x)
674
+ imgs = x.reshape(shape=(x.shape[0], c, h * p, h * p))
675
+ return imgs
676
+
677
+ def forward(self, x, t=None, y=None):
678
+ """
679
+ Forward pass of LightningDiT.
680
+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
681
+ t: (N,) tensor of diffusion timesteps
682
+ y: (N,) tensor of class labels
683
+ use_checkpoint: boolean to toggle checkpointing
684
+ """
685
+
686
+ use_checkpoint = self.use_checkpoint
687
+
688
+ x = self.x_embedder(x) + self.pos_embed # (N, T, D), where T = H * W / patch_size ** 2
689
+ t = self.t_embedder(t) # (N, D)
690
+ y = self.y_embedder(y, self.training) # (N, D)
691
+ c = t + y # (N, D)
692
+
693
+ for block in self.blocks:
694
+ if use_checkpoint:
695
+ x = checkpoint(block, x, c, self.feat_rope, use_reentrant=True)
696
+ else:
697
+ x = block(x, c, self.feat_rope)
698
+
699
+ x = self.final_layer(x, c) # (N, T, patch_size ** 2 * out_channels)
700
+ x = self.unpatchify(x) # (N, out_channels, H, W)
701
+
702
+ if self.learn_sigma:
703
+ x, _ = x.chunk(2, dim=1)
704
+ return x
705
+
706
+ def forward_with_cfg(self, x, t, y, cfg_scale, cfg_interval=None, cfg_interval_start=None):
707
+ """
708
+ Forward pass of LightningDiT, but also batches the unconditional forward pass for classifier-free guidance.
709
+ """
710
+ # https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynb
711
+ half = x[: len(x) // 2]
712
+ combined = torch.cat([half, half], dim=0)
713
+ model_out = self.forward(combined, t, y)
714
+ # For exact reproducibility reasons, we apply classifier-free guidance on only
715
+ # three channels by default. The standard approach to cfg applies it to all channels.
716
+ # This can be done by uncommenting the following line and commenting-out the line following that.
717
+ # eps, rest = model_out[:, :self.in_channels], model_out[:, self.in_channels:]
718
+ eps, rest = model_out[:, :3], model_out[:, 3:]
719
+ cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0)
720
+ half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps)
721
+
722
+ if cfg_interval is True:
723
+ timestep = t[0]
724
+ if timestep < cfg_interval_start:
725
+ half_eps = cond_eps
726
+
727
+ eps = torch.cat([half_eps, half_eps], dim=0)
728
+ return torch.cat([eps, rest], dim=1)
729
+
730
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0):
731
+ """
732
+ grid_size: int of the grid height and width
733
+ return:
734
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
735
+ """
736
+ grid_h = np.arange(grid_size, dtype=np.float32)
737
+ grid_w = np.arange(grid_size, dtype=np.float32)
738
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
739
+ grid = np.stack(grid, axis=0)
740
+
741
+ grid = grid.reshape([2, 1, grid_size, grid_size])
742
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
743
+ if cls_token and extra_tokens > 0:
744
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
745
+ return pos_embed
746
+
747
+
748
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
749
+ assert embed_dim % 2 == 0
750
+
751
+ # use half of dimensions to encode grid_h
752
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
753
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
754
+
755
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
756
+ return emb
757
+
758
+
759
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
760
+ """
761
+ embed_dim: output dimension for each position
762
+ pos: a list of positions to be encoded: size (M,)
763
+ out: (M, D)
764
+ """
765
+ assert embed_dim % 2 == 0
766
+ omega = np.arange(embed_dim // 2, dtype=np.float64)
767
+ omega /= embed_dim / 2.
768
+ omega = 1. / 10000**omega # (D/2,)
769
+
770
+ pos = pos.reshape(-1) # (M,)
771
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
772
+
773
+ emb_sin = np.sin(out) # (M, D/2)
774
+ emb_cos = np.cos(out) # (M, D/2)
775
+
776
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
777
+ return emb
778
+
779
+
780
+ #################################################################################
781
+ # LightningDiT Configs #
782
+ #################################################################################
783
+
784
+ def LightningDiT_XL_1(**kwargs):
785
+ return LightningDiT_Conv(depth=28, hidden_size=1152, patch_size=1, num_heads=16, **kwargs)
786
+
787
+ def LightningDiT_XL_2(**kwargs):
788
+ return LightningDiT_Conv(depth=28, hidden_size=1152, patch_size=2, num_heads=16, **kwargs)
789
+
790
+ def LightningDiT_L_2(**kwargs):
791
+ return LightningDiT_Conv(depth=24, hidden_size=1024, patch_size=2, num_heads=16, **kwargs)
792
+
793
+ def LightningDiT_B_1(**kwargs):
794
+ return LightningDiT_Conv(depth=12, hidden_size=768, patch_size=1, num_heads=12, **kwargs)
795
+
796
+ def LightningDiT_B_2(**kwargs):
797
+ return LightningDiT_Conv(depth=12, hidden_size=768, patch_size=2, num_heads=12, **kwargs)
798
+
799
+ def LightningDiT_1p0B_1(**kwargs):
800
+ return LightningDiT_Conv(depth=24, hidden_size=1536, patch_size=1, num_heads=24, **kwargs)
801
+
802
+ def LightningDiT_1p0B_2(**kwargs):
803
+ return LightningDiT_Conv(depth=24, hidden_size=1536, patch_size=2, num_heads=24, **kwargs)
804
+
805
+ def LightningDiT_1p6B_1(**kwargs):
806
+ return LightningDiT_Conv(depth=28, hidden_size=1792, patch_size=1, num_heads=28, **kwargs)
807
+
808
+ def LightningDiT_1p6B_2(**kwargs):
809
+ return LightningDiT_Conv(depth=28, hidden_size=1792, patch_size=2, num_heads=28, **kwargs)
810
+
811
+ LightningDiT_models = {
812
+ 'LightningDiT-B/1': LightningDiT_B_1, 'LightningDiT-B/2': LightningDiT_B_2,
813
+ 'LightningDiT-L/2': LightningDiT_L_2,
814
+ 'LightningDiT-XL/1': LightningDiT_XL_1, 'LightningDiT-XL/2': LightningDiT_XL_2,
815
+ 'LightningDiT-1p0B/1': LightningDiT_1p0B_1, 'LightningDiT-1p0B/2': LightningDiT_1p0B_2,
816
+ 'LightningDiT-1p6B/1': LightningDiT_1p6B_1, 'LightningDiT-1p6B/2': LightningDiT_1p6B_2,
817
+ }