PRIYANSHUDHAKED commited on
Commit
21b654c
·
verified ·
1 Parent(s): ad1a612

Delete src/miocodec/module/transformer.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/miocodec/module/transformer.py +0 -652
src/miocodec/module/transformer.py DELETED
@@ -1,652 +0,0 @@
1
- # Adapted from https://github.com/meta-llama/llama3/blob/main/llama/model.py
2
- # Copyright (c) Meta Platforms, Inc. and affiliates.
3
- # This software may be used and distributed in accordance with the terms of the Llama 3 Community License Agreement.
4
-
5
-
6
- import torch
7
- import torch.nn.functional as F
8
- from torch import nn
9
-
10
- from ..util import get_logger
11
- from .adaln_zero import AdaLNZero
12
-
13
-
14
- logger = get_logger()
15
-
16
- _FLASH_CPU_WARNED = False
17
-
18
-
19
- def _warn_flash_cpu_once():
20
- global _FLASH_CPU_WARNED
21
- if _FLASH_CPU_WARNED:
22
- return
23
- logger.warning("FlashAttention is installed but input is on CPU; falling back to PyTorch SDPA.")
24
- _FLASH_CPU_WARNED = True
25
-
26
-
27
- try:
28
- from flash_attn import flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache
29
-
30
- FLASH_ATTN_AVAILABLE = True
31
- except ImportError:
32
- FLASH_ATTN_AVAILABLE = False
33
- logger.warning(
34
- "FlashAttention is not installed. Falling back to PyTorch SDPA implementation. There is no warranty that the model will work correctly."
35
- )
36
-
37
-
38
- def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):
39
- freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
40
- t = torch.arange(end, device=freqs.device, dtype=torch.float32)
41
- freqs = torch.outer(t, freqs)
42
- freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64
43
- return freqs_cis
44
-
45
-
46
- def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):
47
- ndim = x.ndim
48
- assert 0 <= 1 < ndim
49
- assert freqs_cis.shape == (x.shape[1], x.shape[-1])
50
- shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]
51
- return freqs_cis.view(*shape)
52
-
53
-
54
- def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:
55
- x_ = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
56
- freqs_cis = reshape_for_broadcast(freqs_cis, x_)
57
- x_out = torch.view_as_real(x_ * freqs_cis).flatten(3)
58
- return x_out.type_as(x)
59
-
60
-
61
- class Attention(nn.Module):
62
- def __init__(
63
- self,
64
- dim: int,
65
- n_heads: int,
66
- dropout: float,
67
- window_size: int | None,
68
- qkv_bias: bool = False,
69
- proj_bias: bool = False,
70
- use_flash_attention: bool = False,
71
- causal: bool = False,
72
- ):
73
- super().__init__()
74
- self.n_heads = n_heads
75
- self.head_dim = dim // n_heads
76
-
77
- self.wq = nn.Linear(dim, n_heads * self.head_dim, bias=qkv_bias)
78
- self.wk = nn.Linear(dim, n_heads * self.head_dim, bias=qkv_bias)
79
- self.wv = nn.Linear(dim, n_heads * self.head_dim, bias=qkv_bias)
80
- self.wo = nn.Linear(n_heads * self.head_dim, dim, bias=proj_bias)
81
-
82
- self.scale = self.head_dim**-0.5
83
- self.dropout = dropout
84
-
85
- # Enable local attention if window_size is specified
86
- self.use_local_attention = window_size is not None
87
- if self.use_local_attention:
88
- assert window_size % 2 == 1, "Window size must be odd for local attention."
89
- self.window_per_side = window_size // 2
90
-
91
- self.use_flash_attention = use_flash_attention
92
-
93
- self.causal = causal
94
-
95
- def create_mask(
96
- self, bsz: int, seqlen: int, mask: torch.Tensor | None, device: torch.device
97
- ) -> torch.Tensor | None:
98
- """Create attention mask combining provided mask and local attention constraints"""
99
- if not self.use_local_attention and mask is None:
100
- return None
101
-
102
- # Start with all positions allowed
103
- attn_mask = torch.ones((seqlen, seqlen), dtype=torch.bool, device=device)
104
-
105
- if self.causal:
106
- # Causal mask: no future positions allowed
107
- attn_mask = torch.tril(attn_mask)
108
-
109
- # Apply local attention constraints
110
- if self.use_local_attention:
111
- attn_mask = torch.triu(attn_mask, diagonal=-self.window_per_side)
112
- attn_mask = torch.tril(attn_mask, diagonal=self.window_per_side)
113
-
114
- # Expand mask to batch size
115
- attn_mask = attn_mask.unsqueeze(0).expand(bsz, -1, -1)
116
-
117
- # Apply global mask if provided
118
- if mask is not None:
119
- assert mask.shape[-1] == seqlen and mask.shape[-2] == seqlen, (
120
- "Mask must be square and match sequence length."
121
- )
122
- # Ensure mask has correct batch dimensions
123
- if mask.dim() == 2:
124
- mask = mask.unsqueeze(0).expand(bsz, -1, -1)
125
- attn_mask = attn_mask & mask
126
-
127
- # Expand to head dimension
128
- attn_mask = attn_mask.unsqueeze(1).expand(-1, self.n_heads, -1, -1)
129
- return attn_mask
130
-
131
- def forward(
132
- self,
133
- x: torch.Tensor,
134
- freqs_cis: torch.Tensor | None,
135
- mask: torch.Tensor | None,
136
- return_kv: bool = False,
137
- key_padding_mask: torch.Tensor | None = None,
138
- cu_seqlens: torch.Tensor | None = None,
139
- max_seqlen: int | None = None,
140
- ) -> torch.Tensor | tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
141
- """Forward pass for multi-head attention.
142
- Args:
143
- x (torch.Tensor): Input tensor of shape (bsz, seqlen, dim).
144
- freqs_cis (torch.Tensor, optional): Precomputed rotary frequencies.
145
- mask (torch.Tensor, optional): Attention mask.
146
- return_kv (bool): Whether to return KV pairs for caching.
147
- key_padding_mask (torch.Tensor, optional): Padding mask of shape (bsz, seqlen).
148
- True indicates valid positions, False indicates padding positions.
149
- cu_seqlens (torch.Tensor, optional): Cumulative sequence lengths for flash_attn_varlen_func.
150
- Shape (bsz + 1,), used for variable-length flash attention.
151
- max_seqlen (int, optional): Maximum sequence length in the batch. Required when cu_seqlens is provided.
152
- Returns:
153
- output (torch.Tensor): Output tensor of shape (bsz, seqlen, dim).
154
- new_kv (tuple, optional): KV pairs if return_kv is True.
155
- """
156
- bsz, seqlen, _ = x.shape
157
- xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)
158
-
159
- xq = xq.view(bsz, seqlen, self.n_heads, self.head_dim)
160
- xk = xk.view(bsz, seqlen, self.n_heads, self.head_dim)
161
- xv = xv.view(bsz, seqlen, self.n_heads, self.head_dim)
162
-
163
- # Apply rotary embeddings if provided
164
- if freqs_cis is not None:
165
- xq = apply_rotary_emb(xq, freqs_cis=freqs_cis[:seqlen])
166
- xk = apply_rotary_emb(xk, freqs_cis=freqs_cis[:seqlen])
167
-
168
- use_flash = self.use_flash_attention and FLASH_ATTN_AVAILABLE and x.is_cuda
169
- if self.use_flash_attention and FLASH_ATTN_AVAILABLE and not x.is_cuda:
170
- _warn_flash_cpu_once()
171
-
172
- if use_flash:
173
- assert mask is None, "Flash attention does not support arbitrary masking."
174
-
175
- window_size = (self.window_per_side, self.window_per_side) if self.use_local_attention else (-1, -1)
176
-
177
- # Use flash_attn_varlen_func for variable-length sequences with padding
178
- if cu_seqlens is not None and max_seqlen is not None:
179
- # Reshape to (total_tokens, n_heads, head_dim) for varlen function
180
- # We need to pack the valid tokens only
181
- if key_padding_mask is not None:
182
- # Pack valid tokens
183
- valid_mask = key_padding_mask.view(-1) # (bsz * seqlen,)
184
- xq_packed = xq.view(-1, self.n_heads, self.head_dim)[valid_mask]
185
- xk_packed = xk.view(-1, self.n_heads, self.head_dim)[valid_mask]
186
- xv_packed = xv.view(-1, self.n_heads, self.head_dim)[valid_mask]
187
- else:
188
- xq_packed = xq.view(-1, self.n_heads, self.head_dim)
189
- xk_packed = xk.view(-1, self.n_heads, self.head_dim)
190
- xv_packed = xv.view(-1, self.n_heads, self.head_dim)
191
-
192
- output_packed = flash_attn_varlen_func(
193
- xq_packed,
194
- xk_packed,
195
- xv_packed,
196
- cu_seqlens_q=cu_seqlens,
197
- cu_seqlens_k=cu_seqlens,
198
- max_seqlen_q=max_seqlen,
199
- max_seqlen_k=max_seqlen,
200
- dropout_p=(self.dropout if self.training else 0.0),
201
- softmax_scale=self.scale,
202
- causal=self.causal,
203
- window_size=window_size,
204
- )
205
-
206
- # Unpack back to (bsz, seqlen, n_heads, head_dim)
207
- if key_padding_mask is not None:
208
- output = torch.zeros(bsz * seqlen, self.n_heads, self.head_dim,
209
- device=x.device, dtype=output_packed.dtype)
210
- output[valid_mask] = output_packed
211
- output = output.view(bsz, seqlen, self.n_heads, self.head_dim)
212
- else:
213
- output = output_packed.view(bsz, seqlen, self.n_heads, self.head_dim)
214
- else:
215
- # Standard flash attention without variable-length handling
216
- output = flash_attn_func(
217
- xq, # (bsz, seqlen, n_heads, head_dim)
218
- xk, # (bsz, seqlen, n_heads, head_dim)
219
- xv, # (bsz, seqlen, n_heads, head_dim)
220
- dropout_p=(self.dropout if self.training else 0.0),
221
- softmax_scale=self.scale,
222
- window_size=window_size,
223
- causal=self.causal,
224
- ) # (bsz, seqlen, n_heads, head_dim)
225
-
226
- else:
227
- attn_mask = self.create_mask(bsz, seqlen, mask, x.device)
228
-
229
- # Apply key_padding_mask to attention mask
230
- if key_padding_mask is not None:
231
- # key_padding_mask: (bsz, seqlen), True = valid, False = padding
232
- # Create mask where padding positions cannot be attended to
233
- padding_mask = key_padding_mask.unsqueeze(1).unsqueeze(2) # (bsz, 1, 1, seqlen)
234
- if attn_mask is None:
235
- attn_mask = padding_mask.expand(-1, self.n_heads, seqlen, -1)
236
- else:
237
- attn_mask = attn_mask & padding_mask
238
-
239
- # SDPA Attention
240
- output = F.scaled_dot_product_attention(
241
- xq.transpose(1, 2), # (bsz, n_heads, seqlen, head_dim)
242
- xk.transpose(1, 2), # (bsz, n_heads, seqlen, head_dim)
243
- xv.transpose(1, 2), # (bsz, n_heads, seqlen, head_dim)
244
- attn_mask=attn_mask, # (bsz, n_heads, seqlen, seqlen) boolean mask
245
- dropout_p=(self.dropout if self.training else 0.0),
246
- scale=self.scale,
247
- ).transpose(1, 2) # (bsz, seqlen, n_heads, head_dim)
248
-
249
- output = output.contiguous().view(bsz, seqlen, -1)
250
- output = self.wo(output)
251
-
252
- if return_kv:
253
- return output, (xk, xv)
254
- return output
255
-
256
- def forward_with_cache(
257
- self,
258
- x: torch.Tensor,
259
- kv_cache: tuple[torch.Tensor, torch.Tensor],
260
- freqs_cis: torch.Tensor,
261
- start_pos: int,
262
- ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
263
- """
264
- Forward pass with KV cache for efficient inference. Only used for inference.
265
-
266
- Args:
267
- x (torch.Tensor): Input tensor for the current step. Shape: (bsz, 1, dim)
268
- kv_cache: A tuple of (key_cache, value_cache) from previous steps.
269
- start_pos (int): The starting position of the new token in the sequence.
270
- freqs_cis (torch.Tensor): Precomputed rotary frequencies.
271
-
272
- Returns:
273
- output (torch.Tensor): Output tensor after attention. Shape: (bsz, 1, dim)
274
- new_kv (tuple): Updated KV cache including the new key and value.
275
- """
276
- bsz, seqlen, _ = x.shape
277
- assert seqlen == 1, "KV cache method is designed for single-token generation."
278
-
279
- xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)
280
-
281
- xq = xq.view(bsz, seqlen, self.n_heads, self.head_dim)
282
- xk = xk.view(bsz, seqlen, self.n_heads, self.head_dim)
283
- xv = xv.view(bsz, seqlen, self.n_heads, self.head_dim)
284
-
285
- # Apply rotary embeddings using the correct positional slice
286
- xq = apply_rotary_emb(xq, freqs_cis=freqs_cis[start_pos : start_pos + seqlen])
287
- xk = apply_rotary_emb(xk, freqs_cis=freqs_cis[start_pos : start_pos + seqlen])
288
-
289
- # Update the KV cache
290
- k_cache, v_cache = kv_cache
291
- new_kv = (xk, xv)
292
- xk = torch.cat([k_cache, xk], dim=1)
293
- xv = torch.cat([v_cache, xv], dim=1)
294
-
295
- # For single token generation, causal mask is implicitly handled.
296
- # We attend to all keys (prefix + previous tokens).
297
- use_flash = self.use_flash_attention and FLASH_ATTN_AVAILABLE and x.is_cuda
298
- if self.use_flash_attention and FLASH_ATTN_AVAILABLE and not x.is_cuda:
299
- _warn_flash_cpu_once()
300
- if use_flash:
301
- # Flash Attention
302
- output = flash_attn_with_kvcache(
303
- xq, # (bsz, 1, n_heads, head_dim)
304
- xk, # (bsz, 1+kv_len, n_heads, head_dim)
305
- xv, # (bsz, 1+kv_len, n_heads, head_dim)
306
- softmax_scale=self.scale,
307
- ) # (bsz, 1, n_heads, head_dim)
308
- else:
309
- # SDPA Attention
310
- output = F.scaled_dot_product_attention(
311
- xq.transpose(1, 2), # (bsz, n_heads, 1, head_dim)
312
- xk.transpose(1, 2), # (bsz, n_heads, 1+kv_len, head_dim)
313
- xv.transpose(1, 2), # (bsz, n_heads, 1+kv_len, head_dim)
314
- scale=self.scale,
315
- ).transpose(1, 2) # (bsz, 1, n_heads, head_dim)
316
-
317
- output = output.contiguous().view(bsz, seqlen, -1)
318
- return self.wo(output), new_kv
319
-
320
-
321
- class FeedForward(nn.Module):
322
- def __init__(
323
- self,
324
- dim: int,
325
- hidden_dim: int,
326
- multiple_of: int,
327
- ffn_dim_multiplier: float | None,
328
- ):
329
- super().__init__()
330
- hidden_dim = int(2 * hidden_dim / 3)
331
- # custom dim factor multiplier
332
- if ffn_dim_multiplier is not None:
333
- hidden_dim = int(ffn_dim_multiplier * hidden_dim)
334
- hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
335
-
336
- self.w1 = nn.Linear(dim, hidden_dim, bias=False)
337
- self.w2 = nn.Linear(hidden_dim, dim, bias=False)
338
- self.w3 = nn.Linear(dim, hidden_dim, bias=False)
339
-
340
- def forward(self, x):
341
- return self.w2(F.silu(self.w1(x)) * self.w3(x))
342
-
343
-
344
- class TransformerBlock(nn.Module):
345
- def __init__(
346
- self,
347
- dim: int,
348
- n_heads: int,
349
- qkv_bias: bool,
350
- proj_bias: bool,
351
- window_size: int | None,
352
- multiple_of: int,
353
- ffn_dim_multiplier: float | None,
354
- dropout: float,
355
- norm_eps: float,
356
- adanorm_condition_dim: int | None = None,
357
- use_flash_attention: bool = False,
358
- use_adaln_zero: bool = False,
359
- causal: bool = False,
360
- ):
361
- super().__init__()
362
- self.attention = Attention(
363
- dim=dim,
364
- n_heads=n_heads,
365
- dropout=dropout,
366
- window_size=window_size,
367
- use_flash_attention=use_flash_attention,
368
- qkv_bias=qkv_bias,
369
- proj_bias=proj_bias,
370
- causal=causal,
371
- )
372
-
373
- self.feed_forward = FeedForward(
374
- dim=dim,
375
- hidden_dim=4 * dim,
376
- multiple_of=multiple_of,
377
- ffn_dim_multiplier=ffn_dim_multiplier,
378
- )
379
-
380
- # Choose between AdaLNZero and regular LayerNorm
381
- self.use_adaln_zero = use_adaln_zero
382
- if self.use_adaln_zero:
383
- assert adanorm_condition_dim is not None, "condition_dim must be provided when using AdaLNZero"
384
- self.attention_norm = AdaLNZero(dim, adanorm_condition_dim, eps=norm_eps, return_gate=True)
385
- self.ffn_norm = AdaLNZero(dim, adanorm_condition_dim, eps=norm_eps, return_gate=True)
386
- else:
387
- self.attention_norm = nn.LayerNorm(dim, eps=norm_eps)
388
- self.ffn_norm = nn.LayerNorm(dim, eps=norm_eps)
389
-
390
- def forward(
391
- self,
392
- x: torch.Tensor,
393
- freqs_cis: torch.Tensor | None,
394
- mask: torch.Tensor | None,
395
- condition: torch.Tensor | None = None,
396
- return_kv: bool = False,
397
- kv_cache: tuple[torch.Tensor, torch.Tensor] | None = None,
398
- start_pos: int | None = None,
399
- key_padding_mask: torch.Tensor | None = None,
400
- cu_seqlens: torch.Tensor | None = None,
401
- max_seqlen: int | None = None,
402
- ) -> torch.Tensor | tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
403
- """
404
- Forward pass for a single Transformer block.
405
- Args:
406
- x (torch.Tensor): Input tensor of shape (bsz, seqlen, dim).
407
- freqs_cis (torch.Tensor, optional): Precomputed rotary frequencies.
408
- mask (torch.Tensor, optional): Attention mask.
409
- condition (torch.Tensor, optional): Conditioning tensor for AdaLNZero.
410
- return_kv (bool): Whether to return KV pairs for caching.
411
- kv_cache (tuple, optional): KV cache for efficient inference.
412
- start_pos (int, optional): Starting position for KV cache.
413
- key_padding_mask (torch.Tensor, optional): Padding mask of shape (bsz, seqlen).
414
- True indicates valid positions, False indicates padding positions.
415
- cu_seqlens (torch.Tensor, optional): Cumulative sequence lengths for flash_attn_varlen_func.
416
- max_seqlen (int, optional): Maximum sequence length in the batch.
417
- Returns:
418
- out (torch.Tensor): Output tensor of shape (bsz, seqlen, dim).
419
- new_kv (tuple, optional): New KV pairs if return_kv is True or kv_cache is provided.
420
- """
421
- # Apply normalization
422
- if self.use_adaln_zero:
423
- assert condition is not None, "condition must be provided when using AdaLNZero"
424
- attn_normed, attn_gate = self.attention_norm(x, condition=condition)
425
- else:
426
- attn_normed = self.attention_norm(x)
427
-
428
- # Forward attention with KV cache if provided
429
- new_kv = None
430
- if kv_cache is not None and start_pos is not None:
431
- # Use KV cache for efficient inference
432
- attn_out, new_kv = self.attention.forward_with_cache(attn_normed, kv_cache, freqs_cis, start_pos)
433
- elif return_kv:
434
- # Return KV pairs for caching
435
- attn_out, new_kv = self.attention(attn_normed, freqs_cis, mask, return_kv=True,
436
- key_padding_mask=key_padding_mask,
437
- cu_seqlens=cu_seqlens, max_seqlen=max_seqlen)
438
- else:
439
- attn_out = self.attention(attn_normed, freqs_cis, mask,
440
- key_padding_mask=key_padding_mask,
441
- cu_seqlens=cu_seqlens, max_seqlen=max_seqlen)
442
-
443
- # Apply gating for attention if using AdaLNZero
444
- if self.use_adaln_zero:
445
- h = x + attn_gate * attn_out # residual + gate * x
446
- else:
447
- h = x + attn_out
448
-
449
- # Apply normalization for feedforward
450
- if self.use_adaln_zero:
451
- ffn_normed, ffn_gate = self.ffn_norm(h, condition=condition)
452
- else:
453
- ffn_normed = self.ffn_norm(h)
454
-
455
- ffn_out = self.feed_forward(ffn_normed)
456
-
457
- # Apply gating for feedforward if using AdaLNZero
458
- if self.use_adaln_zero:
459
- out = h + ffn_gate * ffn_out # residual + gate * x
460
- else:
461
- out = h + ffn_out
462
-
463
- # If using KV cache, return the new KV pairs
464
- if new_kv is not None:
465
- return out, new_kv
466
- return out
467
-
468
-
469
- class Transformer(nn.Module):
470
- def __init__(
471
- self,
472
- dim: int = 4096,
473
- n_layers: int = 32,
474
- n_heads: int = 32,
475
- qkv_bias: bool = False,
476
- proj_bias: bool = False,
477
- window_size: int | None = None,
478
- multiple_of: int = 256,
479
- ffn_dim_multiplier: float | None = None,
480
- dropout: float = 0.1,
481
- norm_eps: float = 1e-5,
482
- use_rope: bool = True,
483
- rope_theta: float = 500000.0,
484
- max_seq_len: int = 2048,
485
- input_dim: int | None = None,
486
- output_dim: int | None = None,
487
- adanorm_condition_dim: int | None = None,
488
- use_flash_attention: bool = False,
489
- use_adaln_zero: bool = False,
490
- use_xavier_init: bool = True,
491
- causal: bool = False,
492
- ):
493
- super().__init__()
494
- self.dim = dim
495
- self.n_heads = n_heads
496
- self.rope_theta = rope_theta
497
- self.use_adaln_zero = use_adaln_zero
498
-
499
- self.layers = nn.ModuleList()
500
- for layer_id in range(n_layers):
501
- self.layers.append(
502
- TransformerBlock(
503
- dim=dim,
504
- n_heads=n_heads,
505
- window_size=window_size,
506
- multiple_of=multiple_of,
507
- ffn_dim_multiplier=ffn_dim_multiplier,
508
- dropout=dropout,
509
- qkv_bias=qkv_bias,
510
- proj_bias=proj_bias,
511
- norm_eps=norm_eps,
512
- adanorm_condition_dim=adanorm_condition_dim,
513
- use_flash_attention=use_flash_attention,
514
- use_adaln_zero=use_adaln_zero,
515
- causal=causal,
516
- )
517
- )
518
-
519
- # Choose between AdaLNZero (without gate) and regular LayerNorm for final norm
520
- if self.use_adaln_zero:
521
- assert adanorm_condition_dim is not None, "condition_dim must be provided when using AdaLNZero"
522
- self.norm = AdaLNZero(dim, adanorm_condition_dim, eps=norm_eps, return_gate=False)
523
- else:
524
- self.norm = nn.LayerNorm(dim, eps=norm_eps)
525
- self.input_proj = nn.Linear(input_dim, dim) if input_dim is not None else nn.Identity()
526
- self.output_proj = nn.Linear(dim, output_dim) if output_dim is not None else nn.Identity()
527
- self.output_dim_ = output_dim if output_dim is not None else dim
528
-
529
- if use_rope:
530
- self.freqs_cis = precompute_freqs_cis(dim // n_heads, max_seq_len * 2, rope_theta)
531
- logger.debug(
532
- f"Using RoPE with theta={rope_theta}, max_seq_len={max_seq_len}, "
533
- f"dim={dim}, n_heads={n_heads}, freqs_cis shape={self.freqs_cis.shape}"
534
- )
535
- else:
536
- self.freqs_cis = None
537
-
538
- if window_size is not None:
539
- logger.debug(f"Using local attention with window size {window_size}")
540
-
541
- if self.use_adaln_zero:
542
- logger.debug(f"Using AdaLNZero conditioning with condition_dim={adanorm_condition_dim}")
543
-
544
- if use_flash_attention:
545
- logger.debug("Using Flash Attention for memory-efficient attention computation")
546
-
547
- if use_xavier_init:
548
- logger.debug("Using Xavier initialization for linear layers")
549
- self.apply(self._init_weights)
550
- self.apply(self._init_adaln_zero)
551
-
552
- @property
553
- def output_dim(self) -> int:
554
- return self.output_dim_
555
-
556
- def _init_weights(self, module: nn.Module):
557
- if isinstance(module, nn.Linear):
558
- nn.init.xavier_normal_(module.weight)
559
- if module.bias is not None:
560
- nn.init.zeros_(module.bias)
561
-
562
- def _init_adaln_zero(self, module: nn.Module):
563
- if isinstance(module, AdaLNZero):
564
- # Initialize condition projection weights to zero
565
- nn.init.zeros_(module.condition_proj[1].weight)
566
- nn.init.zeros_(module.condition_proj[1].bias)
567
-
568
- def forward(
569
- self,
570
- x: torch.Tensor,
571
- mask: torch.Tensor | None = None,
572
- condition: torch.Tensor | None = None,
573
- return_kv: bool = False,
574
- kv_cache: list[tuple[torch.Tensor, torch.Tensor]] | None = None,
575
- start_pos: int | None = None,
576
- key_padding_mask: torch.Tensor | None = None,
577
- ) -> torch.Tensor | tuple[torch.Tensor, list[tuple[torch.Tensor, torch.Tensor]]]:
578
- """
579
- Forward pass for the Transformer model.
580
- Args:
581
- x (torch.Tensor): Input tensor of shape (bsz, seqlen, input_dim).
582
- mask (torch.Tensor, optional): Attention mask.
583
- condition (torch.Tensor, optional): Conditioning tensor for AdaLNZero.
584
- return_kv (bool): Whether to return KV pairs for caching.
585
- kv_cache (list, optional): List of KV caches for each layer for efficient inference.
586
- start_pos (int, optional): Starting position for KV cache.
587
- key_padding_mask (torch.Tensor, optional): Padding mask of shape (bsz, seqlen).
588
- True indicates valid positions, False indicates padding positions.
589
- Used for batch processing with variable-length sequences.
590
- Returns:
591
- output (torch.Tensor): Output tensor of shape (bsz, seqlen, output_dim).
592
- new_kv_list (list, optional): List of new KV pairs for each layer if return_kv is True or kv_cache is provided.
593
- """
594
- bsz, seqlen, _dim = x.shape
595
-
596
- if self.use_adaln_zero:
597
- assert condition is not None, "condition must be provided when using AdaLNZero"
598
-
599
- # Rotary embeddings
600
- if self.freqs_cis is not None:
601
- # Recompute freqs_cis if the sequence length or starting position exceeds the precomputed length
602
- expected_len = (start_pos + 1) if start_pos is not None else seqlen
603
- if expected_len > self.freqs_cis.shape[0]:
604
- logger.warning(
605
- f"Input sequence length {expected_len} exceeds precomputed RoPE length {self.freqs_cis.shape[0]}. Recomputing freqs_cis."
606
- )
607
- self.freqs_cis = precompute_freqs_cis(self.dim // self.n_heads, expected_len * 4, self.rope_theta)
608
-
609
- self.freqs_cis = self.freqs_cis.to(x.device)
610
- freqs_cis = self.freqs_cis
611
- else:
612
- freqs_cis = None
613
-
614
- # Prepare cu_seqlens and max_seqlen for flash_attn_varlen_func if key_padding_mask is provided
615
- cu_seqlens = None
616
- max_seqlen = None
617
- if key_padding_mask is not None and FLASH_ATTN_AVAILABLE and x.is_cuda:
618
- # Calculate actual sequence lengths from padding mask
619
- seq_lens = key_padding_mask.sum(dim=1).to(torch.int32) # (bsz,)
620
- cu_seqlens = torch.zeros(bsz + 1, dtype=torch.int32, device=x.device)
621
- cu_seqlens[1:] = torch.cumsum(seq_lens, dim=0)
622
- max_seqlen = seq_lens.max().item()
623
-
624
- x = self.input_proj(x)
625
- new_kv_list = []
626
- for i, layer in enumerate(self.layers):
627
- # Collect KV cache if provided
628
- if kv_cache is not None and start_pos is not None:
629
- x, new_kv = layer(x, freqs_cis, mask, condition, kv_cache=kv_cache[i], start_pos=start_pos)
630
- new_kv_list.append(new_kv)
631
- elif return_kv:
632
- x, new_kv = layer(x, freqs_cis, mask, condition, return_kv=True,
633
- key_padding_mask=key_padding_mask,
634
- cu_seqlens=cu_seqlens, max_seqlen=max_seqlen)
635
- new_kv_list.append(new_kv)
636
- else:
637
- x = layer(x, freqs_cis, mask, condition,
638
- key_padding_mask=key_padding_mask,
639
- cu_seqlens=cu_seqlens, max_seqlen=max_seqlen)
640
-
641
- # Apply final normalization
642
- if self.use_adaln_zero:
643
- x, _ = self.norm(x, condition=condition) # Final norm doesn't use gate
644
- else:
645
- x = self.norm(x)
646
-
647
- output = self.output_proj(x)
648
-
649
- # If using KV cache, return the new KV pairs
650
- if new_kv_list:
651
- return output, new_kv_list
652
- return output