Spaces:
Running on Zero
Running on Zero
Download direct3d/models/sparse_atten.py from hugging-apps/direct3d-v3: direct link, hf CLI and curl.
- Browser
- Download file 9.66 kB
-
https://huggingface.co/spaces/hugging-apps/direct3d-v3/resolve/main/direct3d/models/sparse_atten.py
- Command line
-
hf download hf://spaces/hugging-apps/direct3d-v3/direct3d/models/sparse_atten.py
-
curl -L -o sparse_atten.py https://huggingface.co/spaces/hugging-apps/direct3d-v3/resolve/main/direct3d/models/sparse_atten.py
9.66 kB
| import math | |
| import flash_attn | |
| import torch | |
| from torch import nn | |
| def sparse_scaled_dot_product_attention(q, kv, layout, layout_kv=None): | |
| device = q.device | |
| q_seqlen = [layout[i].stop - layout[i].start for i in range(len(layout))] | |
| cu_seqlens_q = ( | |
| torch.cat([torch.tensor([0]), torch.cumsum(torch.tensor(q_seqlen), dim=0)]).int().to(device) | |
| ) | |
| if layout_kv is None: | |
| kv_seqlen = [kv.shape[1]] * kv.shape[0] | |
| kv = kv.flatten(0, 1) | |
| else: | |
| kv_seqlen = [part.stop - part.start for part in layout_kv] | |
| cu_seqlens_kv = ( | |
| torch.cat([torch.tensor([0]), torch.cumsum(torch.tensor(kv_seqlen), dim=0)]).int().to(device) | |
| ) | |
| out = flash_attn.flash_attn_varlen_kvpacked_func( | |
| q, kv, cu_seqlens_q, cu_seqlens_kv, max(q_seqlen), max(kv_seqlen) | |
| ) | |
| return out.reshape(q.shape[0], -1) | |
| def calc_window_partition( | |
| coords: torch.Tensor, window_size: int | tuple[int, ...], shift_window: int | tuple[int, ...] = 0 | |
| ) -> tuple[torch.Tensor, torch.Tensor, list[int]]: | |
| """Sort sparse tokens into shifted spatial windows and record the inverse permutation.""" | |
| DIM = coords.shape[1] - 1 | |
| shift_window = (shift_window,) * DIM if isinstance(shift_window, int) else shift_window | |
| window_size = (window_size,) * DIM if isinstance(window_size, int) else window_size | |
| shifted_coords = coords.clone().detach() | |
| shifted_coords[:, 1:] += torch.tensor(shift_window, device=coords.device, dtype=torch.int32).unsqueeze(0) | |
| MAX_COORDS = shifted_coords[:, 1:].max(dim=0).values.tolist() | |
| NUM_WINDOWS = [math.ceil((mc + 1) / ws) for mc, ws in zip(MAX_COORDS, window_size)] | |
| OFFSET = torch.cumprod(torch.tensor([1] + NUM_WINDOWS[::-1]), dim=0).tolist()[::-1] | |
| shifted_coords[:, 1:] //= torch.tensor(window_size, device=coords.device, dtype=torch.int32).unsqueeze(0) | |
| shifted_indices = ( | |
| shifted_coords * torch.tensor(OFFSET, device=coords.device, dtype=torch.int32).unsqueeze(0) | |
| ).sum(dim=1) | |
| fwd_indices = torch.argsort(shifted_indices) | |
| bwd_indices = torch.empty_like(fwd_indices) | |
| bwd_indices[fwd_indices] = torch.arange(fwd_indices.shape[0], device=coords.device) | |
| seq_lens = torch.bincount(shifted_indices) | |
| mask = seq_lens != 0 | |
| seq_lens = seq_lens[mask].tolist() | |
| return fwd_indices, bwd_indices, seq_lens | |
| def sparse_windowed_scaled_dot_product_self_attention(qkv, window_size, partition): | |
| fwd_indices, bwd_indices, seq_lens = partition | |
| H = qkv.shape[2] | |
| C = qkv.shape[3] | |
| qkv_feats = qkv[fwd_indices] # [M, 3, H, C] | |
| if all(seq_len == window_size for seq_len in seq_lens): | |
| B = len(seq_lens) | |
| N = window_size | |
| qkv_feats = qkv_feats.reshape(B, N, 3, H, C) | |
| out = flash_attn.flash_attn_qkvpacked_func(qkv_feats) # [B, N, H, C] | |
| out = out.reshape(B * N, H, C) # [M, H, C] | |
| else: | |
| cu_seqlens = ( | |
| torch.cat([torch.tensor([0]), torch.cumsum(torch.tensor(seq_lens), dim=0)], dim=0) | |
| .to(qkv.device) | |
| .int() | |
| ) | |
| out = flash_attn.flash_attn_varlen_qkvpacked_func(qkv_feats, cu_seqlens, max(seq_lens)) # [M, H, C] | |
| out = out[bwd_indices] # [T, H, C] | |
| return out | |
| class SparseRotaryPositionEmbedder(nn.Module): | |
| def __init__(self, head_dim: int, dim: int = 3, rope_freq: tuple[float, float] = (1.0, 10000.0)): | |
| super().__init__() | |
| assert head_dim % 2 == 0, "Head dim must be divisible by 2" | |
| self.head_dim = head_dim | |
| self.dim = dim | |
| self.rope_freq = rope_freq | |
| self.freq_dim = head_dim // 2 // dim | |
| self.freqs = torch.arange(self.freq_dim, dtype=torch.float32, device="cpu") / self.freq_dim | |
| self.freqs = rope_freq[0] / (rope_freq[1] ** (self.freqs)) | |
| def _get_phases(self, indices: torch.Tensor) -> torch.Tensor: | |
| self.freqs = self.freqs.to(indices.device) | |
| freqs = self.freqs | |
| phases = torch.outer(indices, freqs) | |
| phases = torch.polar(torch.ones_like(phases), phases) | |
| return phases | |
| def _rotary_embedding(self, x: torch.Tensor, phases: torch.Tensor) -> torch.Tensor: | |
| x_complex = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) | |
| x_rotated = x_complex * phases.unsqueeze(-2) | |
| x_embed = torch.view_as_real(x_rotated).reshape(*x_rotated.shape[:-1], -1).to(x.dtype) | |
| return x_embed | |
| def forward( | |
| self, q: torch.Tensor, k: torch.Tensor, indices: torch.Tensor | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """ | |
| Args: | |
| q (SparseTensor): [..., N, H, D] tensor of queries | |
| k (SparseTensor): [..., N, H, D] tensor of keys | |
| """ | |
| phases = self._get_phases(indices.reshape(-1)).reshape(*indices.shape[:-1], -1) | |
| if phases.shape[-1] < self.head_dim // 2: | |
| padn = self.head_dim // 2 - phases.shape[-1] | |
| phases = torch.cat( | |
| [ | |
| phases, | |
| torch.polar( | |
| torch.ones(*phases.shape[:-1], padn, device=phases.device), | |
| torch.zeros(*phases.shape[:-1], padn, device=phases.device), | |
| ), | |
| ], | |
| dim=-1, | |
| ) | |
| q_embed = self._rotary_embedding(q, phases) | |
| k_embed = self._rotary_embedding(k, phases) | |
| return q_embed, k_embed | |
| class CrossAttention(nn.Module): | |
| def __init__( | |
| self, | |
| qdim, | |
| kdim, | |
| num_heads, | |
| qkv_bias=True, | |
| qk_norm=False, | |
| norm_layer=nn.LayerNorm, | |
| ): | |
| super().__init__() | |
| self.qdim = qdim | |
| self.kdim = kdim | |
| self.num_heads = num_heads | |
| assert self.qdim % num_heads == 0, "self.qdim must be divisible by num_heads" | |
| self.head_dim = self.qdim // num_heads | |
| assert self.head_dim % 8 == 0 and self.head_dim <= 128, ( | |
| "Only support head_dim <= 128 and divisible by 8" | |
| ) | |
| self.to_q = nn.Linear(qdim, qdim, bias=qkv_bias) | |
| self.to_k = nn.Linear(kdim, qdim, bias=qkv_bias) | |
| self.to_v = nn.Linear(kdim, qdim, bias=qkv_bias) | |
| self.q_norm = ( | |
| norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() | |
| ) | |
| self.k_norm = ( | |
| norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() | |
| ) | |
| self.out_proj = nn.Linear(qdim, qdim, bias=True) | |
| def forward(self, x, y, layout, layout_y=None): | |
| q = self.to_q(x) | |
| q = q.reshape(q.shape[0], self.num_heads, self.head_dim) # [b, s1, h, d] | |
| k = self.to_k(y).unflatten(-1, (self.num_heads, self.head_dim)) | |
| v = self.to_v(y).unflatten(-1, (self.num_heads, self.head_dim)) | |
| q = self.q_norm(q) | |
| k = self.k_norm(k) | |
| kv = torch.stack([k, v], dim=-3) | |
| context = sparse_scaled_dot_product_attention(q, kv, layout, layout_y) | |
| context = context.reshape(x.shape[0], -1) | |
| out = self.out_proj(context) | |
| return out | |
| class Attention(nn.Module): | |
| """Self-attention over regular and shifted windows, using half the heads for each.""" | |
| def __init__( | |
| self, | |
| dim, | |
| num_heads, | |
| qkv_bias=True, | |
| qk_norm=False, | |
| norm_layer=nn.LayerNorm, | |
| ): | |
| super().__init__() | |
| self.dim = dim | |
| self.num_heads = num_heads | |
| assert self.dim % num_heads == 0, "dim should be divisible by num_heads" | |
| self.head_dim = self.dim // num_heads | |
| # This assertion is aligned with flash attention | |
| assert self.head_dim % 8 == 0 and self.head_dim <= 128, ( | |
| "Only support head_dim <= 128 and divisible by 8" | |
| ) | |
| self.to_q = nn.Linear(dim, dim, bias=qkv_bias) | |
| self.to_k = nn.Linear(dim, dim, bias=qkv_bias) | |
| self.to_v = nn.Linear(dim, dim, bias=qkv_bias) | |
| self.q_norm = ( | |
| norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() | |
| ) | |
| self.k_norm = ( | |
| norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() | |
| ) | |
| self.out_proj = nn.Linear(dim, dim) | |
| self.rope = SparseRotaryPositionEmbedder(head_dim=self.head_dim, dim=3) | |
| assert num_heads % 2 == 0, "Split-window attention requires an even number of heads" | |
| def forward(self, x, index, window_size, window_cache): | |
| N = len(x) | |
| q = self.to_q(x) | |
| k = self.to_k(x) | |
| v = self.to_v(x) | |
| qkv = torch.cat((q, k, v), dim=-1) | |
| qkv = qkv.reshape(N, 3, self.num_heads, -1) | |
| q, k, v = qkv.unbind(dim=1) | |
| q, k = self.rope(q, k, index[..., 1:]) | |
| q = self.q_norm(q) | |
| k = self.k_norm(k) | |
| qkv = torch.stack([q, k, v], dim=1) | |
| half_heads = self.num_heads // 2 | |
| cache0, cache1 = window_cache | |
| x0 = sparse_windowed_scaled_dot_product_self_attention( | |
| qkv[:, :, :half_heads].contiguous(), | |
| window_size, | |
| cache0, | |
| ) | |
| x1 = sparse_windowed_scaled_dot_product_self_attention( | |
| qkv[:, :, half_heads:].contiguous(), | |
| window_size, | |
| cache1, | |
| ) | |
| x = torch.cat([x0, x1], dim=1) | |
| x = x.reshape(x.shape[0], -1) | |
| x = self.out_proj(x) | |
| return x | |
| class MLP(nn.Module): | |
| def __init__(self, *, width: int): | |
| super().__init__() | |
| self.width = width | |
| self.fc1 = nn.Linear(width, width * 4) | |
| self.fc2 = nn.Linear(width * 4, width) | |
| self.gelu = nn.GELU() | |
| def forward(self, x): | |
| return self.fc2(self.gelu(self.fc1(x))) | |