Download upstreams/candi/models/unit_test_attention.py from jyliuAI/diffusion_baseline: direct link, hf CLI and curl.
- Browser
- Download file 3.74 kB
-
https://huggingface.co/jyliuAI/diffusion_baseline/resolve/main/upstreams/candi/models/unit_test_attention.py
- Command line
-
hf download hf://jyliuAI/diffusion_baseline/upstreams/candi/models/unit_test_attention.py
-
curl -L -o unit_test_attention.py https://huggingface.co/jyliuAI/diffusion_baseline/resolve/main/upstreams/candi/models/unit_test_attention.py
3.74 kB
| import unittest | |
| import torch | |
| # from flash_attn import flash_attention | |
| import torch.nn.functional as F | |
| def attention_inner_heads_flash(qkv, num_heads): | |
| """Computes attention with heads inside of qkv in the channel dimension using FlashAttention. | |
| Args: | |
| qkv: Tensor of shape (B, 3*H*C, T) with Qs, Ks, and Vs, where: | |
| H = number of heads, | |
| C = number of channels per head. | |
| num_heads: number of heads. | |
| Returns: | |
| Attention output of shape (B, H*C, T). | |
| """ | |
| # bs, width, length = qkv.shape | |
| # ch = width // (3 * num_heads) | |
| # # Split into (q, k, v) of shape (B, H*C, T). | |
| # q, k, v = qkv.chunk(3, dim=1) | |
| # # Rescale q and k. This makes them contiguous in memory. | |
| # scale = ch ** (-1 / 4) # scale with 4th root = scaling output by sqrt | |
| # q = q * scale | |
| # k = k * scale | |
| # # Reshape q, k, v to (B, H, T, C) for FlashAttention | |
| # q = q.view(bs, num_heads, ch, length).permute(0, 1, 3, 2) # (B, H, T, C) | |
| # k = k.view(bs, num_heads, ch, length).permute(0, 1, 3, 2) # (B, H, T, C) | |
| # v = v.view(bs, num_heads, ch, length).permute(0, 1, 3, 2) # (B, H, T, C) | |
| # # Compute attention using FlashAttention | |
| # out = flash_attention(q, k, v) # (B, H, T, C) | |
| # # Reshape back to (B, H*C, T) | |
| # out = out.permute(0, 1, 3, 2).reshape(bs, num_heads * ch, length) | |
| # return out | |
| bs, width, length = qkv.shape | |
| ch = width // (3 * num_heads) | |
| # Split into (q, k, v) and reshape directly to (B, H, T, C) | |
| q, k, v = qkv.chunk(3, dim=1) | |
| q = q.view(bs, num_heads, ch, length).transpose(2, 3) # (B, H, T, C) | |
| k = k.view(bs, num_heads, ch, length).transpose(2, 3) # (B, H, T, C) | |
| v = v.view(bs, num_heads, ch, length).transpose(2, 3) # (B, H, T, C) | |
| # Compute scaled dot-product attention | |
| out = F.scaled_dot_product_attention(q, k, v) # (B, H, T, C) | |
| # Reshape back to (B, H*C, T) in one step | |
| out = out.transpose(2, 3).reshape(bs, num_heads * ch, length) | |
| return out | |
| class TestAttentionInnerHeadsFlash(unittest.TestCase): | |
| def setUp(self): | |
| # Set up common test variables | |
| self.batch_size = 2 | |
| self.num_heads = 4 | |
| self.seq_len = 8 | |
| self.channels_per_head = 16 | |
| self.qkv_dim = 3 * self.num_heads * self.channels_per_head | |
| # Create a random qkv tensor | |
| self.qkv = torch.randn( | |
| self.batch_size, self.qkv_dim, self.seq_len, dtype=torch.float32 | |
| ) | |
| def attention_inner_heads_old(self, qkv, num_heads): | |
| """Original implementation of attention for reference.""" | |
| bs, width, length = qkv.shape | |
| ch = width // (3 * num_heads) | |
| q, k, v = qkv.chunk(3, dim=1) | |
| scale = ch ** (-1 / 4) | |
| q = q * scale | |
| k = k * scale | |
| q = q.view(bs * num_heads, ch, length) | |
| k = k.view(bs * num_heads, ch, length) | |
| v = v.reshape(bs * num_heads, ch, length) | |
| weight = torch.einsum("bct,bcs->bts", q, k) | |
| weight = torch.softmax(weight.float(), dim=-1).to(weight.dtype) | |
| out = torch.einsum("bts,bcs->bct", weight, v) | |
| return out.reshape(bs, num_heads * ch, length) | |
| def test_attention_inner_heads_flash(self): | |
| # Compute the output using the old and flash attention implementations | |
| output_old = self.attention_inner_heads_old(self.qkv, self.num_heads) | |
| output_flash = attention_inner_heads_flash(self.qkv, self.num_heads) | |
| # Verify the shapes are the same | |
| self.assertEqual(output_old.shape, output_flash.shape) | |
| # Verify that the outputs are close (numerically similar) | |
| self.assertTrue(torch.allclose(output_old, output_flash, atol=1e-5)) | |
| if __name__ == "__main__": | |
| unittest.main() | |