liangsu9988 commited on
Commit
2d3818d
·
verified ·
1 Parent(s): ada1059

Promote latest kernel artifacts to main

Browse files
Files changed (23) hide show
  1. .gitattributes +3 -35
  2. README.md +0 -9
  3. benchmarks/benchmark.py +113 -0
  4. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +189 -0
  5. build/torch211-cxx11-cu128-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
  6. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
  7. build/torch211-cxx11-cu128-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
  8. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
  9. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +189 -0
  10. build/torch211-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
  11. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  12. build/torch211-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
  13. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
  14. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +189 -0
  15. build/torch212-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
  16. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  17. build/torch212-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
  18. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
  19. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +189 -0
  20. build/torch212-cxx11-cu132-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
  21. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
  22. build/torch212-cxx11-cu132-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
  23. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.so filter=lfs diff=lfs merge=lfs -text
2
+ *.pyd filter=lfs diff=lfs merge=lfs -text
3
+ *.dylib filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
README.md DELETED
@@ -1,9 +0,0 @@
1
- # flashrt/linear-attention-primitives
2
-
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
-
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/linear-attention-primitives
7
-
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
benchmarks/benchmark.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Benchmark linear-attention-primitives against PyTorch eager."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ import torch
11
+
12
+ PACKAGE = Path(__file__).resolve().parents[1]
13
+ sys.path.insert(0, str(PACKAGE / "tests"))
14
+ from test_linear_attention_primitives import load_installed_ops, load_source_ops # noqa: E402
15
+
16
+
17
+ def bench(fn, warmup: int, iters: int) -> float:
18
+ for _ in range(warmup):
19
+ fn()
20
+ torch.cuda.synchronize()
21
+ start = torch.cuda.Event(enable_timing=True)
22
+ end = torch.cuda.Event(enable_timing=True)
23
+ start.record()
24
+ for _ in range(iters):
25
+ fn()
26
+ end.record()
27
+ torch.cuda.synchronize()
28
+ return start.elapsed_time(end) * 1000.0 / iters
29
+
30
+
31
+ def main() -> int:
32
+ parser = argparse.ArgumentParser()
33
+ parser.add_argument("--backend", choices=["source", "installed"], default="source")
34
+ parser.add_argument("--artifact", default=None)
35
+ parser.add_argument("--warmup", type=int, default=100)
36
+ parser.add_argument("--iters", type=int, default=1000)
37
+ args = parser.parse_args()
38
+ if not torch.cuda.is_available():
39
+ raise RuntimeError("CUDA is required")
40
+ torch.manual_seed(123)
41
+ ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
42
+
43
+ print("| Workload | Shape | FlashRT us | PyTorch eager us | Speedup |")
44
+ print("|---|---:|---:|---:|---:|")
45
+
46
+ for k, n in [(128, 512), (4096, 1024), (5120, 10240)]:
47
+ x = (torch.randn((k,), device="cuda") * 0.05).to(torch.bfloat16)
48
+ w = (torch.randn((n, k), device="cuda") * 0.05).to(torch.bfloat16)
49
+ fused = bench(lambda: ops.bf16_matvec(x, w), args.warmup, args.iters)
50
+ eager = bench(lambda: x @ w.t(), args.warmup, args.iters)
51
+ print(f"| bf16_matvec | N={n},K={k} | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
52
+
53
+ for m, k, n in [(2, 5120, 96), (3, 5120, 96), (4, 5120, 96)]:
54
+ x = (torch.randn((m, k), device="cuda") * 0.05).to(torch.bfloat16)
55
+ w = (torch.randn((n, k), device="cuda") * 0.05).to(torch.bfloat16)
56
+ fused = bench(lambda: ops.bf16_smallm_matmul(x, w), args.warmup, args.iters)
57
+ eager = bench(lambda: x @ w.t(), args.warmup, args.iters)
58
+ print(f"| bf16_smallm_matmul | M={m},N={n},K={k} | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
59
+
60
+ for rows in [1, 128, 1024]:
61
+ packed = torch.randn((rows, (16 + 16 + 48) * 128), device="cuda", dtype=torch.bfloat16)
62
+ fused = bench(lambda: ops.split_qkv_broadcast_bf16(packed, 16, 16, 48, 128), args.warmup, args.iters)
63
+ eager = bench(
64
+ lambda: (
65
+ packed[:, : 16 * 128].reshape(rows, 16, 128)[:, torch.arange(48, device="cuda") * 16 // 48].contiguous(),
66
+ packed[:, 16 * 128 : 32 * 128].reshape(rows, 16, 128)[:, torch.arange(48, device="cuda") * 16 // 48].contiguous(),
67
+ packed[:, 32 * 128 :].reshape(rows, 48, 128).contiguous(),
68
+ ),
69
+ args.warmup,
70
+ args.iters,
71
+ )
72
+ print(f"| split_qkv_broadcast | rows={rows},heads=16/48,dim=128 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
73
+
74
+ for rows in [1, 128, 1024]:
75
+ q = torch.randn((rows, 16, 128), device="cuda", dtype=torch.bfloat16)
76
+ k = torch.randn((rows, 16, 128), device="cuda", dtype=torch.bfloat16)
77
+ cos = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
78
+ sin = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
79
+
80
+ def eager_rope():
81
+ half = 32
82
+ qo = q.clone()
83
+ ko = k.clone()
84
+ qo[:, :, :half] = ((-q[:, :, half:64].float() * sin[:, None, :half].float()).to(torch.bfloat16).float() + q[:, :, :half].float() * cos[:, None, :half].float()).to(torch.bfloat16)
85
+ qo[:, :, half:64] = ((q[:, :, :half].float() * sin[:, None, half:64].float()).to(torch.bfloat16).float() + q[:, :, half:64].float() * cos[:, None, half:64].float()).to(torch.bfloat16)
86
+ ko[:, :, :half] = ((-k[:, :, half:64].float() * sin[:, None, :half].float()).to(torch.bfloat16).float() + k[:, :, :half].float() * cos[:, None, :half].float()).to(torch.bfloat16)
87
+ ko[:, :, half:64] = ((k[:, :, :half].float() * sin[:, None, half:64].float()).to(torch.bfloat16).float() + k[:, :, half:64].float() * cos[:, None, half:64].float()).to(torch.bfloat16)
88
+ return qo, ko
89
+
90
+ fused = bench(lambda: ops.partial_rope_qk_bf16(q, k, cos, sin, 64), args.warmup, args.iters)
91
+ eager = bench(eager_rope, args.warmup, args.iters)
92
+ print(f"| partial_rope_qk | rows={rows},heads=16,dim=128,rope=64 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
93
+
94
+ for rows in [1, 128, 1024]:
95
+ a = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
96
+ b = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
97
+ neg = torch.randn((48,), device="cuda", dtype=torch.float32) * 0.1
98
+ bias = torch.randn((48,), device="cuda", dtype=torch.float32) * 0.1
99
+ fused = bench(lambda: ops.gated_delta_prepare_bf16(a, b, neg, bias, heads=48, a_stride=64, b_stride=64), args.warmup, args.iters)
100
+ eager = bench(
101
+ lambda: (
102
+ neg[None, :] * torch.nn.functional.softplus(a[:, :48].float() + bias[None, :])
103
+ ).to(torch.bfloat16),
104
+ args.warmup,
105
+ args.iters,
106
+ )
107
+ print(f"| gated_delta_prepare | rows={rows},heads=48 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
108
+
109
+ return 0
110
+
111
+
112
+ if __name__ == "__main__":
113
+ raise SystemExit(main())
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT linear-attention helper kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
13
+ def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
14
+ if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
15
+ raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
16
+ if w.shape[0] < 256:
17
+ raise RuntimeError("bf16_matvec supports N >= 256")
18
+ return None
19
+
20
+
21
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
22
+ def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
23
+ if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
24
+ raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
25
+ if w.shape != (96, 5120):
26
+ raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
27
+ if x.shape[0] < 2 or x.shape[0] > 4:
28
+ raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
29
+ return None
30
+
31
+
32
+ @torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
33
+ def _split_qkv_broadcast_fake(
34
+ packed: torch.Tensor,
35
+ q: torch.Tensor,
36
+ k: torch.Tensor,
37
+ v: torch.Tensor,
38
+ q_heads: int,
39
+ kv_heads: int,
40
+ v_heads: int,
41
+ head_dim: int,
42
+ ) -> None:
43
+ rows = packed.shape[0]
44
+ if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
45
+ raise RuntimeError("packed shape mismatch")
46
+ if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
47
+ raise RuntimeError("q/k/v output shape mismatch")
48
+ return None
49
+
50
+
51
+ @torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
52
+ def _partial_rope_qk_fake(
53
+ q_in: torch.Tensor,
54
+ k_in: torch.Tensor,
55
+ cos: torch.Tensor,
56
+ sin: torch.Tensor,
57
+ q_out: torch.Tensor,
58
+ k_out: torch.Tensor,
59
+ rope_dim: int,
60
+ ) -> None:
61
+ if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
62
+ raise RuntimeError("q_in/k_in shape mismatch")
63
+ rows, _, head_dim = q_in.shape
64
+ if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
65
+ raise RuntimeError("invalid rope_dim")
66
+ if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
67
+ raise RuntimeError("cos/sin shape mismatch")
68
+ if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
69
+ raise RuntimeError("q_out/k_out shape mismatch")
70
+ return None
71
+
72
+
73
+ @torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
74
+ def _gated_delta_prepare_fake(
75
+ a: torch.Tensor,
76
+ b: torch.Tensor,
77
+ neg_exp_a_log: torch.Tensor,
78
+ dt_bias: torch.Tensor,
79
+ g_out: torch.Tensor,
80
+ beta_out: torch.Tensor,
81
+ a_stride: int,
82
+ b_stride: int,
83
+ ) -> None:
84
+ if g_out.dim() != 2 or beta_out.shape != g_out.shape:
85
+ raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
86
+ heads = g_out.shape[1]
87
+ if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
88
+ raise RuntimeError("per-head parameter shape mismatch")
89
+ return None
90
+
91
+
92
+ def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
93
+ """Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
94
+ if out is None:
95
+ out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
96
+ ops.bf16_matvec(x, w, out)
97
+ return out
98
+
99
+
100
+ def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
101
+ """Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
102
+ if out is None:
103
+ out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
104
+ ops.bf16_smallm_matmul(x, w, out)
105
+ return out
106
+
107
+
108
+ def split_qkv_broadcast_bf16(
109
+ packed: torch.Tensor,
110
+ q_heads: int,
111
+ kv_heads: int,
112
+ v_heads: int,
113
+ head_dim: int,
114
+ *,
115
+ q: Optional[torch.Tensor] = None,
116
+ k: Optional[torch.Tensor] = None,
117
+ v: Optional[torch.Tensor] = None,
118
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
119
+ """Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
120
+ rows = packed.shape[0]
121
+ shape = (rows, v_heads, head_dim)
122
+ if q is None:
123
+ q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
124
+ if k is None:
125
+ k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
126
+ if v is None:
127
+ v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
128
+ ops.split_qkv_broadcast_bf16(
129
+ packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
130
+ )
131
+ return q, k, v
132
+
133
+
134
+ def partial_rope_qk_bf16(
135
+ q_in: torch.Tensor,
136
+ k_in: torch.Tensor,
137
+ cos: torch.Tensor,
138
+ sin: torch.Tensor,
139
+ rope_dim: int,
140
+ *,
141
+ q_out: Optional[torch.Tensor] = None,
142
+ k_out: Optional[torch.Tensor] = None,
143
+ ) -> tuple[torch.Tensor, torch.Tensor]:
144
+ """Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
145
+ if q_out is None:
146
+ q_out = torch.empty_like(q_in)
147
+ if k_out is None:
148
+ k_out = torch.empty_like(k_in)
149
+ ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
150
+ return q_out, k_out
151
+
152
+
153
+ def gated_delta_prepare_bf16(
154
+ a: torch.Tensor,
155
+ b: torch.Tensor,
156
+ neg_exp_a_log: torch.Tensor,
157
+ dt_bias: torch.Tensor,
158
+ *,
159
+ heads: Optional[int] = None,
160
+ a_stride: Optional[int] = None,
161
+ b_stride: Optional[int] = None,
162
+ g_out: Optional[torch.Tensor] = None,
163
+ beta_out: Optional[torch.Tensor] = None,
164
+ ) -> tuple[torch.Tensor, torch.Tensor]:
165
+ """Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
166
+ if heads is None:
167
+ heads = neg_exp_a_log.shape[0]
168
+ if a_stride is None:
169
+ a_stride = a.shape[1]
170
+ if b_stride is None:
171
+ b_stride = b.shape[1]
172
+ rows = a.shape[0]
173
+ if g_out is None:
174
+ g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
175
+ if beta_out is None:
176
+ beta_out = torch.empty_like(g_out)
177
+ ops.gated_delta_prepare_bf16(
178
+ a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
179
+ )
180
+ return g_out, beta_out
181
+
182
+
183
+ __all__ = [
184
+ "bf16_matvec",
185
+ "bf16_smallm_matmul",
186
+ "gated_delta_prepare_bf16",
187
+ "partial_rope_qk_bf16",
188
+ "split_qkv_broadcast_bf16",
189
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e4b4f3fa098de58a8a70fafeb3106c385d1caee3bffe57c6df90d0dd417685a
3
+ size 1261616
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _linear_attention_primitives_cuda_f14c443
3
+ ops = torch.ops._linear_attention_primitives_cuda_f14c443
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/linear_attention_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "linear-attention-primitives",
3
+ "id": "_linear_attention_primitives_cuda_f14c443",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT linear-attention helper kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
13
+ def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
14
+ if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
15
+ raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
16
+ if w.shape[0] < 256:
17
+ raise RuntimeError("bf16_matvec supports N >= 256")
18
+ return None
19
+
20
+
21
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
22
+ def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
23
+ if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
24
+ raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
25
+ if w.shape != (96, 5120):
26
+ raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
27
+ if x.shape[0] < 2 or x.shape[0] > 4:
28
+ raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
29
+ return None
30
+
31
+
32
+ @torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
33
+ def _split_qkv_broadcast_fake(
34
+ packed: torch.Tensor,
35
+ q: torch.Tensor,
36
+ k: torch.Tensor,
37
+ v: torch.Tensor,
38
+ q_heads: int,
39
+ kv_heads: int,
40
+ v_heads: int,
41
+ head_dim: int,
42
+ ) -> None:
43
+ rows = packed.shape[0]
44
+ if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
45
+ raise RuntimeError("packed shape mismatch")
46
+ if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
47
+ raise RuntimeError("q/k/v output shape mismatch")
48
+ return None
49
+
50
+
51
+ @torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
52
+ def _partial_rope_qk_fake(
53
+ q_in: torch.Tensor,
54
+ k_in: torch.Tensor,
55
+ cos: torch.Tensor,
56
+ sin: torch.Tensor,
57
+ q_out: torch.Tensor,
58
+ k_out: torch.Tensor,
59
+ rope_dim: int,
60
+ ) -> None:
61
+ if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
62
+ raise RuntimeError("q_in/k_in shape mismatch")
63
+ rows, _, head_dim = q_in.shape
64
+ if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
65
+ raise RuntimeError("invalid rope_dim")
66
+ if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
67
+ raise RuntimeError("cos/sin shape mismatch")
68
+ if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
69
+ raise RuntimeError("q_out/k_out shape mismatch")
70
+ return None
71
+
72
+
73
+ @torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
74
+ def _gated_delta_prepare_fake(
75
+ a: torch.Tensor,
76
+ b: torch.Tensor,
77
+ neg_exp_a_log: torch.Tensor,
78
+ dt_bias: torch.Tensor,
79
+ g_out: torch.Tensor,
80
+ beta_out: torch.Tensor,
81
+ a_stride: int,
82
+ b_stride: int,
83
+ ) -> None:
84
+ if g_out.dim() != 2 or beta_out.shape != g_out.shape:
85
+ raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
86
+ heads = g_out.shape[1]
87
+ if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
88
+ raise RuntimeError("per-head parameter shape mismatch")
89
+ return None
90
+
91
+
92
+ def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
93
+ """Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
94
+ if out is None:
95
+ out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
96
+ ops.bf16_matvec(x, w, out)
97
+ return out
98
+
99
+
100
+ def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
101
+ """Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
102
+ if out is None:
103
+ out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
104
+ ops.bf16_smallm_matmul(x, w, out)
105
+ return out
106
+
107
+
108
+ def split_qkv_broadcast_bf16(
109
+ packed: torch.Tensor,
110
+ q_heads: int,
111
+ kv_heads: int,
112
+ v_heads: int,
113
+ head_dim: int,
114
+ *,
115
+ q: Optional[torch.Tensor] = None,
116
+ k: Optional[torch.Tensor] = None,
117
+ v: Optional[torch.Tensor] = None,
118
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
119
+ """Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
120
+ rows = packed.shape[0]
121
+ shape = (rows, v_heads, head_dim)
122
+ if q is None:
123
+ q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
124
+ if k is None:
125
+ k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
126
+ if v is None:
127
+ v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
128
+ ops.split_qkv_broadcast_bf16(
129
+ packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
130
+ )
131
+ return q, k, v
132
+
133
+
134
+ def partial_rope_qk_bf16(
135
+ q_in: torch.Tensor,
136
+ k_in: torch.Tensor,
137
+ cos: torch.Tensor,
138
+ sin: torch.Tensor,
139
+ rope_dim: int,
140
+ *,
141
+ q_out: Optional[torch.Tensor] = None,
142
+ k_out: Optional[torch.Tensor] = None,
143
+ ) -> tuple[torch.Tensor, torch.Tensor]:
144
+ """Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
145
+ if q_out is None:
146
+ q_out = torch.empty_like(q_in)
147
+ if k_out is None:
148
+ k_out = torch.empty_like(k_in)
149
+ ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
150
+ return q_out, k_out
151
+
152
+
153
+ def gated_delta_prepare_bf16(
154
+ a: torch.Tensor,
155
+ b: torch.Tensor,
156
+ neg_exp_a_log: torch.Tensor,
157
+ dt_bias: torch.Tensor,
158
+ *,
159
+ heads: Optional[int] = None,
160
+ a_stride: Optional[int] = None,
161
+ b_stride: Optional[int] = None,
162
+ g_out: Optional[torch.Tensor] = None,
163
+ beta_out: Optional[torch.Tensor] = None,
164
+ ) -> tuple[torch.Tensor, torch.Tensor]:
165
+ """Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
166
+ if heads is None:
167
+ heads = neg_exp_a_log.shape[0]
168
+ if a_stride is None:
169
+ a_stride = a.shape[1]
170
+ if b_stride is None:
171
+ b_stride = b.shape[1]
172
+ rows = a.shape[0]
173
+ if g_out is None:
174
+ g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
175
+ if beta_out is None:
176
+ beta_out = torch.empty_like(g_out)
177
+ ops.gated_delta_prepare_bf16(
178
+ a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
179
+ )
180
+ return g_out, beta_out
181
+
182
+
183
+ __all__ = [
184
+ "bf16_matvec",
185
+ "bf16_smallm_matmul",
186
+ "gated_delta_prepare_bf16",
187
+ "partial_rope_qk_bf16",
188
+ "split_qkv_broadcast_bf16",
189
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a1c96d5ba309130cef62aab30c009915d90841b13d95b849c9aad1964ee528ce
3
+ size 1183776
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _linear_attention_primitives_cuda_f14c443
3
+ ops = torch.ops._linear_attention_primitives_cuda_f14c443
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "linear-attention-primitives",
3
+ "id": "_linear_attention_primitives_cuda_f14c443",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT linear-attention helper kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
13
+ def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
14
+ if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
15
+ raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
16
+ if w.shape[0] < 256:
17
+ raise RuntimeError("bf16_matvec supports N >= 256")
18
+ return None
19
+
20
+
21
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
22
+ def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
23
+ if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
24
+ raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
25
+ if w.shape != (96, 5120):
26
+ raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
27
+ if x.shape[0] < 2 or x.shape[0] > 4:
28
+ raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
29
+ return None
30
+
31
+
32
+ @torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
33
+ def _split_qkv_broadcast_fake(
34
+ packed: torch.Tensor,
35
+ q: torch.Tensor,
36
+ k: torch.Tensor,
37
+ v: torch.Tensor,
38
+ q_heads: int,
39
+ kv_heads: int,
40
+ v_heads: int,
41
+ head_dim: int,
42
+ ) -> None:
43
+ rows = packed.shape[0]
44
+ if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
45
+ raise RuntimeError("packed shape mismatch")
46
+ if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
47
+ raise RuntimeError("q/k/v output shape mismatch")
48
+ return None
49
+
50
+
51
+ @torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
52
+ def _partial_rope_qk_fake(
53
+ q_in: torch.Tensor,
54
+ k_in: torch.Tensor,
55
+ cos: torch.Tensor,
56
+ sin: torch.Tensor,
57
+ q_out: torch.Tensor,
58
+ k_out: torch.Tensor,
59
+ rope_dim: int,
60
+ ) -> None:
61
+ if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
62
+ raise RuntimeError("q_in/k_in shape mismatch")
63
+ rows, _, head_dim = q_in.shape
64
+ if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
65
+ raise RuntimeError("invalid rope_dim")
66
+ if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
67
+ raise RuntimeError("cos/sin shape mismatch")
68
+ if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
69
+ raise RuntimeError("q_out/k_out shape mismatch")
70
+ return None
71
+
72
+
73
+ @torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
74
+ def _gated_delta_prepare_fake(
75
+ a: torch.Tensor,
76
+ b: torch.Tensor,
77
+ neg_exp_a_log: torch.Tensor,
78
+ dt_bias: torch.Tensor,
79
+ g_out: torch.Tensor,
80
+ beta_out: torch.Tensor,
81
+ a_stride: int,
82
+ b_stride: int,
83
+ ) -> None:
84
+ if g_out.dim() != 2 or beta_out.shape != g_out.shape:
85
+ raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
86
+ heads = g_out.shape[1]
87
+ if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
88
+ raise RuntimeError("per-head parameter shape mismatch")
89
+ return None
90
+
91
+
92
+ def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
93
+ """Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
94
+ if out is None:
95
+ out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
96
+ ops.bf16_matvec(x, w, out)
97
+ return out
98
+
99
+
100
+ def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
101
+ """Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
102
+ if out is None:
103
+ out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
104
+ ops.bf16_smallm_matmul(x, w, out)
105
+ return out
106
+
107
+
108
+ def split_qkv_broadcast_bf16(
109
+ packed: torch.Tensor,
110
+ q_heads: int,
111
+ kv_heads: int,
112
+ v_heads: int,
113
+ head_dim: int,
114
+ *,
115
+ q: Optional[torch.Tensor] = None,
116
+ k: Optional[torch.Tensor] = None,
117
+ v: Optional[torch.Tensor] = None,
118
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
119
+ """Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
120
+ rows = packed.shape[0]
121
+ shape = (rows, v_heads, head_dim)
122
+ if q is None:
123
+ q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
124
+ if k is None:
125
+ k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
126
+ if v is None:
127
+ v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
128
+ ops.split_qkv_broadcast_bf16(
129
+ packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
130
+ )
131
+ return q, k, v
132
+
133
+
134
+ def partial_rope_qk_bf16(
135
+ q_in: torch.Tensor,
136
+ k_in: torch.Tensor,
137
+ cos: torch.Tensor,
138
+ sin: torch.Tensor,
139
+ rope_dim: int,
140
+ *,
141
+ q_out: Optional[torch.Tensor] = None,
142
+ k_out: Optional[torch.Tensor] = None,
143
+ ) -> tuple[torch.Tensor, torch.Tensor]:
144
+ """Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
145
+ if q_out is None:
146
+ q_out = torch.empty_like(q_in)
147
+ if k_out is None:
148
+ k_out = torch.empty_like(k_in)
149
+ ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
150
+ return q_out, k_out
151
+
152
+
153
+ def gated_delta_prepare_bf16(
154
+ a: torch.Tensor,
155
+ b: torch.Tensor,
156
+ neg_exp_a_log: torch.Tensor,
157
+ dt_bias: torch.Tensor,
158
+ *,
159
+ heads: Optional[int] = None,
160
+ a_stride: Optional[int] = None,
161
+ b_stride: Optional[int] = None,
162
+ g_out: Optional[torch.Tensor] = None,
163
+ beta_out: Optional[torch.Tensor] = None,
164
+ ) -> tuple[torch.Tensor, torch.Tensor]:
165
+ """Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
166
+ if heads is None:
167
+ heads = neg_exp_a_log.shape[0]
168
+ if a_stride is None:
169
+ a_stride = a.shape[1]
170
+ if b_stride is None:
171
+ b_stride = b.shape[1]
172
+ rows = a.shape[0]
173
+ if g_out is None:
174
+ g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
175
+ if beta_out is None:
176
+ beta_out = torch.empty_like(g_out)
177
+ ops.gated_delta_prepare_bf16(
178
+ a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
179
+ )
180
+ return g_out, beta_out
181
+
182
+
183
+ __all__ = [
184
+ "bf16_matvec",
185
+ "bf16_smallm_matmul",
186
+ "gated_delta_prepare_bf16",
187
+ "partial_rope_qk_bf16",
188
+ "split_qkv_broadcast_bf16",
189
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d79112942634a548486da765785f6ace2ac1b7e299e1d24f8d03e8ef468b9b63
3
+ size 1190504
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _linear_attention_primitives_cuda_f14c443
3
+ ops = torch.ops._linear_attention_primitives_cuda_f14c443
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "linear-attention-primitives",
3
+ "id": "_linear_attention_primitives_cuda_f14c443",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT linear-attention helper kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
13
+ def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
14
+ if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
15
+ raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
16
+ if w.shape[0] < 256:
17
+ raise RuntimeError("bf16_matvec supports N >= 256")
18
+ return None
19
+
20
+
21
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
22
+ def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
23
+ if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
24
+ raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
25
+ if w.shape != (96, 5120):
26
+ raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
27
+ if x.shape[0] < 2 or x.shape[0] > 4:
28
+ raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
29
+ return None
30
+
31
+
32
+ @torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
33
+ def _split_qkv_broadcast_fake(
34
+ packed: torch.Tensor,
35
+ q: torch.Tensor,
36
+ k: torch.Tensor,
37
+ v: torch.Tensor,
38
+ q_heads: int,
39
+ kv_heads: int,
40
+ v_heads: int,
41
+ head_dim: int,
42
+ ) -> None:
43
+ rows = packed.shape[0]
44
+ if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
45
+ raise RuntimeError("packed shape mismatch")
46
+ if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
47
+ raise RuntimeError("q/k/v output shape mismatch")
48
+ return None
49
+
50
+
51
+ @torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
52
+ def _partial_rope_qk_fake(
53
+ q_in: torch.Tensor,
54
+ k_in: torch.Tensor,
55
+ cos: torch.Tensor,
56
+ sin: torch.Tensor,
57
+ q_out: torch.Tensor,
58
+ k_out: torch.Tensor,
59
+ rope_dim: int,
60
+ ) -> None:
61
+ if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
62
+ raise RuntimeError("q_in/k_in shape mismatch")
63
+ rows, _, head_dim = q_in.shape
64
+ if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
65
+ raise RuntimeError("invalid rope_dim")
66
+ if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
67
+ raise RuntimeError("cos/sin shape mismatch")
68
+ if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
69
+ raise RuntimeError("q_out/k_out shape mismatch")
70
+ return None
71
+
72
+
73
+ @torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
74
+ def _gated_delta_prepare_fake(
75
+ a: torch.Tensor,
76
+ b: torch.Tensor,
77
+ neg_exp_a_log: torch.Tensor,
78
+ dt_bias: torch.Tensor,
79
+ g_out: torch.Tensor,
80
+ beta_out: torch.Tensor,
81
+ a_stride: int,
82
+ b_stride: int,
83
+ ) -> None:
84
+ if g_out.dim() != 2 or beta_out.shape != g_out.shape:
85
+ raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
86
+ heads = g_out.shape[1]
87
+ if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
88
+ raise RuntimeError("per-head parameter shape mismatch")
89
+ return None
90
+
91
+
92
+ def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
93
+ """Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
94
+ if out is None:
95
+ out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
96
+ ops.bf16_matvec(x, w, out)
97
+ return out
98
+
99
+
100
+ def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
101
+ """Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
102
+ if out is None:
103
+ out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
104
+ ops.bf16_smallm_matmul(x, w, out)
105
+ return out
106
+
107
+
108
+ def split_qkv_broadcast_bf16(
109
+ packed: torch.Tensor,
110
+ q_heads: int,
111
+ kv_heads: int,
112
+ v_heads: int,
113
+ head_dim: int,
114
+ *,
115
+ q: Optional[torch.Tensor] = None,
116
+ k: Optional[torch.Tensor] = None,
117
+ v: Optional[torch.Tensor] = None,
118
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
119
+ """Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
120
+ rows = packed.shape[0]
121
+ shape = (rows, v_heads, head_dim)
122
+ if q is None:
123
+ q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
124
+ if k is None:
125
+ k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
126
+ if v is None:
127
+ v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
128
+ ops.split_qkv_broadcast_bf16(
129
+ packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
130
+ )
131
+ return q, k, v
132
+
133
+
134
+ def partial_rope_qk_bf16(
135
+ q_in: torch.Tensor,
136
+ k_in: torch.Tensor,
137
+ cos: torch.Tensor,
138
+ sin: torch.Tensor,
139
+ rope_dim: int,
140
+ *,
141
+ q_out: Optional[torch.Tensor] = None,
142
+ k_out: Optional[torch.Tensor] = None,
143
+ ) -> tuple[torch.Tensor, torch.Tensor]:
144
+ """Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
145
+ if q_out is None:
146
+ q_out = torch.empty_like(q_in)
147
+ if k_out is None:
148
+ k_out = torch.empty_like(k_in)
149
+ ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
150
+ return q_out, k_out
151
+
152
+
153
+ def gated_delta_prepare_bf16(
154
+ a: torch.Tensor,
155
+ b: torch.Tensor,
156
+ neg_exp_a_log: torch.Tensor,
157
+ dt_bias: torch.Tensor,
158
+ *,
159
+ heads: Optional[int] = None,
160
+ a_stride: Optional[int] = None,
161
+ b_stride: Optional[int] = None,
162
+ g_out: Optional[torch.Tensor] = None,
163
+ beta_out: Optional[torch.Tensor] = None,
164
+ ) -> tuple[torch.Tensor, torch.Tensor]:
165
+ """Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
166
+ if heads is None:
167
+ heads = neg_exp_a_log.shape[0]
168
+ if a_stride is None:
169
+ a_stride = a.shape[1]
170
+ if b_stride is None:
171
+ b_stride = b.shape[1]
172
+ rows = a.shape[0]
173
+ if g_out is None:
174
+ g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
175
+ if beta_out is None:
176
+ beta_out = torch.empty_like(g_out)
177
+ ops.gated_delta_prepare_bf16(
178
+ a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
179
+ )
180
+ return g_out, beta_out
181
+
182
+
183
+ __all__ = [
184
+ "bf16_matvec",
185
+ "bf16_smallm_matmul",
186
+ "gated_delta_prepare_bf16",
187
+ "partial_rope_qk_bf16",
188
+ "split_qkv_broadcast_bf16",
189
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75c3513618c230a075ce98a46a559da0eb7a7a2cc4b2cf1cdb0841d7b2d9160a
3
+ size 1194600
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _linear_attention_primitives_cuda_f14c443
3
+ ops = torch.ops._linear_attention_primitives_cuda_f14c443
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/linear_attention_primitives/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "linear-attention-primitives",
3
+ "id": "_linear_attention_primitives_cuda_f14c443",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }