Promote latest kernel artifacts to main
Browse files- .gitattributes +3 -35
- README.md +0 -9
- benchmarks/benchmark.py +113 -0
- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +189 -0
- build/torch211-cxx11-cu128-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu128-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +189 -0
- build/torch211-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +189 -0
- build/torch212-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +189 -0
- build/torch212-cxx11-cu132-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so +3 -0
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu132-x86_64-linux/linear_attention_primitives/__init__.py +26 -0
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.so filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.pyd filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.dylib filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
# flashrt/linear-attention-primitives
|
| 2 |
-
|
| 3 |
-
This repository is a compatibility mirror for older `kernels` clients
|
| 4 |
-
that resolve repositories through the default Hugging Face model repo API.
|
| 5 |
-
|
| 6 |
-
Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/linear-attention-primitives
|
| 7 |
-
|
| 8 |
-
Do not edit this mirror by hand. It is generated from the Kernel Hub
|
| 9 |
-
`vN` branches and contains the same `build/**` artifacts.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
benchmarks/benchmark.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Benchmark linear-attention-primitives against PyTorch eager."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
PACKAGE = Path(__file__).resolve().parents[1]
|
| 13 |
+
sys.path.insert(0, str(PACKAGE / "tests"))
|
| 14 |
+
from test_linear_attention_primitives import load_installed_ops, load_source_ops # noqa: E402
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def bench(fn, warmup: int, iters: int) -> float:
|
| 18 |
+
for _ in range(warmup):
|
| 19 |
+
fn()
|
| 20 |
+
torch.cuda.synchronize()
|
| 21 |
+
start = torch.cuda.Event(enable_timing=True)
|
| 22 |
+
end = torch.cuda.Event(enable_timing=True)
|
| 23 |
+
start.record()
|
| 24 |
+
for _ in range(iters):
|
| 25 |
+
fn()
|
| 26 |
+
end.record()
|
| 27 |
+
torch.cuda.synchronize()
|
| 28 |
+
return start.elapsed_time(end) * 1000.0 / iters
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def main() -> int:
|
| 32 |
+
parser = argparse.ArgumentParser()
|
| 33 |
+
parser.add_argument("--backend", choices=["source", "installed"], default="source")
|
| 34 |
+
parser.add_argument("--artifact", default=None)
|
| 35 |
+
parser.add_argument("--warmup", type=int, default=100)
|
| 36 |
+
parser.add_argument("--iters", type=int, default=1000)
|
| 37 |
+
args = parser.parse_args()
|
| 38 |
+
if not torch.cuda.is_available():
|
| 39 |
+
raise RuntimeError("CUDA is required")
|
| 40 |
+
torch.manual_seed(123)
|
| 41 |
+
ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
|
| 42 |
+
|
| 43 |
+
print("| Workload | Shape | FlashRT us | PyTorch eager us | Speedup |")
|
| 44 |
+
print("|---|---:|---:|---:|---:|")
|
| 45 |
+
|
| 46 |
+
for k, n in [(128, 512), (4096, 1024), (5120, 10240)]:
|
| 47 |
+
x = (torch.randn((k,), device="cuda") * 0.05).to(torch.bfloat16)
|
| 48 |
+
w = (torch.randn((n, k), device="cuda") * 0.05).to(torch.bfloat16)
|
| 49 |
+
fused = bench(lambda: ops.bf16_matvec(x, w), args.warmup, args.iters)
|
| 50 |
+
eager = bench(lambda: x @ w.t(), args.warmup, args.iters)
|
| 51 |
+
print(f"| bf16_matvec | N={n},K={k} | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
|
| 52 |
+
|
| 53 |
+
for m, k, n in [(2, 5120, 96), (3, 5120, 96), (4, 5120, 96)]:
|
| 54 |
+
x = (torch.randn((m, k), device="cuda") * 0.05).to(torch.bfloat16)
|
| 55 |
+
w = (torch.randn((n, k), device="cuda") * 0.05).to(torch.bfloat16)
|
| 56 |
+
fused = bench(lambda: ops.bf16_smallm_matmul(x, w), args.warmup, args.iters)
|
| 57 |
+
eager = bench(lambda: x @ w.t(), args.warmup, args.iters)
|
| 58 |
+
print(f"| bf16_smallm_matmul | M={m},N={n},K={k} | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
|
| 59 |
+
|
| 60 |
+
for rows in [1, 128, 1024]:
|
| 61 |
+
packed = torch.randn((rows, (16 + 16 + 48) * 128), device="cuda", dtype=torch.bfloat16)
|
| 62 |
+
fused = bench(lambda: ops.split_qkv_broadcast_bf16(packed, 16, 16, 48, 128), args.warmup, args.iters)
|
| 63 |
+
eager = bench(
|
| 64 |
+
lambda: (
|
| 65 |
+
packed[:, : 16 * 128].reshape(rows, 16, 128)[:, torch.arange(48, device="cuda") * 16 // 48].contiguous(),
|
| 66 |
+
packed[:, 16 * 128 : 32 * 128].reshape(rows, 16, 128)[:, torch.arange(48, device="cuda") * 16 // 48].contiguous(),
|
| 67 |
+
packed[:, 32 * 128 :].reshape(rows, 48, 128).contiguous(),
|
| 68 |
+
),
|
| 69 |
+
args.warmup,
|
| 70 |
+
args.iters,
|
| 71 |
+
)
|
| 72 |
+
print(f"| split_qkv_broadcast | rows={rows},heads=16/48,dim=128 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
|
| 73 |
+
|
| 74 |
+
for rows in [1, 128, 1024]:
|
| 75 |
+
q = torch.randn((rows, 16, 128), device="cuda", dtype=torch.bfloat16)
|
| 76 |
+
k = torch.randn((rows, 16, 128), device="cuda", dtype=torch.bfloat16)
|
| 77 |
+
cos = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
|
| 78 |
+
sin = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
def eager_rope():
|
| 81 |
+
half = 32
|
| 82 |
+
qo = q.clone()
|
| 83 |
+
ko = k.clone()
|
| 84 |
+
qo[:, :, :half] = ((-q[:, :, half:64].float() * sin[:, None, :half].float()).to(torch.bfloat16).float() + q[:, :, :half].float() * cos[:, None, :half].float()).to(torch.bfloat16)
|
| 85 |
+
qo[:, :, half:64] = ((q[:, :, :half].float() * sin[:, None, half:64].float()).to(torch.bfloat16).float() + q[:, :, half:64].float() * cos[:, None, half:64].float()).to(torch.bfloat16)
|
| 86 |
+
ko[:, :, :half] = ((-k[:, :, half:64].float() * sin[:, None, :half].float()).to(torch.bfloat16).float() + k[:, :, :half].float() * cos[:, None, :half].float()).to(torch.bfloat16)
|
| 87 |
+
ko[:, :, half:64] = ((k[:, :, :half].float() * sin[:, None, half:64].float()).to(torch.bfloat16).float() + k[:, :, half:64].float() * cos[:, None, half:64].float()).to(torch.bfloat16)
|
| 88 |
+
return qo, ko
|
| 89 |
+
|
| 90 |
+
fused = bench(lambda: ops.partial_rope_qk_bf16(q, k, cos, sin, 64), args.warmup, args.iters)
|
| 91 |
+
eager = bench(eager_rope, args.warmup, args.iters)
|
| 92 |
+
print(f"| partial_rope_qk | rows={rows},heads=16,dim=128,rope=64 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
|
| 93 |
+
|
| 94 |
+
for rows in [1, 128, 1024]:
|
| 95 |
+
a = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
|
| 96 |
+
b = torch.randn((rows, 64), device="cuda", dtype=torch.bfloat16)
|
| 97 |
+
neg = torch.randn((48,), device="cuda", dtype=torch.float32) * 0.1
|
| 98 |
+
bias = torch.randn((48,), device="cuda", dtype=torch.float32) * 0.1
|
| 99 |
+
fused = bench(lambda: ops.gated_delta_prepare_bf16(a, b, neg, bias, heads=48, a_stride=64, b_stride=64), args.warmup, args.iters)
|
| 100 |
+
eager = bench(
|
| 101 |
+
lambda: (
|
| 102 |
+
neg[None, :] * torch.nn.functional.softplus(a[:, :48].float() + bias[None, :])
|
| 103 |
+
).to(torch.bfloat16),
|
| 104 |
+
args.warmup,
|
| 105 |
+
args.iters,
|
| 106 |
+
)
|
| 107 |
+
print(f"| gated_delta_prepare | rows={rows},heads=48 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
|
| 108 |
+
|
| 109 |
+
return 0
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
if __name__ == "__main__":
|
| 113 |
+
raise SystemExit(main())
|
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT linear-attention helper kernels."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
|
| 13 |
+
def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
|
| 15 |
+
raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
|
| 16 |
+
if w.shape[0] < 256:
|
| 17 |
+
raise RuntimeError("bf16_matvec supports N >= 256")
|
| 18 |
+
return None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
|
| 22 |
+
def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 23 |
+
if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
|
| 24 |
+
raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
|
| 25 |
+
if w.shape != (96, 5120):
|
| 26 |
+
raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
|
| 27 |
+
if x.shape[0] < 2 or x.shape[0] > 4:
|
| 28 |
+
raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
|
| 29 |
+
return None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
|
| 33 |
+
def _split_qkv_broadcast_fake(
|
| 34 |
+
packed: torch.Tensor,
|
| 35 |
+
q: torch.Tensor,
|
| 36 |
+
k: torch.Tensor,
|
| 37 |
+
v: torch.Tensor,
|
| 38 |
+
q_heads: int,
|
| 39 |
+
kv_heads: int,
|
| 40 |
+
v_heads: int,
|
| 41 |
+
head_dim: int,
|
| 42 |
+
) -> None:
|
| 43 |
+
rows = packed.shape[0]
|
| 44 |
+
if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
|
| 45 |
+
raise RuntimeError("packed shape mismatch")
|
| 46 |
+
if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
|
| 47 |
+
raise RuntimeError("q/k/v output shape mismatch")
|
| 48 |
+
return None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
|
| 52 |
+
def _partial_rope_qk_fake(
|
| 53 |
+
q_in: torch.Tensor,
|
| 54 |
+
k_in: torch.Tensor,
|
| 55 |
+
cos: torch.Tensor,
|
| 56 |
+
sin: torch.Tensor,
|
| 57 |
+
q_out: torch.Tensor,
|
| 58 |
+
k_out: torch.Tensor,
|
| 59 |
+
rope_dim: int,
|
| 60 |
+
) -> None:
|
| 61 |
+
if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
|
| 62 |
+
raise RuntimeError("q_in/k_in shape mismatch")
|
| 63 |
+
rows, _, head_dim = q_in.shape
|
| 64 |
+
if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
|
| 65 |
+
raise RuntimeError("invalid rope_dim")
|
| 66 |
+
if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
|
| 67 |
+
raise RuntimeError("cos/sin shape mismatch")
|
| 68 |
+
if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
|
| 69 |
+
raise RuntimeError("q_out/k_out shape mismatch")
|
| 70 |
+
return None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
|
| 74 |
+
def _gated_delta_prepare_fake(
|
| 75 |
+
a: torch.Tensor,
|
| 76 |
+
b: torch.Tensor,
|
| 77 |
+
neg_exp_a_log: torch.Tensor,
|
| 78 |
+
dt_bias: torch.Tensor,
|
| 79 |
+
g_out: torch.Tensor,
|
| 80 |
+
beta_out: torch.Tensor,
|
| 81 |
+
a_stride: int,
|
| 82 |
+
b_stride: int,
|
| 83 |
+
) -> None:
|
| 84 |
+
if g_out.dim() != 2 or beta_out.shape != g_out.shape:
|
| 85 |
+
raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
|
| 86 |
+
heads = g_out.shape[1]
|
| 87 |
+
if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
|
| 88 |
+
raise RuntimeError("per-head parameter shape mismatch")
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 93 |
+
"""Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
|
| 94 |
+
if out is None:
|
| 95 |
+
out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
|
| 96 |
+
ops.bf16_matvec(x, w, out)
|
| 97 |
+
return out
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 101 |
+
"""Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
|
| 104 |
+
ops.bf16_smallm_matmul(x, w, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def split_qkv_broadcast_bf16(
|
| 109 |
+
packed: torch.Tensor,
|
| 110 |
+
q_heads: int,
|
| 111 |
+
kv_heads: int,
|
| 112 |
+
v_heads: int,
|
| 113 |
+
head_dim: int,
|
| 114 |
+
*,
|
| 115 |
+
q: Optional[torch.Tensor] = None,
|
| 116 |
+
k: Optional[torch.Tensor] = None,
|
| 117 |
+
v: Optional[torch.Tensor] = None,
|
| 118 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 119 |
+
"""Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
|
| 120 |
+
rows = packed.shape[0]
|
| 121 |
+
shape = (rows, v_heads, head_dim)
|
| 122 |
+
if q is None:
|
| 123 |
+
q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 124 |
+
if k is None:
|
| 125 |
+
k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 126 |
+
if v is None:
|
| 127 |
+
v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 128 |
+
ops.split_qkv_broadcast_bf16(
|
| 129 |
+
packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
|
| 130 |
+
)
|
| 131 |
+
return q, k, v
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def partial_rope_qk_bf16(
|
| 135 |
+
q_in: torch.Tensor,
|
| 136 |
+
k_in: torch.Tensor,
|
| 137 |
+
cos: torch.Tensor,
|
| 138 |
+
sin: torch.Tensor,
|
| 139 |
+
rope_dim: int,
|
| 140 |
+
*,
|
| 141 |
+
q_out: Optional[torch.Tensor] = None,
|
| 142 |
+
k_out: Optional[torch.Tensor] = None,
|
| 143 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 144 |
+
"""Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
|
| 145 |
+
if q_out is None:
|
| 146 |
+
q_out = torch.empty_like(q_in)
|
| 147 |
+
if k_out is None:
|
| 148 |
+
k_out = torch.empty_like(k_in)
|
| 149 |
+
ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
|
| 150 |
+
return q_out, k_out
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def gated_delta_prepare_bf16(
|
| 154 |
+
a: torch.Tensor,
|
| 155 |
+
b: torch.Tensor,
|
| 156 |
+
neg_exp_a_log: torch.Tensor,
|
| 157 |
+
dt_bias: torch.Tensor,
|
| 158 |
+
*,
|
| 159 |
+
heads: Optional[int] = None,
|
| 160 |
+
a_stride: Optional[int] = None,
|
| 161 |
+
b_stride: Optional[int] = None,
|
| 162 |
+
g_out: Optional[torch.Tensor] = None,
|
| 163 |
+
beta_out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 165 |
+
"""Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
|
| 166 |
+
if heads is None:
|
| 167 |
+
heads = neg_exp_a_log.shape[0]
|
| 168 |
+
if a_stride is None:
|
| 169 |
+
a_stride = a.shape[1]
|
| 170 |
+
if b_stride is None:
|
| 171 |
+
b_stride = b.shape[1]
|
| 172 |
+
rows = a.shape[0]
|
| 173 |
+
if g_out is None:
|
| 174 |
+
g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
|
| 175 |
+
if beta_out is None:
|
| 176 |
+
beta_out = torch.empty_like(g_out)
|
| 177 |
+
ops.gated_delta_prepare_bf16(
|
| 178 |
+
a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
|
| 179 |
+
)
|
| 180 |
+
return g_out, beta_out
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
__all__ = [
|
| 184 |
+
"bf16_matvec",
|
| 185 |
+
"bf16_smallm_matmul",
|
| 186 |
+
"gated_delta_prepare_bf16",
|
| 187 |
+
"partial_rope_qk_bf16",
|
| 188 |
+
"split_qkv_broadcast_bf16",
|
| 189 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7e4b4f3fa098de58a8a70fafeb3106c385d1caee3bffe57c6df90d0dd417685a
|
| 3 |
+
size 1261616
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _linear_attention_primitives_cuda_f14c443
|
| 3 |
+
ops = torch.ops._linear_attention_primitives_cuda_f14c443
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/linear_attention_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "linear-attention-primitives",
|
| 3 |
+
"id": "_linear_attention_primitives_cuda_f14c443",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"10.1",
|
| 12 |
+
"12.0+PTX",
|
| 13 |
+
"7.0",
|
| 14 |
+
"7.2",
|
| 15 |
+
"7.5",
|
| 16 |
+
"8.0",
|
| 17 |
+
"8.6",
|
| 18 |
+
"8.7",
|
| 19 |
+
"8.9",
|
| 20 |
+
"9.0"
|
| 21 |
+
]
|
| 22 |
+
}
|
| 23 |
+
}
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT linear-attention helper kernels."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
|
| 13 |
+
def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
|
| 15 |
+
raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
|
| 16 |
+
if w.shape[0] < 256:
|
| 17 |
+
raise RuntimeError("bf16_matvec supports N >= 256")
|
| 18 |
+
return None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
|
| 22 |
+
def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 23 |
+
if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
|
| 24 |
+
raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
|
| 25 |
+
if w.shape != (96, 5120):
|
| 26 |
+
raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
|
| 27 |
+
if x.shape[0] < 2 or x.shape[0] > 4:
|
| 28 |
+
raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
|
| 29 |
+
return None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
|
| 33 |
+
def _split_qkv_broadcast_fake(
|
| 34 |
+
packed: torch.Tensor,
|
| 35 |
+
q: torch.Tensor,
|
| 36 |
+
k: torch.Tensor,
|
| 37 |
+
v: torch.Tensor,
|
| 38 |
+
q_heads: int,
|
| 39 |
+
kv_heads: int,
|
| 40 |
+
v_heads: int,
|
| 41 |
+
head_dim: int,
|
| 42 |
+
) -> None:
|
| 43 |
+
rows = packed.shape[0]
|
| 44 |
+
if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
|
| 45 |
+
raise RuntimeError("packed shape mismatch")
|
| 46 |
+
if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
|
| 47 |
+
raise RuntimeError("q/k/v output shape mismatch")
|
| 48 |
+
return None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
|
| 52 |
+
def _partial_rope_qk_fake(
|
| 53 |
+
q_in: torch.Tensor,
|
| 54 |
+
k_in: torch.Tensor,
|
| 55 |
+
cos: torch.Tensor,
|
| 56 |
+
sin: torch.Tensor,
|
| 57 |
+
q_out: torch.Tensor,
|
| 58 |
+
k_out: torch.Tensor,
|
| 59 |
+
rope_dim: int,
|
| 60 |
+
) -> None:
|
| 61 |
+
if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
|
| 62 |
+
raise RuntimeError("q_in/k_in shape mismatch")
|
| 63 |
+
rows, _, head_dim = q_in.shape
|
| 64 |
+
if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
|
| 65 |
+
raise RuntimeError("invalid rope_dim")
|
| 66 |
+
if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
|
| 67 |
+
raise RuntimeError("cos/sin shape mismatch")
|
| 68 |
+
if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
|
| 69 |
+
raise RuntimeError("q_out/k_out shape mismatch")
|
| 70 |
+
return None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
|
| 74 |
+
def _gated_delta_prepare_fake(
|
| 75 |
+
a: torch.Tensor,
|
| 76 |
+
b: torch.Tensor,
|
| 77 |
+
neg_exp_a_log: torch.Tensor,
|
| 78 |
+
dt_bias: torch.Tensor,
|
| 79 |
+
g_out: torch.Tensor,
|
| 80 |
+
beta_out: torch.Tensor,
|
| 81 |
+
a_stride: int,
|
| 82 |
+
b_stride: int,
|
| 83 |
+
) -> None:
|
| 84 |
+
if g_out.dim() != 2 or beta_out.shape != g_out.shape:
|
| 85 |
+
raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
|
| 86 |
+
heads = g_out.shape[1]
|
| 87 |
+
if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
|
| 88 |
+
raise RuntimeError("per-head parameter shape mismatch")
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 93 |
+
"""Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
|
| 94 |
+
if out is None:
|
| 95 |
+
out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
|
| 96 |
+
ops.bf16_matvec(x, w, out)
|
| 97 |
+
return out
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 101 |
+
"""Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
|
| 104 |
+
ops.bf16_smallm_matmul(x, w, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def split_qkv_broadcast_bf16(
|
| 109 |
+
packed: torch.Tensor,
|
| 110 |
+
q_heads: int,
|
| 111 |
+
kv_heads: int,
|
| 112 |
+
v_heads: int,
|
| 113 |
+
head_dim: int,
|
| 114 |
+
*,
|
| 115 |
+
q: Optional[torch.Tensor] = None,
|
| 116 |
+
k: Optional[torch.Tensor] = None,
|
| 117 |
+
v: Optional[torch.Tensor] = None,
|
| 118 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 119 |
+
"""Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
|
| 120 |
+
rows = packed.shape[0]
|
| 121 |
+
shape = (rows, v_heads, head_dim)
|
| 122 |
+
if q is None:
|
| 123 |
+
q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 124 |
+
if k is None:
|
| 125 |
+
k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 126 |
+
if v is None:
|
| 127 |
+
v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 128 |
+
ops.split_qkv_broadcast_bf16(
|
| 129 |
+
packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
|
| 130 |
+
)
|
| 131 |
+
return q, k, v
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def partial_rope_qk_bf16(
|
| 135 |
+
q_in: torch.Tensor,
|
| 136 |
+
k_in: torch.Tensor,
|
| 137 |
+
cos: torch.Tensor,
|
| 138 |
+
sin: torch.Tensor,
|
| 139 |
+
rope_dim: int,
|
| 140 |
+
*,
|
| 141 |
+
q_out: Optional[torch.Tensor] = None,
|
| 142 |
+
k_out: Optional[torch.Tensor] = None,
|
| 143 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 144 |
+
"""Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
|
| 145 |
+
if q_out is None:
|
| 146 |
+
q_out = torch.empty_like(q_in)
|
| 147 |
+
if k_out is None:
|
| 148 |
+
k_out = torch.empty_like(k_in)
|
| 149 |
+
ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
|
| 150 |
+
return q_out, k_out
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def gated_delta_prepare_bf16(
|
| 154 |
+
a: torch.Tensor,
|
| 155 |
+
b: torch.Tensor,
|
| 156 |
+
neg_exp_a_log: torch.Tensor,
|
| 157 |
+
dt_bias: torch.Tensor,
|
| 158 |
+
*,
|
| 159 |
+
heads: Optional[int] = None,
|
| 160 |
+
a_stride: Optional[int] = None,
|
| 161 |
+
b_stride: Optional[int] = None,
|
| 162 |
+
g_out: Optional[torch.Tensor] = None,
|
| 163 |
+
beta_out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 165 |
+
"""Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
|
| 166 |
+
if heads is None:
|
| 167 |
+
heads = neg_exp_a_log.shape[0]
|
| 168 |
+
if a_stride is None:
|
| 169 |
+
a_stride = a.shape[1]
|
| 170 |
+
if b_stride is None:
|
| 171 |
+
b_stride = b.shape[1]
|
| 172 |
+
rows = a.shape[0]
|
| 173 |
+
if g_out is None:
|
| 174 |
+
g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
|
| 175 |
+
if beta_out is None:
|
| 176 |
+
beta_out = torch.empty_like(g_out)
|
| 177 |
+
ops.gated_delta_prepare_bf16(
|
| 178 |
+
a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
|
| 179 |
+
)
|
| 180 |
+
return g_out, beta_out
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
__all__ = [
|
| 184 |
+
"bf16_matvec",
|
| 185 |
+
"bf16_smallm_matmul",
|
| 186 |
+
"gated_delta_prepare_bf16",
|
| 187 |
+
"partial_rope_qk_bf16",
|
| 188 |
+
"split_qkv_broadcast_bf16",
|
| 189 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a1c96d5ba309130cef62aab30c009915d90841b13d95b849c9aad1964ee528ce
|
| 3 |
+
size 1183776
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _linear_attention_primitives_cuda_f14c443
|
| 3 |
+
ops = torch.ops._linear_attention_primitives_cuda_f14c443
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "linear-attention-primitives",
|
| 3 |
+
"id": "_linear_attention_primitives_cuda_f14c443",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT linear-attention helper kernels."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
|
| 13 |
+
def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
|
| 15 |
+
raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
|
| 16 |
+
if w.shape[0] < 256:
|
| 17 |
+
raise RuntimeError("bf16_matvec supports N >= 256")
|
| 18 |
+
return None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
|
| 22 |
+
def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 23 |
+
if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
|
| 24 |
+
raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
|
| 25 |
+
if w.shape != (96, 5120):
|
| 26 |
+
raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
|
| 27 |
+
if x.shape[0] < 2 or x.shape[0] > 4:
|
| 28 |
+
raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
|
| 29 |
+
return None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
|
| 33 |
+
def _split_qkv_broadcast_fake(
|
| 34 |
+
packed: torch.Tensor,
|
| 35 |
+
q: torch.Tensor,
|
| 36 |
+
k: torch.Tensor,
|
| 37 |
+
v: torch.Tensor,
|
| 38 |
+
q_heads: int,
|
| 39 |
+
kv_heads: int,
|
| 40 |
+
v_heads: int,
|
| 41 |
+
head_dim: int,
|
| 42 |
+
) -> None:
|
| 43 |
+
rows = packed.shape[0]
|
| 44 |
+
if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
|
| 45 |
+
raise RuntimeError("packed shape mismatch")
|
| 46 |
+
if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
|
| 47 |
+
raise RuntimeError("q/k/v output shape mismatch")
|
| 48 |
+
return None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
|
| 52 |
+
def _partial_rope_qk_fake(
|
| 53 |
+
q_in: torch.Tensor,
|
| 54 |
+
k_in: torch.Tensor,
|
| 55 |
+
cos: torch.Tensor,
|
| 56 |
+
sin: torch.Tensor,
|
| 57 |
+
q_out: torch.Tensor,
|
| 58 |
+
k_out: torch.Tensor,
|
| 59 |
+
rope_dim: int,
|
| 60 |
+
) -> None:
|
| 61 |
+
if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
|
| 62 |
+
raise RuntimeError("q_in/k_in shape mismatch")
|
| 63 |
+
rows, _, head_dim = q_in.shape
|
| 64 |
+
if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
|
| 65 |
+
raise RuntimeError("invalid rope_dim")
|
| 66 |
+
if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
|
| 67 |
+
raise RuntimeError("cos/sin shape mismatch")
|
| 68 |
+
if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
|
| 69 |
+
raise RuntimeError("q_out/k_out shape mismatch")
|
| 70 |
+
return None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
|
| 74 |
+
def _gated_delta_prepare_fake(
|
| 75 |
+
a: torch.Tensor,
|
| 76 |
+
b: torch.Tensor,
|
| 77 |
+
neg_exp_a_log: torch.Tensor,
|
| 78 |
+
dt_bias: torch.Tensor,
|
| 79 |
+
g_out: torch.Tensor,
|
| 80 |
+
beta_out: torch.Tensor,
|
| 81 |
+
a_stride: int,
|
| 82 |
+
b_stride: int,
|
| 83 |
+
) -> None:
|
| 84 |
+
if g_out.dim() != 2 or beta_out.shape != g_out.shape:
|
| 85 |
+
raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
|
| 86 |
+
heads = g_out.shape[1]
|
| 87 |
+
if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
|
| 88 |
+
raise RuntimeError("per-head parameter shape mismatch")
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 93 |
+
"""Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
|
| 94 |
+
if out is None:
|
| 95 |
+
out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
|
| 96 |
+
ops.bf16_matvec(x, w, out)
|
| 97 |
+
return out
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 101 |
+
"""Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
|
| 104 |
+
ops.bf16_smallm_matmul(x, w, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def split_qkv_broadcast_bf16(
|
| 109 |
+
packed: torch.Tensor,
|
| 110 |
+
q_heads: int,
|
| 111 |
+
kv_heads: int,
|
| 112 |
+
v_heads: int,
|
| 113 |
+
head_dim: int,
|
| 114 |
+
*,
|
| 115 |
+
q: Optional[torch.Tensor] = None,
|
| 116 |
+
k: Optional[torch.Tensor] = None,
|
| 117 |
+
v: Optional[torch.Tensor] = None,
|
| 118 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 119 |
+
"""Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
|
| 120 |
+
rows = packed.shape[0]
|
| 121 |
+
shape = (rows, v_heads, head_dim)
|
| 122 |
+
if q is None:
|
| 123 |
+
q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 124 |
+
if k is None:
|
| 125 |
+
k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 126 |
+
if v is None:
|
| 127 |
+
v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 128 |
+
ops.split_qkv_broadcast_bf16(
|
| 129 |
+
packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
|
| 130 |
+
)
|
| 131 |
+
return q, k, v
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def partial_rope_qk_bf16(
|
| 135 |
+
q_in: torch.Tensor,
|
| 136 |
+
k_in: torch.Tensor,
|
| 137 |
+
cos: torch.Tensor,
|
| 138 |
+
sin: torch.Tensor,
|
| 139 |
+
rope_dim: int,
|
| 140 |
+
*,
|
| 141 |
+
q_out: Optional[torch.Tensor] = None,
|
| 142 |
+
k_out: Optional[torch.Tensor] = None,
|
| 143 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 144 |
+
"""Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
|
| 145 |
+
if q_out is None:
|
| 146 |
+
q_out = torch.empty_like(q_in)
|
| 147 |
+
if k_out is None:
|
| 148 |
+
k_out = torch.empty_like(k_in)
|
| 149 |
+
ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
|
| 150 |
+
return q_out, k_out
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def gated_delta_prepare_bf16(
|
| 154 |
+
a: torch.Tensor,
|
| 155 |
+
b: torch.Tensor,
|
| 156 |
+
neg_exp_a_log: torch.Tensor,
|
| 157 |
+
dt_bias: torch.Tensor,
|
| 158 |
+
*,
|
| 159 |
+
heads: Optional[int] = None,
|
| 160 |
+
a_stride: Optional[int] = None,
|
| 161 |
+
b_stride: Optional[int] = None,
|
| 162 |
+
g_out: Optional[torch.Tensor] = None,
|
| 163 |
+
beta_out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 165 |
+
"""Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
|
| 166 |
+
if heads is None:
|
| 167 |
+
heads = neg_exp_a_log.shape[0]
|
| 168 |
+
if a_stride is None:
|
| 169 |
+
a_stride = a.shape[1]
|
| 170 |
+
if b_stride is None:
|
| 171 |
+
b_stride = b.shape[1]
|
| 172 |
+
rows = a.shape[0]
|
| 173 |
+
if g_out is None:
|
| 174 |
+
g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
|
| 175 |
+
if beta_out is None:
|
| 176 |
+
beta_out = torch.empty_like(g_out)
|
| 177 |
+
ops.gated_delta_prepare_bf16(
|
| 178 |
+
a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
|
| 179 |
+
)
|
| 180 |
+
return g_out, beta_out
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
__all__ = [
|
| 184 |
+
"bf16_matvec",
|
| 185 |
+
"bf16_smallm_matmul",
|
| 186 |
+
"gated_delta_prepare_bf16",
|
| 187 |
+
"partial_rope_qk_bf16",
|
| 188 |
+
"split_qkv_broadcast_bf16",
|
| 189 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d79112942634a548486da765785f6ace2ac1b7e299e1d24f8d03e8ef468b9b63
|
| 3 |
+
size 1190504
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _linear_attention_primitives_cuda_f14c443
|
| 3 |
+
ops = torch.ops._linear_attention_primitives_cuda_f14c443
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/linear_attention_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "linear-attention-primitives",
|
| 3 |
+
"id": "_linear_attention_primitives_cuda_f14c443",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT linear-attention helper kernels."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_matvec"))
|
| 13 |
+
def _bf16_matvec_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 14 |
+
if x.dim() != 1 or w.dim() != 2 or w.shape[1] != x.shape[0] or out.shape != (w.shape[0],):
|
| 15 |
+
raise RuntimeError("bf16_matvec expects x (K,), w (N,K), out (N,)")
|
| 16 |
+
if w.shape[0] < 256:
|
| 17 |
+
raise RuntimeError("bf16_matvec supports N >= 256")
|
| 18 |
+
return None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_smallm_matmul"))
|
| 22 |
+
def _bf16_smallm_matmul_fake(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor) -> None:
|
| 23 |
+
if x.dim() != 2 or w.dim() != 2 or w.shape[1] != x.shape[1] or out.shape != (x.shape[0], w.shape[0]):
|
| 24 |
+
raise RuntimeError("bf16_smallm_matmul expects x (M,K), w (N,K), out (M,N)")
|
| 25 |
+
if w.shape != (96, 5120):
|
| 26 |
+
raise RuntimeError("bf16_smallm_matmul supports tuned AB96 shape N=96,K=5120")
|
| 27 |
+
if x.shape[0] < 2 or x.shape[0] > 4:
|
| 28 |
+
raise RuntimeError("bf16_smallm_matmul supports 2 <= M <= 4")
|
| 29 |
+
return None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@torch.library.register_fake(add_op_namespace_prefix("split_qkv_broadcast_bf16"))
|
| 33 |
+
def _split_qkv_broadcast_fake(
|
| 34 |
+
packed: torch.Tensor,
|
| 35 |
+
q: torch.Tensor,
|
| 36 |
+
k: torch.Tensor,
|
| 37 |
+
v: torch.Tensor,
|
| 38 |
+
q_heads: int,
|
| 39 |
+
kv_heads: int,
|
| 40 |
+
v_heads: int,
|
| 41 |
+
head_dim: int,
|
| 42 |
+
) -> None:
|
| 43 |
+
rows = packed.shape[0]
|
| 44 |
+
if packed.shape != (rows, (q_heads + kv_heads + v_heads) * head_dim):
|
| 45 |
+
raise RuntimeError("packed shape mismatch")
|
| 46 |
+
if q.shape != (rows, v_heads, head_dim) or k.shape != (rows, v_heads, head_dim) or v.shape != (rows, v_heads, head_dim):
|
| 47 |
+
raise RuntimeError("q/k/v output shape mismatch")
|
| 48 |
+
return None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@torch.library.register_fake(add_op_namespace_prefix("partial_rope_qk_bf16"))
|
| 52 |
+
def _partial_rope_qk_fake(
|
| 53 |
+
q_in: torch.Tensor,
|
| 54 |
+
k_in: torch.Tensor,
|
| 55 |
+
cos: torch.Tensor,
|
| 56 |
+
sin: torch.Tensor,
|
| 57 |
+
q_out: torch.Tensor,
|
| 58 |
+
k_out: torch.Tensor,
|
| 59 |
+
rope_dim: int,
|
| 60 |
+
) -> None:
|
| 61 |
+
if q_in.dim() != 3 or k_in.dim() != 3 or q_in.shape[0] != k_in.shape[0] or q_in.shape[2] != k_in.shape[2]:
|
| 62 |
+
raise RuntimeError("q_in/k_in shape mismatch")
|
| 63 |
+
rows, _, head_dim = q_in.shape
|
| 64 |
+
if rope_dim <= 0 or rope_dim > head_dim or rope_dim % 2 != 0:
|
| 65 |
+
raise RuntimeError("invalid rope_dim")
|
| 66 |
+
if cos.shape != (rows, rope_dim) or sin.shape != (rows, rope_dim):
|
| 67 |
+
raise RuntimeError("cos/sin shape mismatch")
|
| 68 |
+
if q_out.shape != q_in.shape or k_out.shape != k_in.shape:
|
| 69 |
+
raise RuntimeError("q_out/k_out shape mismatch")
|
| 70 |
+
return None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@torch.library.register_fake(add_op_namespace_prefix("gated_delta_prepare_bf16"))
|
| 74 |
+
def _gated_delta_prepare_fake(
|
| 75 |
+
a: torch.Tensor,
|
| 76 |
+
b: torch.Tensor,
|
| 77 |
+
neg_exp_a_log: torch.Tensor,
|
| 78 |
+
dt_bias: torch.Tensor,
|
| 79 |
+
g_out: torch.Tensor,
|
| 80 |
+
beta_out: torch.Tensor,
|
| 81 |
+
a_stride: int,
|
| 82 |
+
b_stride: int,
|
| 83 |
+
) -> None:
|
| 84 |
+
if g_out.dim() != 2 or beta_out.shape != g_out.shape:
|
| 85 |
+
raise RuntimeError("g_out/beta_out must have shape (rows, heads)")
|
| 86 |
+
heads = g_out.shape[1]
|
| 87 |
+
if neg_exp_a_log.shape != (heads,) or dt_bias.shape != (heads,):
|
| 88 |
+
raise RuntimeError("per-head parameter shape mismatch")
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def bf16_matvec(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 93 |
+
"""Compute `out = x @ w.T` for BF16 `x (K,)` and `w (N, K)`."""
|
| 94 |
+
if out is None:
|
| 95 |
+
out = torch.empty((w.shape[0],), device=x.device, dtype=torch.bfloat16)
|
| 96 |
+
ops.bf16_matvec(x, w, out)
|
| 97 |
+
return out
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def bf16_smallm_matmul(x: torch.Tensor, w: torch.Tensor, *, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 101 |
+
"""Compute `out = x @ w.T` for BF16 `x (M,K)` and small `M`."""
|
| 102 |
+
if out is None:
|
| 103 |
+
out = torch.empty((x.shape[0], w.shape[0]), device=x.device, dtype=torch.bfloat16)
|
| 104 |
+
ops.bf16_smallm_matmul(x, w, out)
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def split_qkv_broadcast_bf16(
|
| 109 |
+
packed: torch.Tensor,
|
| 110 |
+
q_heads: int,
|
| 111 |
+
kv_heads: int,
|
| 112 |
+
v_heads: int,
|
| 113 |
+
head_dim: int,
|
| 114 |
+
*,
|
| 115 |
+
q: Optional[torch.Tensor] = None,
|
| 116 |
+
k: Optional[torch.Tensor] = None,
|
| 117 |
+
v: Optional[torch.Tensor] = None,
|
| 118 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 119 |
+
"""Split packed Q/K/V and broadcast Q/K groups to `v_heads`."""
|
| 120 |
+
rows = packed.shape[0]
|
| 121 |
+
shape = (rows, v_heads, head_dim)
|
| 122 |
+
if q is None:
|
| 123 |
+
q = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 124 |
+
if k is None:
|
| 125 |
+
k = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 126 |
+
if v is None:
|
| 127 |
+
v = torch.empty(shape, device=packed.device, dtype=torch.bfloat16)
|
| 128 |
+
ops.split_qkv_broadcast_bf16(
|
| 129 |
+
packed, q, k, v, int(q_heads), int(kv_heads), int(v_heads), int(head_dim)
|
| 130 |
+
)
|
| 131 |
+
return q, k, v
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def partial_rope_qk_bf16(
|
| 135 |
+
q_in: torch.Tensor,
|
| 136 |
+
k_in: torch.Tensor,
|
| 137 |
+
cos: torch.Tensor,
|
| 138 |
+
sin: torch.Tensor,
|
| 139 |
+
rope_dim: int,
|
| 140 |
+
*,
|
| 141 |
+
q_out: Optional[torch.Tensor] = None,
|
| 142 |
+
k_out: Optional[torch.Tensor] = None,
|
| 143 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 144 |
+
"""Apply split-half RoPE to the first `rope_dim` channels of Q and K."""
|
| 145 |
+
if q_out is None:
|
| 146 |
+
q_out = torch.empty_like(q_in)
|
| 147 |
+
if k_out is None:
|
| 148 |
+
k_out = torch.empty_like(k_in)
|
| 149 |
+
ops.partial_rope_qk_bf16(q_in, k_in, cos, sin, q_out, k_out, int(rope_dim))
|
| 150 |
+
return q_out, k_out
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def gated_delta_prepare_bf16(
|
| 154 |
+
a: torch.Tensor,
|
| 155 |
+
b: torch.Tensor,
|
| 156 |
+
neg_exp_a_log: torch.Tensor,
|
| 157 |
+
dt_bias: torch.Tensor,
|
| 158 |
+
*,
|
| 159 |
+
heads: Optional[int] = None,
|
| 160 |
+
a_stride: Optional[int] = None,
|
| 161 |
+
b_stride: Optional[int] = None,
|
| 162 |
+
g_out: Optional[torch.Tensor] = None,
|
| 163 |
+
beta_out: Optional[torch.Tensor] = None,
|
| 164 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 165 |
+
"""Prepare BF16 Gated DeltaNet `g` and `beta` tensors from projected a/b."""
|
| 166 |
+
if heads is None:
|
| 167 |
+
heads = neg_exp_a_log.shape[0]
|
| 168 |
+
if a_stride is None:
|
| 169 |
+
a_stride = a.shape[1]
|
| 170 |
+
if b_stride is None:
|
| 171 |
+
b_stride = b.shape[1]
|
| 172 |
+
rows = a.shape[0]
|
| 173 |
+
if g_out is None:
|
| 174 |
+
g_out = torch.empty((rows, heads), device=a.device, dtype=torch.bfloat16)
|
| 175 |
+
if beta_out is None:
|
| 176 |
+
beta_out = torch.empty_like(g_out)
|
| 177 |
+
ops.gated_delta_prepare_bf16(
|
| 178 |
+
a, b, neg_exp_a_log, dt_bias, g_out, beta_out, int(a_stride), int(b_stride)
|
| 179 |
+
)
|
| 180 |
+
return g_out, beta_out
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
__all__ = [
|
| 184 |
+
"bf16_matvec",
|
| 185 |
+
"bf16_smallm_matmul",
|
| 186 |
+
"gated_delta_prepare_bf16",
|
| 187 |
+
"partial_rope_qk_bf16",
|
| 188 |
+
"split_qkv_broadcast_bf16",
|
| 189 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/_linear_attention_primitives_cuda_f14c443.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:75c3513618c230a075ce98a46a559da0eb7a7a2cc4b2cf1cdb0841d7b2d9160a
|
| 3 |
+
size 1194600
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _linear_attention_primitives_cuda_f14c443
|
| 3 |
+
ops = torch.ops._linear_attention_primitives_cuda_f14c443
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_linear_attention_primitives_cuda_f14c443::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/linear_attention_primitives/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "linear-attention-primitives",
|
| 3 |
+
"id": "_linear_attention_primitives_cuda_f14c443",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|