Promote latest kernel artifacts to main
Browse files- README.md +20 -6
- build/torch213-cxx11-cu130-aarch64-linux/__init__.py +57 -0
- build/torch213-cxx11-cu130-aarch64-linux/_masked_mha_runtime_cuda_8a66d8b.abi3.so +3 -0
- build/torch213-cxx11-cu130-aarch64-linux/_ops.py +6 -0
- build/torch213-cxx11-cu130-aarch64-linux/masked_mha_runtime/__init__.py +14 -0
- build/torch213-cxx11-cu130-aarch64-linux/metadata.json +28 -0
README.md
CHANGED
|
@@ -1,9 +1,23 @@
|
|
| 1 |
-
#
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
|
|
|
| 5 |
|
| 6 |
-
|
|
|
|
|
|
|
| 7 |
|
| 8 |
-
|
| 9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# masked-mha-runtime
|
| 2 |
|
| 3 |
+
FlashRT native SM110 masked FP16/BF16 MHA for fixed-shape CUDA Graph runtimes.
|
| 4 |
+
It masks padded logits during softmax, supports fused-QKV token strides, and
|
| 5 |
+
keeps caller-owned logits/output buffers stable across graph replay.
|
| 6 |
|
| 7 |
+
```python
|
| 8 |
+
from kernels import get_kernel
|
| 9 |
+
import torch
|
| 10 |
|
| 11 |
+
ops = get_kernel("flashrt/masked-mha-runtime", version=1)
|
| 12 |
+
logits = ops.allocate_workspace(q, k)
|
| 13 |
+
out = torch.empty_like(q, memory_format=torch.contiguous_format)
|
| 14 |
+
ops.forward_static(q, k, v, logits=logits, out=out)
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
Public functions are `allocate_workspace`, `forward_static`, and `forward`.
|
| 18 |
+
Inputs use `(sequence, heads, head_dim)`. The production GROOT gate covers
|
| 19 |
+
DiT `(41, 32, 48)`, ViT/LLM sequence lengths `277/1024`, padded boundaries
|
| 20 |
+
`1025/2048`, FP16 and BF16, fused strides, and bitwise CUDA Graph replay.
|
| 21 |
+
|
| 22 |
+
See [CARD.md](CARD.md) for the complete contract. Source provenance is FlashRT
|
| 23 |
+
commit `24df793f4fa2d50780aea03b644208c6e0cb4162`.
|
build/torch213-cxx11-cu130-aarch64-linux/__init__.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Allocation-free masked MHA runtime operators."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@torch.library.register_fake(add_op_namespace_prefix("forward_static"))
|
| 14 |
+
def _forward_static_fake(q, k, v, logits, out, scale: float) -> None:
|
| 15 |
+
del scale
|
| 16 |
+
if q.dim() != 3 or k.dim() != 3 or v.dim() != 3:
|
| 17 |
+
raise RuntimeError("q/k/v must have shape (sequence, heads, head_dim)")
|
| 18 |
+
if out.shape != q.shape:
|
| 19 |
+
raise RuntimeError("out must match q")
|
| 20 |
+
if logits.dim() != 3 or logits.shape[:2] != (q.shape[1], q.shape[0]):
|
| 21 |
+
raise RuntimeError("logits must have shape (heads, sequence_q, stride)")
|
| 22 |
+
if logits.shape[2] < k.shape[0]:
|
| 23 |
+
raise RuntimeError("logits stride must cover sequence_kv")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def allocate_workspace(q: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
|
| 27 |
+
"""Allocate padded logits scratch once, outside the hot path."""
|
| 28 |
+
stride = (k.shape[0] + 7) // 8 * 8
|
| 29 |
+
return torch.empty(
|
| 30 |
+
(q.shape[1], q.shape[0], stride), device=q.device, dtype=q.dtype
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def forward_static(
|
| 35 |
+
q: torch.Tensor,
|
| 36 |
+
k: torch.Tensor,
|
| 37 |
+
v: torch.Tensor,
|
| 38 |
+
*,
|
| 39 |
+
logits: torch.Tensor,
|
| 40 |
+
out: torch.Tensor,
|
| 41 |
+
scale: Optional[float] = None,
|
| 42 |
+
) -> torch.Tensor:
|
| 43 |
+
"""Run MHA without pre-filling padded logits; all buffers are caller-owned."""
|
| 44 |
+
if scale is None:
|
| 45 |
+
scale = 1.0 / math.sqrt(q.shape[-1])
|
| 46 |
+
ops.forward_static(q, k, v, logits, out, float(scale))
|
| 47 |
+
return out
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def forward(q, k, v, *, scale: Optional[float] = None):
|
| 51 |
+
"""Convenience allocation wrapper; use ``forward_static`` in hot paths."""
|
| 52 |
+
logits = allocate_workspace(q, k)
|
| 53 |
+
out = torch.empty_like(q, memory_format=torch.contiguous_format)
|
| 54 |
+
return forward_static(q, k, v, logits=logits, out=out, scale=scale)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
__all__ = ["allocate_workspace", "forward", "forward_static"]
|
build/torch213-cxx11-cu130-aarch64-linux/_masked_mha_runtime_cuda_8a66d8b.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6d345bec0ac01d2d2e7bccc782ad4043a89d9151ef89e06e40f76bec6a0ba924
|
| 3 |
+
size 309792
|
build/torch213-cxx11-cu130-aarch64-linux/_ops.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _masked_mha_runtime_cuda_8a66d8b
|
| 3 |
+
ops = torch.ops._masked_mha_runtime_cuda_8a66d8b
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
return f"_masked_mha_runtime_cuda_8a66d8b::{op_name}"
|
build/torch213-cxx11-cu130-aarch64-linux/masked_mha_runtime/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
def _import_from_path(file_path: Path):
|
| 7 |
+
path_hash = '{:x}'.format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 8 |
+
spec = importlib.util.spec_from_file_location(path_hash, file_path)
|
| 9 |
+
module = importlib.util.module_from_spec(spec)
|
| 10 |
+
sys.modules[path_hash] = module
|
| 11 |
+
spec.loader.exec_module(module)
|
| 12 |
+
return module
|
| 13 |
+
|
| 14 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / '__init__.py')))
|
build/torch213-cxx11-cu130-aarch64-linux/metadata.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "masked-mha-runtime",
|
| 3 |
+
"id": "_masked_mha_runtime_cuda_8a66d8b",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"11.0a"
|
| 11 |
+
]
|
| 12 |
+
},
|
| 13 |
+
"digest": {
|
| 14 |
+
"algorithm": "sha256",
|
| 15 |
+
"files": {
|
| 16 |
+
"__init__.py": "8t7pBfUTc7GVM5lQQwW1gcEI91L1P441YxQJZF7rLB8=",
|
| 17 |
+
"_masked_mha_runtime_cuda_8a66d8b.abi3.so": "bTRb7ArAHS0ue8zHgq1AQ6idkVHvieBuQPdr7GoLqSQ=",
|
| 18 |
+
"_ops.py": "NBt1+0PV72VhTKtuvZwFvN1Eo6tuwbTJRoDke3DVqEI=",
|
| 19 |
+
"masked_mha_runtime/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
|
| 20 |
+
}
|
| 21 |
+
},
|
| 22 |
+
"provenance": {
|
| 23 |
+
"kernel": {
|
| 24 |
+
"sha": "8a66d8b",
|
| 25 |
+
"dirty": false
|
| 26 |
+
}
|
| 27 |
+
}
|
| 28 |
+
}
|