Promote latest kernel artifacts to main
Browse files- benchmarks/RESULTS.md +3 -24
- build/torch213-cxx11-cu130-aarch64-linux/__init__.py +34 -1
- build/torch213-cxx11-cu130-aarch64-linux/_ops.py +3 -3
- build/torch213-cxx11-cu130-aarch64-linux/{_masked_mha_runtime_cuda_8a66d8b.abi3.so → masked_mha_runtime_source_test.abi3.so} +2 -2
- build/torch213-cxx11-cu130-aarch64-linux/metadata.json +10 -6
benchmarks/RESULTS.md
CHANGED
|
@@ -1,26 +1,5 @@
|
|
| 1 |
# Results
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
- Runtime: PyTorch 2.13.0 + CUDA 13.0
|
| 7 |
-
- Variant: `torch213-cxx11-cu130-aarch64-linux`
|
| 8 |
-
- Native source: FlashRT `24df793f4fa2d50780aea03b644208c6e0cb4162`
|
| 9 |
-
|
| 10 |
-
The installed-artifact full gate passed 6/6. Representative BF16 rows:
|
| 11 |
-
|
| 12 |
-
| Shape | p99 absolute error | Cosine | CUDA Graph replay |
|
| 13 |
-
|---|---:|---:|---|
|
| 14 |
-
| `sk=1024` | 0.000452 | 0.99999362 | bitwise |
|
| 15 |
-
| `sk=1025` | 0.000452 | 0.99999356 | bitwise |
|
| 16 |
-
| `sk=2048` | 0.000174 | 0.99999666 | bitwise |
|
| 17 |
-
|
| 18 |
-
The Hub Tensor wrapper and the raw registered native op were measured in the
|
| 19 |
-
same process with CUDA events. Direct wrapper/raw latency was
|
| 20 |
-
`24.7932/24.7544 us` (`1.0016x`). With the caller-owned static buffers used by
|
| 21 |
-
the production CUDA Graph path, wrapper/raw latency was
|
| 22 |
-
`19.0034/19.0034 us` (`1.0000x`).
|
| 23 |
-
|
| 24 |
-
These numbers establish package parity with the FlashRT native symbol. They
|
| 25 |
-
are not presented as a universal speedup over PyTorch SDPA; backend selection
|
| 26 |
-
must still be profiled for the target sequence and padding profile.
|
|
|
|
| 1 |
# Results
|
| 2 |
|
| 3 |
+
Pending the clean SM110 installed-artifact benchmark. No speedup claim is
|
| 4 |
+
published until the package is measured against both PyTorch SDPA and the
|
| 5 |
+
FlashRT native symbol on the same Thor host.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch213-cxx11-cu130-aarch64-linux/__init__.py
CHANGED
|
@@ -23,6 +23,17 @@ def _forward_static_fake(q, k, v, logits, out, scale: float) -> None:
|
|
| 23 |
raise RuntimeError("logits stride must cover sequence_kv")
|
| 24 |
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
def allocate_workspace(q: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
|
| 27 |
"""Allocate padded logits scratch once, outside the hot path."""
|
| 28 |
stride = (k.shape[0] + 7) // 8 * 8
|
|
@@ -54,4 +65,26 @@ def forward(q, k, v, *, scale: Optional[float] = None):
|
|
| 54 |
return forward_static(q, k, v, logits=logits, out=out, scale=scale)
|
| 55 |
|
| 56 |
|
| 57 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
raise RuntimeError("logits stride must cover sequence_kv")
|
| 24 |
|
| 25 |
|
| 26 |
+
@torch.library.register_fake(add_op_namespace_prefix("forward_seqused_static"))
|
| 27 |
+
def _forward_seqused_static_fake(q, k, v, valid_k, logits, out, scale: float) -> None:
|
| 28 |
+
del scale
|
| 29 |
+
if q.dim() != 3 or k.dim() != 2 or v.dim() != 2:
|
| 30 |
+
raise RuntimeError("q must be (sequence, heads, head_dim); k/v must be (max_sequence, head_dim)")
|
| 31 |
+
if valid_k.numel() != 1 or valid_k.dtype != torch.int32:
|
| 32 |
+
raise RuntimeError("valid_k must be an int32 scalar tensor")
|
| 33 |
+
if out.shape != q.shape or logits.shape != (q.shape[0] * q.shape[1], k.shape[0]):
|
| 34 |
+
raise RuntimeError("caller-owned output shapes do not match q/k")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
def allocate_workspace(q: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
|
| 38 |
"""Allocate padded logits scratch once, outside the hot path."""
|
| 39 |
stride = (k.shape[0] + 7) // 8 * 8
|
|
|
|
| 65 |
return forward_static(q, k, v, logits=logits, out=out, scale=scale)
|
| 66 |
|
| 67 |
|
| 68 |
+
def forward_seqused_static(
|
| 69 |
+
q: torch.Tensor,
|
| 70 |
+
k: torch.Tensor,
|
| 71 |
+
v: torch.Tensor,
|
| 72 |
+
valid_k: torch.Tensor,
|
| 73 |
+
*,
|
| 74 |
+
logits: torch.Tensor,
|
| 75 |
+
out: torch.Tensor,
|
| 76 |
+
scale: Optional[float] = None,
|
| 77 |
+
) -> torch.Tensor:
|
| 78 |
+
"""Run FP16 shared-KV attention with a graph-resident valid-key count."""
|
| 79 |
+
if scale is None:
|
| 80 |
+
scale = 1.0 / math.sqrt(q.shape[-1])
|
| 81 |
+
ops.forward_seqused_static(q, k, v, valid_k, logits, out, float(scale))
|
| 82 |
+
return out
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
__all__ = [
|
| 86 |
+
"allocate_workspace",
|
| 87 |
+
"forward",
|
| 88 |
+
"forward_static",
|
| 89 |
+
"forward_seqused_static",
|
| 90 |
+
]
|
build/torch213-cxx11-cu130-aarch64-linux/_ops.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import masked_mha_runtime_source_test
|
| 3 |
+
ops = torch.ops.masked_mha_runtime_source_test
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
return f"masked_mha_runtime_source_test::{op_name}"
|
build/torch213-cxx11-cu130-aarch64-linux/{_masked_mha_runtime_cuda_8a66d8b.abi3.so → masked_mha_runtime_source_test.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3b5458ef4502155dc761c2e6742e7d2d7ee2904f85c3e0a6e6e095acc6b64ad0
|
| 3 |
+
size 379728
|
build/torch213-cxx11-cu130-aarch64-linux/metadata.json
CHANGED
|
@@ -1,28 +1,32 @@
|
|
| 1 |
{
|
| 2 |
"name": "masked-mha-runtime",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
| 7 |
"backend": {
|
| 8 |
"type": "cuda",
|
| 9 |
"archs": [
|
| 10 |
-
"11.
|
| 11 |
]
|
| 12 |
},
|
| 13 |
"digest": {
|
| 14 |
"algorithm": "sha256",
|
| 15 |
"files": {
|
| 16 |
-
"__init__.py": "
|
| 17 |
-
"
|
| 18 |
-
"_ops.py": "
|
| 19 |
"masked_mha_runtime/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
| 23 |
"kernel": {
|
| 24 |
-
"sha": "
|
| 25 |
"dirty": false
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
}
|
| 27 |
}
|
| 28 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "masked-mha-runtime",
|
| 3 |
+
"id": "masked_mha_runtime_source_test",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
| 7 |
"backend": {
|
| 8 |
"type": "cuda",
|
| 9 |
"archs": [
|
| 10 |
+
"11.0"
|
| 11 |
]
|
| 12 |
},
|
| 13 |
"digest": {
|
| 14 |
"algorithm": "sha256",
|
| 15 |
"files": {
|
| 16 |
+
"__init__.py": "sZLa3RWzOCoLFNml3p4SGOBbeTUtLYDp2tmIL0FlToo=",
|
| 17 |
+
"masked_mha_runtime_source_test.abi3.so": "O1RY70UCFV3HYcLmdC59LX7ikE+Fw+Cm5uCVrMa2StA=",
|
| 18 |
+
"_ops.py": "dXHlvnzts0eDL5fgM/I0tMNAsCB3uhZT03SIVpj4bvU=",
|
| 19 |
"masked_mha_runtime/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
| 23 |
"kernel": {
|
| 24 |
+
"sha": "3a505c0",
|
| 25 |
"dirty": false
|
| 26 |
+
},
|
| 27 |
+
"validation": {
|
| 28 |
+
"torch": "2.13.0+cu130",
|
| 29 |
+
"cuda": "13.0"
|
| 30 |
}
|
| 31 |
}
|
| 32 |
}
|