liangsu9988 commited on
Commit
c1d8b2f
·
verified ·
1 Parent(s): 510a09c

Promote latest kernel artifacts to main

Browse files
benchmarks/RESULTS.md CHANGED
@@ -1,26 +1,5 @@
1
  # Results
2
 
3
- ## NVIDIA Thor installed artifact
4
-
5
- - Device: NVIDIA Thor, SM110, aarch64
6
- - Runtime: PyTorch 2.13.0 + CUDA 13.0
7
- - Variant: `torch213-cxx11-cu130-aarch64-linux`
8
- - Native source: FlashRT `24df793f4fa2d50780aea03b644208c6e0cb4162`
9
-
10
- The installed-artifact full gate passed 6/6. Representative BF16 rows:
11
-
12
- | Shape | p99 absolute error | Cosine | CUDA Graph replay |
13
- |---|---:|---:|---|
14
- | `sk=1024` | 0.000452 | 0.99999362 | bitwise |
15
- | `sk=1025` | 0.000452 | 0.99999356 | bitwise |
16
- | `sk=2048` | 0.000174 | 0.99999666 | bitwise |
17
-
18
- The Hub Tensor wrapper and the raw registered native op were measured in the
19
- same process with CUDA events. Direct wrapper/raw latency was
20
- `24.7932/24.7544 us` (`1.0016x`). With the caller-owned static buffers used by
21
- the production CUDA Graph path, wrapper/raw latency was
22
- `19.0034/19.0034 us` (`1.0000x`).
23
-
24
- These numbers establish package parity with the FlashRT native symbol. They
25
- are not presented as a universal speedup over PyTorch SDPA; backend selection
26
- must still be profiled for the target sequence and padding profile.
 
1
  # Results
2
 
3
+ Pending the clean SM110 installed-artifact benchmark. No speedup claim is
4
+ published until the package is measured against both PyTorch SDPA and the
5
+ FlashRT native symbol on the same Thor host.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch213-cxx11-cu130-aarch64-linux/__init__.py CHANGED
@@ -23,6 +23,17 @@ def _forward_static_fake(q, k, v, logits, out, scale: float) -> None:
23
  raise RuntimeError("logits stride must cover sequence_kv")
24
 
25
 
 
 
 
 
 
 
 
 
 
 
 
26
  def allocate_workspace(q: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
27
  """Allocate padded logits scratch once, outside the hot path."""
28
  stride = (k.shape[0] + 7) // 8 * 8
@@ -54,4 +65,26 @@ def forward(q, k, v, *, scale: Optional[float] = None):
54
  return forward_static(q, k, v, logits=logits, out=out, scale=scale)
55
 
56
 
57
- __all__ = ["allocate_workspace", "forward", "forward_static"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  raise RuntimeError("logits stride must cover sequence_kv")
24
 
25
 
26
+ @torch.library.register_fake(add_op_namespace_prefix("forward_seqused_static"))
27
+ def _forward_seqused_static_fake(q, k, v, valid_k, logits, out, scale: float) -> None:
28
+ del scale
29
+ if q.dim() != 3 or k.dim() != 2 or v.dim() != 2:
30
+ raise RuntimeError("q must be (sequence, heads, head_dim); k/v must be (max_sequence, head_dim)")
31
+ if valid_k.numel() != 1 or valid_k.dtype != torch.int32:
32
+ raise RuntimeError("valid_k must be an int32 scalar tensor")
33
+ if out.shape != q.shape or logits.shape != (q.shape[0] * q.shape[1], k.shape[0]):
34
+ raise RuntimeError("caller-owned output shapes do not match q/k")
35
+
36
+
37
  def allocate_workspace(q: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
38
  """Allocate padded logits scratch once, outside the hot path."""
39
  stride = (k.shape[0] + 7) // 8 * 8
 
65
  return forward_static(q, k, v, logits=logits, out=out, scale=scale)
66
 
67
 
68
+ def forward_seqused_static(
69
+ q: torch.Tensor,
70
+ k: torch.Tensor,
71
+ v: torch.Tensor,
72
+ valid_k: torch.Tensor,
73
+ *,
74
+ logits: torch.Tensor,
75
+ out: torch.Tensor,
76
+ scale: Optional[float] = None,
77
+ ) -> torch.Tensor:
78
+ """Run FP16 shared-KV attention with a graph-resident valid-key count."""
79
+ if scale is None:
80
+ scale = 1.0 / math.sqrt(q.shape[-1])
81
+ ops.forward_seqused_static(q, k, v, valid_k, logits, out, float(scale))
82
+ return out
83
+
84
+
85
+ __all__ = [
86
+ "allocate_workspace",
87
+ "forward",
88
+ "forward_static",
89
+ "forward_seqused_static",
90
+ ]
build/torch213-cxx11-cu130-aarch64-linux/_ops.py CHANGED
@@ -1,6 +1,6 @@
1
  import torch
2
- from . import _masked_mha_runtime_cuda_8a66d8b
3
- ops = torch.ops._masked_mha_runtime_cuda_8a66d8b
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
- return f"_masked_mha_runtime_cuda_8a66d8b::{op_name}"
 
1
  import torch
2
+ from . import masked_mha_runtime_source_test
3
+ ops = torch.ops.masked_mha_runtime_source_test
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
+ return f"masked_mha_runtime_source_test::{op_name}"
build/torch213-cxx11-cu130-aarch64-linux/{_masked_mha_runtime_cuda_8a66d8b.abi3.so → masked_mha_runtime_source_test.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6d345bec0ac01d2d2e7bccc782ad4043a89d9151ef89e06e40f76bec6a0ba924
3
- size 309792
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b5458ef4502155dc761c2e6742e7d2d7ee2904f85c3e0a6e6e095acc6b64ad0
3
+ size 379728
build/torch213-cxx11-cu130-aarch64-linux/metadata.json CHANGED
@@ -1,28 +1,32 @@
1
  {
2
  "name": "masked-mha-runtime",
3
- "id": "_masked_mha_runtime_cuda_8a66d8b",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "11.0a"
11
  ]
12
  },
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "8t7pBfUTc7GVM5lQQwW1gcEI91L1P441YxQJZF7rLB8=",
17
- "_masked_mha_runtime_cuda_8a66d8b.abi3.so": "bTRb7ArAHS0ue8zHgq1AQ6idkVHvieBuQPdr7GoLqSQ=",
18
- "_ops.py": "NBt1+0PV72VhTKtuvZwFvN1Eo6tuwbTJRoDke3DVqEI=",
19
  "masked_mha_runtime/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
  }
21
  },
22
  "provenance": {
23
  "kernel": {
24
- "sha": "8a66d8b",
25
  "dirty": false
 
 
 
 
26
  }
27
  }
28
  }
 
1
  {
2
  "name": "masked-mha-runtime",
3
+ "id": "masked_mha_runtime_source_test",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
+ "11.0"
11
  ]
12
  },
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "sZLa3RWzOCoLFNml3p4SGOBbeTUtLYDp2tmIL0FlToo=",
17
+ "masked_mha_runtime_source_test.abi3.so": "O1RY70UCFV3HYcLmdC59LX7ikE+Fw+Cm5uCVrMa2StA=",
18
+ "_ops.py": "dXHlvnzts0eDL5fgM/I0tMNAsCB3uhZT03SIVpj4bvU=",
19
  "masked_mha_runtime/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
  }
21
  },
22
  "provenance": {
23
  "kernel": {
24
+ "sha": "3a505c0",
25
  "dirty": false
26
+ },
27
+ "validation": {
28
+ "torch": "2.13.0+cu130",
29
+ "cuda": "13.0"
30
  }
31
  }
32
  }