liangsu9988 commited on
Commit
e79bfa1
·
verified ·
1 Parent(s): 8035ffd

Promote latest kernel artifacts to main

Browse files
README.md CHANGED
@@ -1,9 +1,23 @@
1
- # flashrt/masked-mha-runtime
2
 
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
 
5
 
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/masked-mha-runtime
 
 
7
 
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # masked-mha-runtime
2
 
3
+ FlashRT native SM110 masked FP16/BF16 MHA for fixed-shape CUDA Graph runtimes.
4
+ It masks padded logits during softmax, supports fused-QKV token strides, and
5
+ keeps caller-owned logits/output buffers stable across graph replay.
6
 
7
+ ```python
8
+ from kernels import get_kernel
9
+ import torch
10
 
11
+ ops = get_kernel("flashrt/masked-mha-runtime", version=1)
12
+ logits = ops.allocate_workspace(q, k)
13
+ out = torch.empty_like(q, memory_format=torch.contiguous_format)
14
+ ops.forward_static(q, k, v, logits=logits, out=out)
15
+ ```
16
+
17
+ Public functions are `allocate_workspace`, `forward_static`, and `forward`.
18
+ Inputs use `(sequence, heads, head_dim)`. The production GROOT gate covers
19
+ DiT `(41, 32, 48)`, ViT/LLM sequence lengths `277/1024`, padded boundaries
20
+ `1025/2048`, FP16 and BF16, fused strides, and bitwise CUDA Graph replay.
21
+
22
+ See [CARD.md](CARD.md) for the complete contract. Source provenance is FlashRT
23
+ commit `24df793f4fa2d50780aea03b644208c6e0cb4162`.
build/torch213-cxx11-cu130-aarch64-linux/__init__.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Allocation-free masked MHA runtime operators."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from ._ops import add_op_namespace_prefix, ops
11
+
12
+
13
+ @torch.library.register_fake(add_op_namespace_prefix("forward_static"))
14
+ def _forward_static_fake(q, k, v, logits, out, scale: float) -> None:
15
+ del scale
16
+ if q.dim() != 3 or k.dim() != 3 or v.dim() != 3:
17
+ raise RuntimeError("q/k/v must have shape (sequence, heads, head_dim)")
18
+ if out.shape != q.shape:
19
+ raise RuntimeError("out must match q")
20
+ if logits.dim() != 3 or logits.shape[:2] != (q.shape[1], q.shape[0]):
21
+ raise RuntimeError("logits must have shape (heads, sequence_q, stride)")
22
+ if logits.shape[2] < k.shape[0]:
23
+ raise RuntimeError("logits stride must cover sequence_kv")
24
+
25
+
26
+ def allocate_workspace(q: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
27
+ """Allocate padded logits scratch once, outside the hot path."""
28
+ stride = (k.shape[0] + 7) // 8 * 8
29
+ return torch.empty(
30
+ (q.shape[1], q.shape[0], stride), device=q.device, dtype=q.dtype
31
+ )
32
+
33
+
34
+ def forward_static(
35
+ q: torch.Tensor,
36
+ k: torch.Tensor,
37
+ v: torch.Tensor,
38
+ *,
39
+ logits: torch.Tensor,
40
+ out: torch.Tensor,
41
+ scale: Optional[float] = None,
42
+ ) -> torch.Tensor:
43
+ """Run MHA without pre-filling padded logits; all buffers are caller-owned."""
44
+ if scale is None:
45
+ scale = 1.0 / math.sqrt(q.shape[-1])
46
+ ops.forward_static(q, k, v, logits, out, float(scale))
47
+ return out
48
+
49
+
50
+ def forward(q, k, v, *, scale: Optional[float] = None):
51
+ """Convenience allocation wrapper; use ``forward_static`` in hot paths."""
52
+ logits = allocate_workspace(q, k)
53
+ out = torch.empty_like(q, memory_format=torch.contiguous_format)
54
+ return forward_static(q, k, v, logits=logits, out=out, scale=scale)
55
+
56
+
57
+ __all__ = ["allocate_workspace", "forward", "forward_static"]
build/torch213-cxx11-cu130-aarch64-linux/_masked_mha_runtime_cuda_8a66d8b.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ab78fa64e990a989882158cd431f0c40d9317e0a5dc5d65abc1f2a2b3b89fa2
3
+ size 306688
build/torch213-cxx11-cu130-aarch64-linux/_ops.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _masked_mha_runtime_cuda_8a66d8b
3
+ ops = torch.ops._masked_mha_runtime_cuda_8a66d8b
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ return f"_masked_mha_runtime_cuda_8a66d8b::{op_name}"
build/torch213-cxx11-cu130-aarch64-linux/masked_mha_runtime/__init__.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ def _import_from_path(file_path: Path):
7
+ path_hash = '{:x}'.format(ctypes.c_size_t(hash(file_path.absolute())).value)
8
+ spec = importlib.util.spec_from_file_location(path_hash, file_path)
9
+ module = importlib.util.module_from_spec(spec)
10
+ sys.modules[path_hash] = module
11
+ spec.loader.exec_module(module)
12
+ return module
13
+
14
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / '__init__.py')))
build/torch213-cxx11-cu130-aarch64-linux/metadata.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "masked-mha-runtime",
3
+ "id": "_masked_mha_runtime_cuda_8a66d8b",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0a"
11
+ ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "8t7pBfUTc7GVM5lQQwW1gcEI91L1P441YxQJZF7rLB8=",
17
+ "_masked_mha_runtime_cuda_8a66d8b.abi3.so": "CrePpk6ZCpiYghWM1DHwxA2TF+Cl3F1lq8HyorO4n6I=",
18
+ "_ops.py": "NBt1+0PV72VhTKtuvZwFvN1Eo6tuwbTJRoDke3DVqEI=",
19
+ "masked_mha_runtime/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
+ }
21
+ },
22
+ "provenance": {
23
+ "kernel": {
24
+ "sha": "8a66d8b",
25
+ "dirty": false
26
+ }
27
+ }
28
+ }