liangsu9988 commited on
Commit
40197ac
·
verified ·
1 Parent(s): 523c958

Promote latest kernel artifacts to main

Browse files
Files changed (49) hide show
  1. .gitattributes +3 -35
  2. README.md +0 -9
  3. benchmarks/benchmark.py +535 -0
  4. benchmarks/benchmark_kernel.py +99 -0
  5. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +227 -0
  6. build/torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +3 -0
  7. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
  8. build/torch211-cxx11-cu128-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  9. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
  10. build/torch211-cxx11-cu130-aarch64-linux/__init__.py +227 -0
  11. build/torch211-cxx11-cu130-aarch64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +3 -0
  12. build/torch211-cxx11-cu130-aarch64-linux/_ops.py +6 -0
  13. build/torch211-cxx11-cu130-aarch64-linux/flashrt_fp8_ffn/__init__.py +14 -0
  14. build/torch211-cxx11-cu130-aarch64-linux/metadata.json +14 -0
  15. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +227 -0
  16. build/torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +3 -0
  17. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  18. build/torch211-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  19. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
  20. build/torch211-cxx11-rocm71-x86_64-linux/__init__.py +227 -0
  21. build/torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so +3 -0
  22. build/torch211-cxx11-rocm71-x86_64-linux/_ops.py +9 -0
  23. build/torch211-cxx11-rocm71-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  24. build/torch211-cxx11-rocm71-x86_64-linux/metadata.json +13 -0
  25. build/torch211-cxx11-rocm72-x86_64-linux/__init__.py +227 -0
  26. build/torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so +3 -0
  27. build/torch211-cxx11-rocm72-x86_64-linux/_ops.py +9 -0
  28. build/torch211-cxx11-rocm72-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  29. build/torch211-cxx11-rocm72-x86_64-linux/metadata.json +13 -0
  30. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +227 -0
  31. build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +3 -0
  32. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  33. build/torch212-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  34. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
  35. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +227 -0
  36. build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +3 -0
  37. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
  38. build/torch212-cxx11-cu132-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  39. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
  40. build/torch212-cxx11-rocm71-x86_64-linux/__init__.py +227 -0
  41. build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so +3 -0
  42. build/torch212-cxx11-rocm71-x86_64-linux/_ops.py +9 -0
  43. build/torch212-cxx11-rocm71-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  44. build/torch212-cxx11-rocm71-x86_64-linux/metadata.json +13 -0
  45. build/torch212-cxx11-rocm72-x86_64-linux/__init__.py +227 -0
  46. build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so +3 -0
  47. build/torch212-cxx11-rocm72-x86_64-linux/_ops.py +9 -0
  48. build/torch212-cxx11-rocm72-x86_64-linux/flashrt_fp8_ffn/__init__.py +26 -0
  49. build/torch212-cxx11-rocm72-x86_64-linux/metadata.json +13 -0
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.so filter=lfs diff=lfs merge=lfs -text
2
+ *.pyd filter=lfs diff=lfs merge=lfs -text
3
+ *.dylib filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
README.md DELETED
@@ -1,9 +0,0 @@
1
- # flashrt/flashrt-fp8-ffn
2
-
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
-
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-fp8-ffn
7
-
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
benchmarks/benchmark.py ADDED
@@ -0,0 +1,535 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Benchmark flashrt-fp8-ffn against PyTorch eager/compile references."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import ctypes
8
+ import ctypes.util
9
+ import importlib
10
+ import json
11
+ import math
12
+ import os
13
+ import sys
14
+ from dataclasses import asdict, dataclass
15
+ from pathlib import Path
16
+
17
+ import torch
18
+
19
+
20
+ ROOT = Path(__file__).resolve().parents[2]
21
+ PACKAGE = ROOT / "flashrt-fp8-ffn"
22
+ REGISTRATION_INCLUDE = (
23
+ ROOT.parent
24
+ / "kernels"
25
+ / "kernel-builder"
26
+ / "src"
27
+ / "pyproject"
28
+ / "templates"
29
+ / "torch"
30
+ )
31
+
32
+
33
+ SHAPES = {
34
+ # PI0.5 decoder chunks. Production default is 10 denoising steps.
35
+ "pi05_decoder_ffn_m1": (1, 1024, 4096, 1024, 18),
36
+ "pi05_decoder_ffn_m8": (8, 1024, 4096, 1024, 18),
37
+ "pi05_decoder_ffn_m10": (10, 1024, 4096, 1024, 18),
38
+ "pi05_decoder_ffn_m16": (16, 1024, 4096, 1024, 18),
39
+ # Backward-compatible headline alias.
40
+ "pi05_decoder_ffn": (10, 1024, 4096, 1024, 18),
41
+ # PI0.5 SigLIP-L FFN. One view is 256 visual tokens.
42
+ "pi05_vision_ffn_1view": (256, 1152, 4304, 1152, 27),
43
+ "pi05_vision_ffn_2view": (512, 1152, 4304, 1152, 27),
44
+ "pi05_vision_ffn_3view": (768, 1152, 4304, 1152, 27),
45
+ # GROOT/Qwen3-VL ViT FFN.
46
+ "groot_vit_ffn_1view": (256, 1024, 4096, 1024, 24),
47
+ "groot_vit_ffn_2view": (512, 1024, 4096, 1024, 24),
48
+ "groot_vit_ffn_4view": (1024, 1024, 4096, 1024, 24),
49
+ # GROOT DeepStack merger. Two-view ViT taps produce 128 merged tokens.
50
+ "groot_deepstack_merge_2view": (128, 4096, 4096, 2048, 3),
51
+ # GROOT VL self-attention FFN. Sequence length changes with vision/text mix.
52
+ "groot_vl_self_attn_ffn_seq512": (512, 2048, 8192, 2048, 4),
53
+ "groot_vl_self_attn_ffn_seq1024": (1024, 2048, 8192, 2048, 4),
54
+ "groot_vl_self_attn_ffn_seq2520": (2520, 2048, 8192, 2048, 4),
55
+ # Backward-compatible headline alias.
56
+ "groot_vl_self_attn_ffn": (1024, 2048, 8192, 2048, 4),
57
+ # GROOT action DiT GELU FFN. This is exact GELU shape, but the production
58
+ # path currently uses BF16 GEMMs; report it as a shape fit, not a deployed
59
+ # FP8 action-head claim until model wiring is done.
60
+ "groot_action_dit_ffn": (41, 1536, 6144, 1536, 32),
61
+ }
62
+
63
+ SHAPE_GROUPS = {
64
+ "headline": [
65
+ "pi05_decoder_ffn_m10",
66
+ "pi05_vision_ffn_2view",
67
+ "groot_vit_ffn_2view",
68
+ "groot_vl_self_attn_ffn_seq1024",
69
+ ],
70
+ "pi05": [
71
+ "pi05_decoder_ffn_m1",
72
+ "pi05_decoder_ffn_m8",
73
+ "pi05_decoder_ffn_m10",
74
+ "pi05_decoder_ffn_m16",
75
+ "pi05_vision_ffn_1view",
76
+ "pi05_vision_ffn_2view",
77
+ "pi05_vision_ffn_3view",
78
+ ],
79
+ "groot": [
80
+ "groot_vit_ffn_1view",
81
+ "groot_vit_ffn_2view",
82
+ "groot_vit_ffn_4view",
83
+ "groot_deepstack_merge_2view",
84
+ "groot_vl_self_attn_ffn_seq512",
85
+ "groot_vl_self_attn_ffn_seq1024",
86
+ "groot_vl_self_attn_ffn_seq2520",
87
+ "groot_action_dit_ffn",
88
+ ],
89
+ }
90
+ SHAPE_GROUPS["all"] = SHAPE_GROUPS["pi05"] + SHAPE_GROUPS["groot"]
91
+
92
+
93
+ @dataclass
94
+ class Result:
95
+ shape: str
96
+ M: int
97
+ K: int
98
+ H: int
99
+ N: int
100
+ layers: int
101
+ flashrt_us: float
102
+ torch_eager_us: float
103
+ torch_compile_us: float | None
104
+ speedup_vs_eager: float
105
+ speedup_vs_compile: float | None
106
+ compile_status: str
107
+ max_abs: float
108
+ p99_abs: float
109
+ p99_rel_floor1: float
110
+ max_rel_floor1: float
111
+ status: str
112
+
113
+
114
+ class SourceOps:
115
+ def __init__(self, namespace: str) -> None:
116
+ self._ops = getattr(torch.ops, namespace)
117
+
118
+ def fp8_gelu_mlp_bf16(
119
+ self,
120
+ x,
121
+ up_w,
122
+ up_b,
123
+ down_w,
124
+ down_b,
125
+ x_scale,
126
+ up_w_scale,
127
+ hidden_scale,
128
+ down_w_scale,
129
+ hidden=None,
130
+ hidden_fp8=None,
131
+ out=None,
132
+ ):
133
+ if hidden is None:
134
+ hidden = torch.empty((x.shape[0], up_w.shape[0]), device=x.device, dtype=torch.bfloat16)
135
+ if hidden_fp8 is None:
136
+ hidden_fp8 = torch.empty_like(hidden, dtype=fp8_dtype())
137
+ if out is None:
138
+ out = torch.empty((x.shape[0], down_w.shape[0]), device=x.device, dtype=torch.bfloat16)
139
+ self._ops.fp8_gelu_mlp_bf16(
140
+ x,
141
+ up_w,
142
+ up_b,
143
+ down_w,
144
+ down_b,
145
+ x_scale,
146
+ up_w_scale,
147
+ hidden_scale,
148
+ down_w_scale,
149
+ hidden,
150
+ hidden_fp8,
151
+ out,
152
+ )
153
+ return out
154
+
155
+
156
+ def _preload_cublaslt() -> None:
157
+ for parent in Path(torch.__file__).resolve().parents:
158
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
159
+ if candidate.exists():
160
+ ctypes.CDLL(str(candidate), mode=ctypes.RTLD_GLOBAL)
161
+ return
162
+ library = ctypes.util.find_library("cublasLt")
163
+ if library:
164
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
165
+
166
+
167
+ def _current_arch_list() -> str:
168
+ major, minor = torch.cuda.get_device_capability(0)
169
+ return f"{major}.{minor}"
170
+
171
+
172
+ def load_source_ops() -> SourceOps:
173
+ from torch.utils.cpp_extension import load
174
+
175
+ if not REGISTRATION_INCLUDE.is_dir():
176
+ raise RuntimeError(f"missing kernel-builder registration include: {REGISTRATION_INCLUDE}")
177
+ _preload_cublaslt()
178
+ os.environ.setdefault("TORCH_CUDA_ARCH_LIST", _current_arch_list())
179
+ namespace = "flashrt_fp8_ffn_benchmark"
180
+ load(
181
+ name=namespace,
182
+ sources=[
183
+ str(PACKAGE / "torch-ext" / "torch_binding.cpp"),
184
+ str(PACKAGE / "csrc" / "fp8_ffn.cu"),
185
+ ],
186
+ extra_include_paths=[str(PACKAGE / "csrc"), str(REGISTRATION_INCLUDE)],
187
+ extra_cflags=["-O3", "-DCUDA_KERNEL"],
188
+ extra_cuda_cflags=["-O3", "--expt-relaxed-constexpr", "-DCUDA_KERNEL"],
189
+ verbose=False,
190
+ )
191
+ return SourceOps(namespace)
192
+
193
+
194
+ def load_installed_ops(artifact: str | None):
195
+ if artifact:
196
+ sys.path.insert(0, artifact)
197
+ try:
198
+ return importlib.import_module("flashrt_fp8_ffn")
199
+ finally:
200
+ if artifact:
201
+ sys.path.remove(artifact)
202
+
203
+
204
+ def load_hub_ops(repo_id: str, version: int):
205
+ from kernels import get_kernel
206
+
207
+ return get_kernel(repo_id, version=version)
208
+
209
+
210
+ def fp8_dtype() -> torch.dtype:
211
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
212
+ return torch.float8_e4m3fnuz
213
+ return torch.float8_e4m3fn
214
+
215
+
216
+ def fp8_max() -> float:
217
+ return 240.0 if torch.version.hip is not None else 448.0
218
+
219
+
220
+ def quantize_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
221
+ limit = fp8_max()
222
+ return torch.clamp(x.float() / scale.float(), -limit, limit).to(fp8_dtype())
223
+
224
+
225
+ def dequant_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
226
+ return x.float() * scale.float()
227
+
228
+
229
+ def compiler_disable(fn):
230
+ compiler = getattr(torch, "compiler", None)
231
+ if compiler is not None and hasattr(compiler, "disable"):
232
+ return compiler.disable(fn)
233
+ return torch._dynamo.disable(fn)
234
+
235
+
236
+ def gelu_quantize_fp8_boundary(
237
+ hidden: torch.Tensor, bias: torch.Tensor, scale: torch.Tensor
238
+ ) -> torch.Tensor:
239
+ hidden = torch.nn.functional.gelu(
240
+ hidden.float() + bias.float(), approximate="tanh"
241
+ )
242
+ return quantize_fp8(hidden, scale)
243
+
244
+
245
+ def bf16_bias_add_boundary(out: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
246
+ return (out.float() + bias.float()).to(torch.bfloat16)
247
+
248
+
249
+ stable_gelu_quantize_fp8 = compiler_disable(gelu_quantize_fp8_boundary)
250
+ stable_bf16_bias_add = compiler_disable(bf16_bias_add_boundary)
251
+
252
+
253
+ def torch_mlp(x, up_w, up_b, down_w, down_b, x_s, up_s, hid_s, dn_s):
254
+ hidden = (dequant_fp8(x, x_s) @ dequant_fp8(up_w, up_s).T).to(torch.bfloat16)
255
+ hidden = torch.nn.functional.gelu(hidden + up_b.float(), approximate="tanh")
256
+ limit = fp8_max()
257
+ hidden_fp8 = torch.clamp(hidden / hid_s.float(), -limit, limit).to(fp8_dtype())
258
+ out = (dequant_fp8(hidden_fp8, hid_s) @ dequant_fp8(down_w, dn_s).T).to(torch.bfloat16)
259
+ return (out + down_b.float()).to(torch.bfloat16)
260
+
261
+
262
+ def torch_mlp_compile_stable(x, up_w, up_b, down_w, down_b, x_s, up_s, hid_s, dn_s):
263
+ hidden = (dequant_fp8(x, x_s) @ dequant_fp8(up_w, up_s).T).to(torch.bfloat16)
264
+ hidden_fp8 = stable_gelu_quantize_fp8(hidden, up_b, hid_s)
265
+ out = (dequant_fp8(hidden_fp8, hid_s) @ dequant_fp8(down_w, dn_s).T).to(torch.bfloat16)
266
+ return stable_bf16_bias_add(out, down_b)
267
+
268
+
269
+ def make_inputs(M: int, K: int, H: int, N: int, layers: int):
270
+ x_scale = torch.tensor([0.05], device="cuda", dtype=torch.float32)
271
+ up_scale = torch.tensor([0.04], device="cuda", dtype=torch.float32)
272
+ hidden_scale = torch.tensor([0.25], device="cuda", dtype=torch.float32)
273
+ down_scale = torch.tensor([0.04], device="cuda", dtype=torch.float32)
274
+ xs = [
275
+ quantize_fp8(torch.randn((M, K), device="cuda", dtype=torch.bfloat16), x_scale)
276
+ for _ in range(layers)
277
+ ]
278
+ up_ws = [
279
+ quantize_fp8(torch.randn((H, K), device="cuda", dtype=torch.bfloat16), up_scale)
280
+ for _ in range(layers)
281
+ ]
282
+ down_ws = [
283
+ quantize_fp8(torch.randn((N, H), device="cuda", dtype=torch.bfloat16), down_scale)
284
+ for _ in range(layers)
285
+ ]
286
+ up_bs = [torch.randn((H,), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
287
+ down_bs = [torch.randn((N,), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
288
+ hidden = [torch.empty((M, H), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
289
+ hidden_fp8 = [torch.empty((M, H), device="cuda", dtype=fp8_dtype()) for _ in range(layers)]
290
+ outs = [torch.empty((M, N), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
291
+ return xs, up_ws, up_bs, down_ws, down_bs, x_scale, up_scale, hidden_scale, down_scale, hidden, hidden_fp8, outs
292
+
293
+
294
+ def time_us(fn, *, warmup: int, iters: int) -> float:
295
+ for _ in range(warmup):
296
+ fn()
297
+ torch.cuda.synchronize()
298
+ start = torch.cuda.Event(enable_timing=True)
299
+ end = torch.cuda.Event(enable_timing=True)
300
+ start.record()
301
+ for _ in range(iters):
302
+ fn()
303
+ end.record()
304
+ torch.cuda.synchronize()
305
+ return start.elapsed_time(end) * 1000.0 / iters
306
+
307
+
308
+ def percentile(x: torch.Tensor, q: float) -> torch.Tensor:
309
+ flat = x.flatten()
310
+ k = max(1, min(flat.numel(), math.ceil(q * flat.numel())))
311
+ return flat.kthvalue(k).values
312
+
313
+
314
+ def _outputs_close(got, expected) -> bool:
315
+ if isinstance(got, (tuple, list)) and isinstance(expected, (tuple, list)):
316
+ return len(got) == len(expected) and all(
317
+ _outputs_close(g, e) for g, e in zip(got, expected)
318
+ )
319
+ return bool(torch.allclose(got, expected, rtol=3e-2, atol=1.25e-1))
320
+
321
+
322
+ def compile_time_us(fn, expected, *, warmup: int, iters: int) -> tuple[float | None, str]:
323
+ try:
324
+ compiled = torch.compile(fn, fullgraph=False, mode="reduce-overhead")
325
+ compiled_out = compiled()
326
+ torch.cuda.synchronize()
327
+ if not _outputs_close(compiled_out, expected):
328
+ return None, "unsupported: compiled reference output mismatch"
329
+ return time_us(compiled, warmup=warmup, iters=iters), "ok"
330
+ except Exception as exc: # noqa: BLE001
331
+ return None, f"unsupported: {type(exc).__name__}: {exc}"
332
+
333
+
334
+ def run_shape(ops, name: str, shape, args) -> Result:
335
+ M, K, H, N, layers = shape
336
+ xs, up_ws, up_bs, down_ws, down_bs, x_s, up_s, hid_s, dn_s, hidden, hidden_fp8, outs = make_inputs(
337
+ M, K, H, N, layers
338
+ )
339
+
340
+ def flashrt_stack():
341
+ result = []
342
+ for i in range(layers):
343
+ result.append(
344
+ ops.fp8_gelu_mlp_bf16(
345
+ xs[i],
346
+ up_ws[i],
347
+ up_bs[i],
348
+ down_ws[i],
349
+ down_bs[i],
350
+ x_s,
351
+ up_s,
352
+ hid_s,
353
+ dn_s,
354
+ hidden[i],
355
+ hidden_fp8[i],
356
+ outs[i],
357
+ )
358
+ )
359
+ return tuple(result)
360
+
361
+ def torch_stack():
362
+ return tuple(
363
+ torch_mlp(xs[i], up_ws[i], up_bs[i], down_ws[i], down_bs[i], x_s, up_s, hid_s, dn_s)
364
+ for i in range(layers)
365
+ )
366
+
367
+ def torch_stack_compile_stable():
368
+ return tuple(
369
+ torch_mlp_compile_stable(
370
+ xs[i], up_ws[i], up_bs[i], down_ws[i], down_bs[i], x_s, up_s, hid_s, dn_s
371
+ )
372
+ for i in range(layers)
373
+ )
374
+
375
+ flashrt_stack()
376
+ expected0 = torch_mlp(xs[0], up_ws[0], up_bs[0], down_ws[0], down_bs[0], x_s, up_s, hid_s, dn_s)
377
+ diff = (outs[0].float() - expected0.float()).abs().flatten()
378
+ rel = diff / expected0.float().abs().flatten().clamp_min(1.0)
379
+ max_abs = float(diff.max().item())
380
+ p99_abs = float(percentile(diff, 0.99).item())
381
+ p99_rel = float(percentile(rel, 0.99).item())
382
+ max_rel = float(rel.max().item())
383
+ status = (
384
+ "PASS"
385
+ if p99_abs <= args.p99_abs_limit and p99_rel <= args.p99_rel_floor1_limit
386
+ else "FAIL"
387
+ )
388
+
389
+ flashrt_us = time_us(flashrt_stack, warmup=args.warmup, iters=args.iters)
390
+ eager_us = time_us(torch_stack, warmup=args.warmup, iters=args.iters)
391
+ compile_us = None
392
+ compile_status = "not_requested"
393
+ if args.compile_baseline:
394
+ eager_expected = torch_stack()
395
+ stable_expected = torch_stack_compile_stable()
396
+ torch.cuda.synchronize()
397
+ if not _outputs_close(stable_expected, eager_expected):
398
+ compile_status = "unsupported: stable compile reference differs from eager"
399
+ else:
400
+ compile_us, compile_status = compile_time_us(
401
+ torch_stack_compile_stable,
402
+ eager_expected,
403
+ warmup=args.warmup,
404
+ iters=args.iters,
405
+ )
406
+
407
+ return Result(
408
+ shape=name,
409
+ M=M,
410
+ K=K,
411
+ H=H,
412
+ N=N,
413
+ layers=layers,
414
+ flashrt_us=flashrt_us,
415
+ torch_eager_us=eager_us,
416
+ torch_compile_us=compile_us,
417
+ speedup_vs_eager=eager_us / flashrt_us,
418
+ speedup_vs_compile=compile_us / flashrt_us if compile_us is not None else None,
419
+ compile_status=compile_status,
420
+ max_abs=max_abs,
421
+ p99_abs=p99_abs,
422
+ p99_rel_floor1=p99_rel,
423
+ max_rel_floor1=max_rel,
424
+ status=status,
425
+ )
426
+
427
+
428
+ def write_markdown(path: Path, results: list[Result], args) -> None:
429
+ lines = [
430
+ "# Benchmark Results: flashrt-fp8-ffn",
431
+ "",
432
+ f"- Backend: `{args.backend}`",
433
+ f"- Device: `{torch.cuda.get_device_name(0)}`",
434
+ f"- Torch: `{torch.__version__}`",
435
+ f"- Warmup/iters: `{args.warmup}/{args.iters}`",
436
+ f"- Precision gate: p99_abs <= `{args.p99_abs_limit}` and "
437
+ f"p99_rel_floor1 <= `{args.p99_rel_floor1_limit}`",
438
+ "- Compile baseline: reported only when compiled reference output "
439
+ "matches eager reference output.",
440
+ "",
441
+ "| Shape | M,K,H,N | Layers | FlashRT us | Eager us | vs eager | Compile us | vs compile | Compile status | P99 abs | P99 rel | Max abs | Status |",
442
+ "|---|---:|---:|---:|---:|---:|---:|---:|---|---:|---:|---:|---:|",
443
+ ]
444
+ for r in results:
445
+ compile_us = f"{r.torch_compile_us:.3f}" if r.torch_compile_us is not None else "n/a"
446
+ compile_speedup = f"{r.speedup_vs_compile:.2f}x" if r.speedup_vs_compile is not None else "n/a"
447
+ lines.append(
448
+ f"| {r.shape} | {r.M},{r.K},{r.H},{r.N} | {r.layers} | "
449
+ f"{r.flashrt_us:.3f} | {r.torch_eager_us:.3f} | {r.speedup_vs_eager:.2f}x | "
450
+ f"{compile_us} | {compile_speedup} | {r.compile_status} | {r.p99_abs:.4f} | "
451
+ f"{r.p99_rel_floor1:.6f} | {r.max_abs:.4f} | {r.status} |"
452
+ )
453
+ lines.append("")
454
+ path.write_text("\n".join(lines), encoding="utf-8")
455
+
456
+
457
+ def main() -> None:
458
+ parser = argparse.ArgumentParser()
459
+ parser.add_argument("--backend", choices=["source", "installed", "hub"], default="source")
460
+ parser.add_argument("--artifact", default=None)
461
+ parser.add_argument("--repo-id", default="flashrt/flashrt-fp8-ffn")
462
+ parser.add_argument("--version", type=int, default=1)
463
+ parser.add_argument("--shapes", default="all")
464
+ parser.add_argument("--warmup", type=int, default=5)
465
+ parser.add_argument("--iters", type=int, default=20)
466
+ parser.add_argument("--compile-baseline", action="store_true")
467
+ parser.add_argument("--p99-abs-limit", type=float, default=1.0)
468
+ parser.add_argument("--p99-rel-floor1-limit", type=float, default=0.05)
469
+ parser.add_argument("--output", type=Path, default=None)
470
+ parser.add_argument("--markdown", type=Path, default=None)
471
+ parser.add_argument("--list-shapes", action="store_true")
472
+ args = parser.parse_args()
473
+
474
+ if args.list_shapes:
475
+ print("Shape groups:")
476
+ for group, names in SHAPE_GROUPS.items():
477
+ print(f" {group}: {','.join(names)}")
478
+ print("\nShapes:")
479
+ for name, shape in SHAPES.items():
480
+ print(f" {name}: M,K,H,N,layers={shape}")
481
+ return
482
+
483
+ if not torch.cuda.is_available():
484
+ raise SystemExit("CUDA is required")
485
+ torch.manual_seed(17)
486
+ if args.backend == "source":
487
+ ops = load_source_ops()
488
+ elif args.backend == "installed":
489
+ ops = load_installed_ops(args.artifact)
490
+ else:
491
+ ops = load_hub_ops(args.repo_id, args.version)
492
+ requested = [s.strip() for s in args.shapes.split(",")]
493
+ names: list[str] = []
494
+ for item in requested:
495
+ if item in SHAPE_GROUPS:
496
+ names.extend(SHAPE_GROUPS[item])
497
+ else:
498
+ names.append(item)
499
+ unknown = [name for name in names if name not in SHAPES]
500
+ if unknown:
501
+ raise SystemExit(f"unknown shapes/groups: {unknown}")
502
+
503
+ results = []
504
+ for name in names:
505
+ results.append(run_shape(ops, name, SHAPES[name], args))
506
+ torch.cuda.empty_cache()
507
+
508
+ for r in results:
509
+ compile_part = (
510
+ f", compile={r.torch_compile_us:.3f}us, vs_compile={r.speedup_vs_compile:.2f}x"
511
+ if r.torch_compile_us is not None
512
+ else f", compile={r.compile_status}"
513
+ )
514
+ print(
515
+ f"{r.shape}: flashrt={r.flashrt_us:.3f}us, eager={r.torch_eager_us:.3f}us, "
516
+ f"vs_eager={r.speedup_vs_eager:.2f}x{compile_part}, "
517
+ f"p99_abs={r.p99_abs:.4f}, max_abs={r.max_abs:.4f}, {r.status}"
518
+ )
519
+
520
+ payload = {
521
+ "backend": args.backend,
522
+ "torch": torch.__version__,
523
+ "device": torch.cuda.get_device_name(0),
524
+ "results": [asdict(r) for r in results],
525
+ }
526
+ if args.output:
527
+ args.output.parent.mkdir(parents=True, exist_ok=True)
528
+ args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
529
+ if args.markdown:
530
+ args.markdown.parent.mkdir(parents=True, exist_ok=True)
531
+ write_markdown(args.markdown, results, args)
532
+
533
+
534
+ if __name__ == "__main__":
535
+ main()
benchmarks/benchmark_kernel.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kernels.benchmark import Benchmark
4
+
5
+
6
+ def _quantize_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
7
+ return torch.clamp(x.float() / scale.float(), -448.0, 448.0).to(torch.float8_e4m3fn)
8
+
9
+
10
+ def _dequant_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
11
+ return x.float() * scale.float()
12
+
13
+
14
+ def _compiler_disable(fn):
15
+ compiler = getattr(torch, "compiler", None)
16
+ if compiler is not None and hasattr(compiler, "disable"):
17
+ return compiler.disable(fn)
18
+ return torch._dynamo.disable(fn)
19
+
20
+
21
+ def _gelu_quantize_fp8_boundary(
22
+ hidden: torch.Tensor, bias: torch.Tensor, scale: torch.Tensor
23
+ ) -> torch.Tensor:
24
+ hidden = torch.nn.functional.gelu(
25
+ hidden.float() + bias.float(), approximate="tanh"
26
+ )
27
+ return _quantize_fp8(hidden, scale)
28
+
29
+
30
+ def _bf16_bias_add_boundary(out: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
31
+ return (out.float() + bias.float()).to(torch.bfloat16)
32
+
33
+
34
+ _stable_gelu_quantize_fp8 = _compiler_disable(_gelu_quantize_fp8_boundary)
35
+ _stable_bf16_bias_add = _compiler_disable(_bf16_bias_add_boundary)
36
+
37
+
38
+ class FP8GeluMlpBenchmark(Benchmark):
39
+ seed = 17
40
+
41
+ def _setup_shape(self, M: int, K: int, H: int, N: int) -> None:
42
+ self.M, self.K, self.H, self.N = M, K, H, N
43
+ self.x_scale = torch.tensor([0.05], device=self.device, dtype=torch.float32)
44
+ self.up_scale = torch.tensor([0.04], device=self.device, dtype=torch.float32)
45
+ self.hidden_scale = torch.tensor([0.25], device=self.device, dtype=torch.float32)
46
+ self.down_scale = torch.tensor([0.04], device=self.device, dtype=torch.float32)
47
+ self.x = _quantize_fp8(
48
+ torch.randn((M, K), device=self.device, dtype=torch.bfloat16),
49
+ self.x_scale,
50
+ )
51
+ self.up_w = _quantize_fp8(
52
+ torch.randn((H, K), device=self.device, dtype=torch.bfloat16),
53
+ self.up_scale,
54
+ )
55
+ self.down_w = _quantize_fp8(
56
+ torch.randn((N, H), device=self.device, dtype=torch.bfloat16),
57
+ self.down_scale,
58
+ )
59
+ self.up_b = torch.randn((H,), device=self.device, dtype=torch.bfloat16)
60
+ self.down_b = torch.randn((N,), device=self.device, dtype=torch.bfloat16)
61
+ self.hidden = torch.empty((M, H), device=self.device, dtype=torch.bfloat16)
62
+ self.hidden_fp8 = torch.empty((M, H), device=self.device, dtype=torch.float8_e4m3fn)
63
+ self.out = torch.empty((M, N), device=self.device, dtype=torch.bfloat16)
64
+
65
+ def _reference(self) -> torch.Tensor:
66
+ hidden = (
67
+ _dequant_fp8(self.x, self.x_scale)
68
+ @ _dequant_fp8(self.up_w, self.up_scale).T
69
+ ).to(torch.bfloat16)
70
+ hidden_fp8 = _stable_gelu_quantize_fp8(
71
+ hidden, self.up_b, self.hidden_scale
72
+ )
73
+ out = (
74
+ _dequant_fp8(hidden_fp8, self.hidden_scale)
75
+ @ _dequant_fp8(self.down_w, self.down_scale).T
76
+ ).to(torch.bfloat16)
77
+ return _stable_bf16_bias_add(out, self.down_b)
78
+
79
+ def setup_smoke_mlp(self) -> None:
80
+ self._setup_shape(16, 128, 256, 128)
81
+
82
+ def benchmark_smoke_mlp(self) -> None:
83
+ self.kernel.fp8_gelu_mlp_bf16(
84
+ self.x,
85
+ self.up_w,
86
+ self.up_b,
87
+ self.down_w,
88
+ self.down_b,
89
+ self.x_scale,
90
+ self.up_scale,
91
+ self.hidden_scale,
92
+ self.down_scale,
93
+ hidden_bf16=self.hidden,
94
+ hidden_fp8=self.hidden_fp8,
95
+ out=self.out,
96
+ )
97
+
98
+ def verify_smoke_mlp(self) -> torch.Tensor:
99
+ return self._reference()
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49d584720e4739dbf34f58426b49bd60f9ce619de6956132177812c850af7479
3
+ size 300056
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu130-aarch64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch211-cxx11-cu130-aarch64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53e2c1e678c8b4d0e3e33410b9ab79ef5d31913e6ed21e3a35c3dab301497a3e
3
+ size 242032
build/torch211-cxx11-cu130-aarch64-linux/_ops.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
build/torch211-cxx11-cu130-aarch64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ def _import_from_path(file_path: Path):
7
+ path_hash = '{:x}'.format(ctypes.c_size_t(hash(file_path.absolute())).value)
8
+ spec = importlib.util.spec_from_file_location(path_hash, file_path)
9
+ module = importlib.util.module_from_spec(spec)
10
+ sys.modules[path_hash] = module
11
+ spec.loader.exec_module(module)
12
+ return module
13
+
14
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / '__init__.py')))
build/torch211-cxx11-cu130-aarch64-linux/metadata.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0",
11
+ "12.1+PTX"
12
+ ]
13
+ }
14
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3113678b5434234f55790ee0a215dffc4da6936faa0af7f6e1822211a720372f
3
+ size 298176
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch211-cxx11-rocm71-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d56b2ca87eae36419a8aeee3fe444a1894f7c1024c70eaabdea7848201374e3
3
+ size 145192
build/torch211-cxx11-rocm71-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_rocm_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
build/torch211-cxx11-rocm71-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-rocm71-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "rocm",
9
+ "archs": [
10
+ "gfx942"
11
+ ]
12
+ }
13
+ }
build/torch211-cxx11-rocm72-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74b3c142dc669c0e4354b31e05cdf0b080617a74cdf451761aa4c5520d86f651
3
+ size 143848
build/torch211-cxx11-rocm72-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_rocm_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
build/torch211-cxx11-rocm72-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-rocm72-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "rocm",
9
+ "archs": [
10
+ "gfx942"
11
+ ]
12
+ }
13
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f30f0cd5e5d36d0c5eb7ee59f26230eaa452d9eb4e346853a39965ad5ed5a7a6
3
+ size 298032
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14e82facb9b426efcf66b0855d0f281192c806d20d4e4bd00ef63c1a94ea4d48
3
+ size 298032
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-rocm71-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dc2093b25e985a0af258cd2702fb3e28ac98e4ff04a406c423ce48dd66cea11
3
+ size 146416
build/torch212-cxx11-rocm71-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_rocm_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
build/torch212-cxx11-rocm71-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-rocm71-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "rocm",
9
+ "archs": [
10
+ "gfx942"
11
+ ]
12
+ }
13
+ }
build/torch212-cxx11-rocm72-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ output_scale: torch.Tensor,
66
+ hidden_bf16: torch.Tensor,
67
+ out_fp8: torch.Tensor,
68
+ ) -> None:
69
+ expected = (input.shape[0], weight.shape[0])
70
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
71
+ raise RuntimeError(
72
+ "hidden_bf16 and out_fp8 shapes must be "
73
+ "(input.shape[0], weight.shape[0])"
74
+ )
75
+ return None
76
+
77
+
78
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
79
+ def _fp8_gelu_mlp_bf16_fake(
80
+ input: torch.Tensor,
81
+ up_weight: torch.Tensor,
82
+ up_bias: torch.Tensor,
83
+ down_weight: torch.Tensor,
84
+ down_bias: torch.Tensor,
85
+ input_scale: torch.Tensor,
86
+ up_weight_scale: torch.Tensor,
87
+ hidden_scale: torch.Tensor,
88
+ down_weight_scale: torch.Tensor,
89
+ hidden_bf16: torch.Tensor,
90
+ hidden_fp8: torch.Tensor,
91
+ out: torch.Tensor,
92
+ ) -> None:
93
+ hidden_shape = (input.shape[0], up_weight.shape[0])
94
+ out_shape = (input.shape[0], down_weight.shape[0])
95
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
96
+ raise RuntimeError(
97
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
98
+ )
99
+ if out.shape != out_shape:
100
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
101
+ return None
102
+
103
+
104
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
106
+
107
+
108
+ def fp8_gemm_bf16(
109
+ input: torch.Tensor,
110
+ weight: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ weight_scale: torch.Tensor,
113
+ out: torch.Tensor | None = None,
114
+ ) -> torch.Tensor:
115
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
116
+
117
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
118
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
119
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
120
+ """
121
+
122
+ if out is None:
123
+ out = torch.empty(
124
+ (input.shape[0], weight.shape[0]),
125
+ device=input.device,
126
+ dtype=torch.bfloat16,
127
+ )
128
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
129
+ return out
130
+
131
+
132
+ def fp8_linear_bias_gelu_quant_bf16(
133
+ input: torch.Tensor,
134
+ weight: torch.Tensor,
135
+ bias: torch.Tensor,
136
+ input_scale: torch.Tensor,
137
+ weight_scale: torch.Tensor,
138
+ output_scale: torch.Tensor,
139
+ hidden_bf16: torch.Tensor | None = None,
140
+ out_fp8: torch.Tensor | None = None,
141
+ ) -> tuple[torch.Tensor, torch.Tensor]:
142
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
143
+
144
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
145
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
146
+ """
147
+
148
+ if hidden_bf16 is None:
149
+ hidden_bf16 = torch.empty(
150
+ (input.shape[0], weight.shape[0]),
151
+ device=input.device,
152
+ dtype=torch.bfloat16,
153
+ )
154
+ if out_fp8 is None:
155
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
156
+ ops.fp8_linear_bias_gelu_quant_bf16(
157
+ input,
158
+ weight,
159
+ bias,
160
+ input_scale,
161
+ weight_scale,
162
+ output_scale,
163
+ hidden_bf16,
164
+ out_fp8,
165
+ )
166
+ return hidden_bf16, out_fp8
167
+
168
+
169
+ def fp8_gelu_mlp_bf16(
170
+ input: torch.Tensor,
171
+ up_weight: torch.Tensor,
172
+ up_bias: torch.Tensor,
173
+ down_weight: torch.Tensor,
174
+ down_bias: torch.Tensor,
175
+ input_scale: torch.Tensor,
176
+ up_weight_scale: torch.Tensor,
177
+ hidden_scale: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ hidden_bf16: torch.Tensor | None = None,
180
+ hidden_fp8: torch.Tensor | None = None,
181
+ out: torch.Tensor | None = None,
182
+ ) -> torch.Tensor:
183
+ """FP8 GELU MLP block with BF16 output.
184
+
185
+ Computes:
186
+
187
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
188
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
189
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
190
+ """
191
+
192
+ if hidden_bf16 is None:
193
+ hidden_bf16 = torch.empty(
194
+ (input.shape[0], up_weight.shape[0]),
195
+ device=input.device,
196
+ dtype=torch.bfloat16,
197
+ )
198
+ if hidden_fp8 is None:
199
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
200
+ if out is None:
201
+ out = torch.empty(
202
+ (input.shape[0], down_weight.shape[0]),
203
+ device=input.device,
204
+ dtype=torch.bfloat16,
205
+ )
206
+ ops.fp8_gelu_mlp_bf16(
207
+ input,
208
+ up_weight,
209
+ up_bias,
210
+ down_weight,
211
+ down_bias,
212
+ input_scale,
213
+ up_weight_scale,
214
+ hidden_scale,
215
+ down_weight_scale,
216
+ hidden_bf16,
217
+ hidden_fp8,
218
+ out,
219
+ )
220
+ return out
221
+
222
+
223
+ __all__ = [
224
+ "fp8_gemm_bf16",
225
+ "fp8_gelu_mlp_bf16",
226
+ "fp8_linear_bias_gelu_quant_bf16",
227
+ ]
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:387239194294371025c6fa91fd16e3ac550052f1632b73ec37b7879371cc8b9d
3
+ size 145040
build/torch212-cxx11-rocm72-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_rocm_4a17df5
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
build/torch212-cxx11-rocm72-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-rocm72-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "rocm",
9
+ "archs": [
10
+ "gfx942"
11
+ ]
12
+ }
13
+ }