liangsu9988 commited on
Commit
1f0c850
·
verified ·
1 Parent(s): 38805dc

Uploaded using `kernel-builder`.

Browse files
benchmarks/benchmark.py ADDED
@@ -0,0 +1,510 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Benchmark flashrt-fp8-ffn against PyTorch eager/compile references."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import ctypes
8
+ import ctypes.util
9
+ import importlib
10
+ import json
11
+ import math
12
+ import os
13
+ import sys
14
+ from dataclasses import asdict, dataclass
15
+ from pathlib import Path
16
+
17
+ import torch
18
+
19
+
20
+ ROOT = Path(__file__).resolve().parents[2]
21
+ PACKAGE = ROOT / "flashrt-fp8-ffn"
22
+ REGISTRATION_INCLUDE = (
23
+ ROOT.parent
24
+ / "kernels"
25
+ / "kernel-builder"
26
+ / "src"
27
+ / "pyproject"
28
+ / "templates"
29
+ / "torch"
30
+ )
31
+
32
+
33
+ SHAPES = {
34
+ # PI0.5 decoder chunks. Production default is 10 denoising steps.
35
+ "pi05_decoder_ffn_m1": (1, 1024, 4096, 1024, 18),
36
+ "pi05_decoder_ffn_m8": (8, 1024, 4096, 1024, 18),
37
+ "pi05_decoder_ffn_m10": (10, 1024, 4096, 1024, 18),
38
+ "pi05_decoder_ffn_m16": (16, 1024, 4096, 1024, 18),
39
+ # Backward-compatible headline alias.
40
+ "pi05_decoder_ffn": (10, 1024, 4096, 1024, 18),
41
+ # PI0.5 SigLIP-L FFN. One view is 256 visual tokens.
42
+ "pi05_vision_ffn_1view": (256, 1152, 4304, 1152, 27),
43
+ "pi05_vision_ffn_2view": (512, 1152, 4304, 1152, 27),
44
+ "pi05_vision_ffn_3view": (768, 1152, 4304, 1152, 27),
45
+ # GROOT/Qwen3-VL ViT FFN.
46
+ "groot_vit_ffn_1view": (256, 1024, 4096, 1024, 24),
47
+ "groot_vit_ffn_2view": (512, 1024, 4096, 1024, 24),
48
+ "groot_vit_ffn_4view": (1024, 1024, 4096, 1024, 24),
49
+ # GROOT DeepStack merger. Two-view ViT taps produce 128 merged tokens.
50
+ "groot_deepstack_merge_2view": (128, 4096, 4096, 2048, 3),
51
+ # GROOT VL self-attention FFN. Sequence length changes with vision/text mix.
52
+ "groot_vl_self_attn_ffn_seq512": (512, 2048, 8192, 2048, 4),
53
+ "groot_vl_self_attn_ffn_seq1024": (1024, 2048, 8192, 2048, 4),
54
+ "groot_vl_self_attn_ffn_seq2520": (2520, 2048, 8192, 2048, 4),
55
+ # Backward-compatible headline alias.
56
+ "groot_vl_self_attn_ffn": (1024, 2048, 8192, 2048, 4),
57
+ # GROOT action DiT GELU FFN. This is exact GELU shape, but the production
58
+ # path currently uses BF16 GEMMs; report it as a shape fit, not a deployed
59
+ # FP8 action-head claim until model wiring is done.
60
+ "groot_action_dit_ffn": (41, 1536, 6144, 1536, 32),
61
+ }
62
+
63
+ SHAPE_GROUPS = {
64
+ "headline": [
65
+ "pi05_decoder_ffn_m10",
66
+ "pi05_vision_ffn_2view",
67
+ "groot_vit_ffn_2view",
68
+ "groot_vl_self_attn_ffn_seq1024",
69
+ ],
70
+ "pi05": [
71
+ "pi05_decoder_ffn_m1",
72
+ "pi05_decoder_ffn_m8",
73
+ "pi05_decoder_ffn_m10",
74
+ "pi05_decoder_ffn_m16",
75
+ "pi05_vision_ffn_1view",
76
+ "pi05_vision_ffn_2view",
77
+ "pi05_vision_ffn_3view",
78
+ ],
79
+ "groot": [
80
+ "groot_vit_ffn_1view",
81
+ "groot_vit_ffn_2view",
82
+ "groot_vit_ffn_4view",
83
+ "groot_deepstack_merge_2view",
84
+ "groot_vl_self_attn_ffn_seq512",
85
+ "groot_vl_self_attn_ffn_seq1024",
86
+ "groot_vl_self_attn_ffn_seq2520",
87
+ "groot_action_dit_ffn",
88
+ ],
89
+ }
90
+ SHAPE_GROUPS["all"] = SHAPE_GROUPS["pi05"] + SHAPE_GROUPS["groot"]
91
+
92
+
93
+ @dataclass
94
+ class Result:
95
+ shape: str
96
+ M: int
97
+ K: int
98
+ H: int
99
+ N: int
100
+ layers: int
101
+ flashrt_us: float
102
+ torch_eager_us: float
103
+ torch_compile_us: float | None
104
+ speedup_vs_eager: float
105
+ speedup_vs_compile: float | None
106
+ compile_status: str
107
+ max_abs: float
108
+ p99_abs: float
109
+ p99_rel_floor1: float
110
+ max_rel_floor1: float
111
+ status: str
112
+
113
+
114
+ class SourceOps:
115
+ def __init__(self, namespace: str) -> None:
116
+ self._ops = getattr(torch.ops, namespace)
117
+
118
+ def fp8_gelu_mlp_bf16(
119
+ self,
120
+ x,
121
+ up_w,
122
+ up_b,
123
+ down_w,
124
+ down_b,
125
+ x_scale,
126
+ up_w_scale,
127
+ hidden_scale,
128
+ down_w_scale,
129
+ hidden=None,
130
+ hidden_fp8=None,
131
+ out=None,
132
+ ):
133
+ if hidden is None:
134
+ hidden = torch.empty((x.shape[0], up_w.shape[0]), device=x.device, dtype=torch.bfloat16)
135
+ if hidden_fp8 is None:
136
+ hidden_fp8 = torch.empty_like(hidden, dtype=torch.float8_e4m3fn)
137
+ if out is None:
138
+ out = torch.empty((x.shape[0], down_w.shape[0]), device=x.device, dtype=torch.bfloat16)
139
+ self._ops.fp8_gelu_mlp_bf16(
140
+ x,
141
+ up_w,
142
+ up_b,
143
+ down_w,
144
+ down_b,
145
+ x_scale,
146
+ up_w_scale,
147
+ hidden_scale,
148
+ down_w_scale,
149
+ hidden,
150
+ hidden_fp8,
151
+ out,
152
+ )
153
+ return out
154
+
155
+
156
+ def _preload_cublaslt() -> None:
157
+ for parent in Path(torch.__file__).resolve().parents:
158
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
159
+ if candidate.exists():
160
+ ctypes.CDLL(str(candidate), mode=ctypes.RTLD_GLOBAL)
161
+ return
162
+ library = ctypes.util.find_library("cublasLt")
163
+ if library:
164
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
165
+
166
+
167
+ def _current_arch_list() -> str:
168
+ major, minor = torch.cuda.get_device_capability(0)
169
+ return f"{major}.{minor}"
170
+
171
+
172
+ def load_source_ops() -> SourceOps:
173
+ from torch.utils.cpp_extension import load
174
+
175
+ if not REGISTRATION_INCLUDE.is_dir():
176
+ raise RuntimeError(f"missing kernel-builder registration include: {REGISTRATION_INCLUDE}")
177
+ _preload_cublaslt()
178
+ os.environ.setdefault("TORCH_CUDA_ARCH_LIST", _current_arch_list())
179
+ namespace = "flashrt_fp8_ffn_benchmark"
180
+ load(
181
+ name=namespace,
182
+ sources=[
183
+ str(PACKAGE / "torch-ext" / "torch_binding.cpp"),
184
+ str(PACKAGE / "csrc" / "fp8_ffn.cu"),
185
+ ],
186
+ extra_include_paths=[str(PACKAGE / "csrc"), str(REGISTRATION_INCLUDE)],
187
+ extra_cflags=["-O3", "-DCUDA_KERNEL"],
188
+ extra_cuda_cflags=["-O3", "--expt-relaxed-constexpr", "-DCUDA_KERNEL"],
189
+ verbose=False,
190
+ )
191
+ return SourceOps(namespace)
192
+
193
+
194
+ def load_installed_ops(artifact: str | None):
195
+ if artifact:
196
+ sys.path.insert(0, artifact)
197
+ try:
198
+ return importlib.import_module("flashrt_fp8_ffn")
199
+ finally:
200
+ if artifact:
201
+ sys.path.remove(artifact)
202
+
203
+
204
+ def quantize_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
205
+ return torch.clamp(x.float() / scale.float(), -448.0, 448.0).to(torch.float8_e4m3fn)
206
+
207
+
208
+ def dequant_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
209
+ return x.float() * scale.float()
210
+
211
+
212
+ def compiler_disable(fn):
213
+ compiler = getattr(torch, "compiler", None)
214
+ if compiler is not None and hasattr(compiler, "disable"):
215
+ return compiler.disable(fn)
216
+ return torch._dynamo.disable(fn)
217
+
218
+
219
+ def gelu_quantize_fp8_boundary(
220
+ hidden: torch.Tensor, bias: torch.Tensor, scale: torch.Tensor
221
+ ) -> torch.Tensor:
222
+ hidden = torch.nn.functional.gelu(
223
+ hidden.float() + bias.float(), approximate="tanh"
224
+ )
225
+ return quantize_fp8(hidden, scale)
226
+
227
+
228
+ def bf16_bias_add_boundary(out: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
229
+ return (out.float() + bias.float()).to(torch.bfloat16)
230
+
231
+
232
+ stable_gelu_quantize_fp8 = compiler_disable(gelu_quantize_fp8_boundary)
233
+ stable_bf16_bias_add = compiler_disable(bf16_bias_add_boundary)
234
+
235
+
236
+ def torch_mlp(x, up_w, up_b, down_w, down_b, x_s, up_s, hid_s, dn_s):
237
+ hidden = (dequant_fp8(x, x_s) @ dequant_fp8(up_w, up_s).T).to(torch.bfloat16)
238
+ hidden = torch.nn.functional.gelu(hidden + up_b.float(), approximate="tanh")
239
+ hidden_fp8 = torch.clamp(hidden / hid_s.float(), -448.0, 448.0).to(torch.float8_e4m3fn)
240
+ out = (dequant_fp8(hidden_fp8, hid_s) @ dequant_fp8(down_w, dn_s).T).to(torch.bfloat16)
241
+ return (out + down_b.float()).to(torch.bfloat16)
242
+
243
+
244
+ def torch_mlp_compile_stable(x, up_w, up_b, down_w, down_b, x_s, up_s, hid_s, dn_s):
245
+ hidden = (dequant_fp8(x, x_s) @ dequant_fp8(up_w, up_s).T).to(torch.bfloat16)
246
+ hidden_fp8 = stable_gelu_quantize_fp8(hidden, up_b, hid_s)
247
+ out = (dequant_fp8(hidden_fp8, hid_s) @ dequant_fp8(down_w, dn_s).T).to(torch.bfloat16)
248
+ return stable_bf16_bias_add(out, down_b)
249
+
250
+
251
+ def make_inputs(M: int, K: int, H: int, N: int, layers: int):
252
+ x_scale = torch.tensor([0.05], device="cuda", dtype=torch.float32)
253
+ up_scale = torch.tensor([0.04], device="cuda", dtype=torch.float32)
254
+ hidden_scale = torch.tensor([0.25], device="cuda", dtype=torch.float32)
255
+ down_scale = torch.tensor([0.04], device="cuda", dtype=torch.float32)
256
+ xs = [
257
+ quantize_fp8(torch.randn((M, K), device="cuda", dtype=torch.bfloat16), x_scale)
258
+ for _ in range(layers)
259
+ ]
260
+ up_ws = [
261
+ quantize_fp8(torch.randn((H, K), device="cuda", dtype=torch.bfloat16), up_scale)
262
+ for _ in range(layers)
263
+ ]
264
+ down_ws = [
265
+ quantize_fp8(torch.randn((N, H), device="cuda", dtype=torch.bfloat16), down_scale)
266
+ for _ in range(layers)
267
+ ]
268
+ up_bs = [torch.randn((H,), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
269
+ down_bs = [torch.randn((N,), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
270
+ hidden = [torch.empty((M, H), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
271
+ hidden_fp8 = [torch.empty((M, H), device="cuda", dtype=torch.float8_e4m3fn) for _ in range(layers)]
272
+ outs = [torch.empty((M, N), device="cuda", dtype=torch.bfloat16) for _ in range(layers)]
273
+ return xs, up_ws, up_bs, down_ws, down_bs, x_scale, up_scale, hidden_scale, down_scale, hidden, hidden_fp8, outs
274
+
275
+
276
+ def time_us(fn, *, warmup: int, iters: int) -> float:
277
+ for _ in range(warmup):
278
+ fn()
279
+ torch.cuda.synchronize()
280
+ start = torch.cuda.Event(enable_timing=True)
281
+ end = torch.cuda.Event(enable_timing=True)
282
+ start.record()
283
+ for _ in range(iters):
284
+ fn()
285
+ end.record()
286
+ torch.cuda.synchronize()
287
+ return start.elapsed_time(end) * 1000.0 / iters
288
+
289
+
290
+ def percentile(x: torch.Tensor, q: float) -> torch.Tensor:
291
+ flat = x.flatten()
292
+ k = max(1, min(flat.numel(), math.ceil(q * flat.numel())))
293
+ return flat.kthvalue(k).values
294
+
295
+
296
+ def _outputs_close(got, expected) -> bool:
297
+ if isinstance(got, (tuple, list)) and isinstance(expected, (tuple, list)):
298
+ return len(got) == len(expected) and all(
299
+ _outputs_close(g, e) for g, e in zip(got, expected)
300
+ )
301
+ return bool(torch.allclose(got, expected, rtol=3e-2, atol=1.25e-1))
302
+
303
+
304
+ def compile_time_us(fn, expected, *, warmup: int, iters: int) -> tuple[float | None, str]:
305
+ try:
306
+ compiled = torch.compile(fn, fullgraph=False, mode="reduce-overhead")
307
+ compiled_out = compiled()
308
+ torch.cuda.synchronize()
309
+ if not _outputs_close(compiled_out, expected):
310
+ return None, "unsupported: compiled reference output mismatch"
311
+ return time_us(compiled, warmup=warmup, iters=iters), "ok"
312
+ except Exception as exc: # noqa: BLE001
313
+ return None, f"unsupported: {type(exc).__name__}: {exc}"
314
+
315
+
316
+ def run_shape(ops, name: str, shape, args) -> Result:
317
+ M, K, H, N, layers = shape
318
+ xs, up_ws, up_bs, down_ws, down_bs, x_s, up_s, hid_s, dn_s, hidden, hidden_fp8, outs = make_inputs(
319
+ M, K, H, N, layers
320
+ )
321
+
322
+ def flashrt_stack():
323
+ result = []
324
+ for i in range(layers):
325
+ result.append(
326
+ ops.fp8_gelu_mlp_bf16(
327
+ xs[i],
328
+ up_ws[i],
329
+ up_bs[i],
330
+ down_ws[i],
331
+ down_bs[i],
332
+ x_s,
333
+ up_s,
334
+ hid_s,
335
+ dn_s,
336
+ hidden[i],
337
+ hidden_fp8[i],
338
+ outs[i],
339
+ )
340
+ )
341
+ return tuple(result)
342
+
343
+ def torch_stack():
344
+ return tuple(
345
+ torch_mlp(xs[i], up_ws[i], up_bs[i], down_ws[i], down_bs[i], x_s, up_s, hid_s, dn_s)
346
+ for i in range(layers)
347
+ )
348
+
349
+ def torch_stack_compile_stable():
350
+ return tuple(
351
+ torch_mlp_compile_stable(
352
+ xs[i], up_ws[i], up_bs[i], down_ws[i], down_bs[i], x_s, up_s, hid_s, dn_s
353
+ )
354
+ for i in range(layers)
355
+ )
356
+
357
+ flashrt_stack()
358
+ expected0 = torch_mlp(xs[0], up_ws[0], up_bs[0], down_ws[0], down_bs[0], x_s, up_s, hid_s, dn_s)
359
+ diff = (outs[0].float() - expected0.float()).abs().flatten()
360
+ rel = diff / expected0.float().abs().flatten().clamp_min(1.0)
361
+ max_abs = float(diff.max().item())
362
+ p99_abs = float(percentile(diff, 0.99).item())
363
+ p99_rel = float(percentile(rel, 0.99).item())
364
+ max_rel = float(rel.max().item())
365
+ status = (
366
+ "PASS"
367
+ if p99_abs <= args.p99_abs_limit and p99_rel <= args.p99_rel_floor1_limit
368
+ else "FAIL"
369
+ )
370
+
371
+ flashrt_us = time_us(flashrt_stack, warmup=args.warmup, iters=args.iters)
372
+ eager_us = time_us(torch_stack, warmup=args.warmup, iters=args.iters)
373
+ compile_us = None
374
+ compile_status = "not_requested"
375
+ if args.compile_baseline:
376
+ eager_expected = torch_stack()
377
+ stable_expected = torch_stack_compile_stable()
378
+ torch.cuda.synchronize()
379
+ if not _outputs_close(stable_expected, eager_expected):
380
+ compile_status = "unsupported: stable compile reference differs from eager"
381
+ else:
382
+ compile_us, compile_status = compile_time_us(
383
+ torch_stack_compile_stable,
384
+ eager_expected,
385
+ warmup=args.warmup,
386
+ iters=args.iters,
387
+ )
388
+
389
+ return Result(
390
+ shape=name,
391
+ M=M,
392
+ K=K,
393
+ H=H,
394
+ N=N,
395
+ layers=layers,
396
+ flashrt_us=flashrt_us,
397
+ torch_eager_us=eager_us,
398
+ torch_compile_us=compile_us,
399
+ speedup_vs_eager=eager_us / flashrt_us,
400
+ speedup_vs_compile=compile_us / flashrt_us if compile_us is not None else None,
401
+ compile_status=compile_status,
402
+ max_abs=max_abs,
403
+ p99_abs=p99_abs,
404
+ p99_rel_floor1=p99_rel,
405
+ max_rel_floor1=max_rel,
406
+ status=status,
407
+ )
408
+
409
+
410
+ def write_markdown(path: Path, results: list[Result], args) -> None:
411
+ lines = [
412
+ "# Benchmark Results: flashrt-fp8-ffn",
413
+ "",
414
+ f"- Backend: `{args.backend}`",
415
+ f"- Device: `{torch.cuda.get_device_name(0)}`",
416
+ f"- Torch: `{torch.__version__}`",
417
+ f"- Warmup/iters: `{args.warmup}/{args.iters}`",
418
+ f"- Precision gate: p99_abs <= `{args.p99_abs_limit}` and "
419
+ f"p99_rel_floor1 <= `{args.p99_rel_floor1_limit}`",
420
+ "- Compile baseline: reported only when compiled reference output "
421
+ "matches eager reference output.",
422
+ "",
423
+ "| Shape | M,K,H,N | Layers | FlashRT us | Eager us | vs eager | Compile us | vs compile | Compile status | P99 abs | P99 rel | Max abs | Status |",
424
+ "|---|---:|---:|---:|---:|---:|---:|---:|---|---:|---:|---:|---:|",
425
+ ]
426
+ for r in results:
427
+ compile_us = f"{r.torch_compile_us:.3f}" if r.torch_compile_us is not None else "n/a"
428
+ compile_speedup = f"{r.speedup_vs_compile:.2f}x" if r.speedup_vs_compile is not None else "n/a"
429
+ lines.append(
430
+ f"| {r.shape} | {r.M},{r.K},{r.H},{r.N} | {r.layers} | "
431
+ f"{r.flashrt_us:.3f} | {r.torch_eager_us:.3f} | {r.speedup_vs_eager:.2f}x | "
432
+ f"{compile_us} | {compile_speedup} | {r.compile_status} | {r.p99_abs:.4f} | "
433
+ f"{r.p99_rel_floor1:.6f} | {r.max_abs:.4f} | {r.status} |"
434
+ )
435
+ lines.append("")
436
+ path.write_text("\n".join(lines), encoding="utf-8")
437
+
438
+
439
+ def main() -> None:
440
+ parser = argparse.ArgumentParser()
441
+ parser.add_argument("--backend", choices=["source", "installed"], default="source")
442
+ parser.add_argument("--artifact", default=None)
443
+ parser.add_argument("--shapes", default="all")
444
+ parser.add_argument("--warmup", type=int, default=5)
445
+ parser.add_argument("--iters", type=int, default=20)
446
+ parser.add_argument("--compile-baseline", action="store_true")
447
+ parser.add_argument("--p99-abs-limit", type=float, default=1.0)
448
+ parser.add_argument("--p99-rel-floor1-limit", type=float, default=0.05)
449
+ parser.add_argument("--output", type=Path, default=None)
450
+ parser.add_argument("--markdown", type=Path, default=None)
451
+ parser.add_argument("--list-shapes", action="store_true")
452
+ args = parser.parse_args()
453
+
454
+ if args.list_shapes:
455
+ print("Shape groups:")
456
+ for group, names in SHAPE_GROUPS.items():
457
+ print(f" {group}: {','.join(names)}")
458
+ print("\nShapes:")
459
+ for name, shape in SHAPES.items():
460
+ print(f" {name}: M,K,H,N,layers={shape}")
461
+ return
462
+
463
+ if not torch.cuda.is_available():
464
+ raise SystemExit("CUDA is required")
465
+ torch.manual_seed(17)
466
+ ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
467
+ requested = [s.strip() for s in args.shapes.split(",")]
468
+ names: list[str] = []
469
+ for item in requested:
470
+ if item in SHAPE_GROUPS:
471
+ names.extend(SHAPE_GROUPS[item])
472
+ else:
473
+ names.append(item)
474
+ unknown = [name for name in names if name not in SHAPES]
475
+ if unknown:
476
+ raise SystemExit(f"unknown shapes/groups: {unknown}")
477
+
478
+ results = []
479
+ for name in names:
480
+ results.append(run_shape(ops, name, SHAPES[name], args))
481
+ torch.cuda.empty_cache()
482
+
483
+ for r in results:
484
+ compile_part = (
485
+ f", compile={r.torch_compile_us:.3f}us, vs_compile={r.speedup_vs_compile:.2f}x"
486
+ if r.torch_compile_us is not None
487
+ else f", compile={r.compile_status}"
488
+ )
489
+ print(
490
+ f"{r.shape}: flashrt={r.flashrt_us:.3f}us, eager={r.torch_eager_us:.3f}us, "
491
+ f"vs_eager={r.speedup_vs_eager:.2f}x{compile_part}, "
492
+ f"p99_abs={r.p99_abs:.4f}, max_abs={r.max_abs:.4f}, {r.status}"
493
+ )
494
+
495
+ payload = {
496
+ "backend": args.backend,
497
+ "torch": torch.__version__,
498
+ "device": torch.cuda.get_device_name(0),
499
+ "results": [asdict(r) for r in results],
500
+ }
501
+ if args.output:
502
+ args.output.parent.mkdir(parents=True, exist_ok=True)
503
+ args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
504
+ if args.markdown:
505
+ args.markdown.parent.mkdir(parents=True, exist_ok=True)
506
+ write_markdown(args.markdown, results, args)
507
+
508
+
509
+ if __name__ == "__main__":
510
+ main()
benchmarks/benchmark_kernel.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kernels.benchmark import Benchmark
4
+
5
+
6
+ def _quantize_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
7
+ return torch.clamp(x.float() / scale.float(), -448.0, 448.0).to(torch.float8_e4m3fn)
8
+
9
+
10
+ def _dequant_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
11
+ return x.float() * scale.float()
12
+
13
+
14
+ def _compiler_disable(fn):
15
+ compiler = getattr(torch, "compiler", None)
16
+ if compiler is not None and hasattr(compiler, "disable"):
17
+ return compiler.disable(fn)
18
+ return torch._dynamo.disable(fn)
19
+
20
+
21
+ def _gelu_quantize_fp8_boundary(
22
+ hidden: torch.Tensor, bias: torch.Tensor, scale: torch.Tensor
23
+ ) -> torch.Tensor:
24
+ hidden = torch.nn.functional.gelu(
25
+ hidden.float() + bias.float(), approximate="tanh"
26
+ )
27
+ return _quantize_fp8(hidden, scale)
28
+
29
+
30
+ def _bf16_bias_add_boundary(out: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
31
+ return (out.float() + bias.float()).to(torch.bfloat16)
32
+
33
+
34
+ _stable_gelu_quantize_fp8 = _compiler_disable(_gelu_quantize_fp8_boundary)
35
+ _stable_bf16_bias_add = _compiler_disable(_bf16_bias_add_boundary)
36
+
37
+
38
+ class FP8GeluMlpBenchmark(Benchmark):
39
+ seed = 17
40
+
41
+ def _setup_shape(self, M: int, K: int, H: int, N: int) -> None:
42
+ self.M, self.K, self.H, self.N = M, K, H, N
43
+ self.x_scale = torch.tensor([0.05], device=self.device, dtype=torch.float32)
44
+ self.up_scale = torch.tensor([0.04], device=self.device, dtype=torch.float32)
45
+ self.hidden_scale = torch.tensor([0.25], device=self.device, dtype=torch.float32)
46
+ self.down_scale = torch.tensor([0.04], device=self.device, dtype=torch.float32)
47
+ self.x = _quantize_fp8(
48
+ torch.randn((M, K), device=self.device, dtype=torch.bfloat16),
49
+ self.x_scale,
50
+ )
51
+ self.up_w = _quantize_fp8(
52
+ torch.randn((H, K), device=self.device, dtype=torch.bfloat16),
53
+ self.up_scale,
54
+ )
55
+ self.down_w = _quantize_fp8(
56
+ torch.randn((N, H), device=self.device, dtype=torch.bfloat16),
57
+ self.down_scale,
58
+ )
59
+ self.up_b = torch.randn((H,), device=self.device, dtype=torch.bfloat16)
60
+ self.down_b = torch.randn((N,), device=self.device, dtype=torch.bfloat16)
61
+ self.hidden = torch.empty((M, H), device=self.device, dtype=torch.bfloat16)
62
+ self.hidden_fp8 = torch.empty((M, H), device=self.device, dtype=torch.float8_e4m3fn)
63
+ self.out = torch.empty((M, N), device=self.device, dtype=torch.bfloat16)
64
+
65
+ def _reference(self) -> torch.Tensor:
66
+ hidden = (
67
+ _dequant_fp8(self.x, self.x_scale)
68
+ @ _dequant_fp8(self.up_w, self.up_scale).T
69
+ ).to(torch.bfloat16)
70
+ hidden_fp8 = _stable_gelu_quantize_fp8(
71
+ hidden, self.up_b, self.hidden_scale
72
+ )
73
+ out = (
74
+ _dequant_fp8(hidden_fp8, self.hidden_scale)
75
+ @ _dequant_fp8(self.down_w, self.down_scale).T
76
+ ).to(torch.bfloat16)
77
+ return _stable_bf16_bias_add(out, self.down_b)
78
+
79
+ def setup_smoke_mlp(self) -> None:
80
+ self._setup_shape(16, 128, 256, 128)
81
+
82
+ def benchmark_smoke_mlp(self) -> None:
83
+ self.kernel.fp8_gelu_mlp_bf16(
84
+ self.x,
85
+ self.up_w,
86
+ self.up_b,
87
+ self.down_w,
88
+ self.down_b,
89
+ self.x_scale,
90
+ self.up_scale,
91
+ self.hidden_scale,
92
+ self.down_scale,
93
+ hidden_bf16=self.hidden,
94
+ hidden_fp8=self.hidden_fp8,
95
+ out=self.out,
96
+ )
97
+
98
+ def verify_smoke_mlp(self) -> torch.Tensor:
99
+ return self._reference()
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import ops
35
+
36
+
37
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
38
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
39
+
40
+
41
+ def fp8_gemm_bf16(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor | None = None,
47
+ ) -> torch.Tensor:
48
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
49
+
50
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
51
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
52
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
53
+ """
54
+
55
+ if out is None:
56
+ out = torch.empty(
57
+ (input.shape[0], weight.shape[0]),
58
+ device=input.device,
59
+ dtype=torch.bfloat16,
60
+ )
61
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
62
+ return out
63
+
64
+
65
+ def fp8_linear_bias_gelu_quant_bf16(
66
+ input: torch.Tensor,
67
+ weight: torch.Tensor,
68
+ bias: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ weight_scale: torch.Tensor,
71
+ output_scale: torch.Tensor,
72
+ hidden_bf16: torch.Tensor | None = None,
73
+ out_fp8: torch.Tensor | None = None,
74
+ ) -> tuple[torch.Tensor, torch.Tensor]:
75
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
76
+
77
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
78
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
79
+ """
80
+
81
+ if hidden_bf16 is None:
82
+ hidden_bf16 = torch.empty(
83
+ (input.shape[0], weight.shape[0]),
84
+ device=input.device,
85
+ dtype=torch.bfloat16,
86
+ )
87
+ if out_fp8 is None:
88
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
89
+ ops.fp8_linear_bias_gelu_quant_bf16(
90
+ input,
91
+ weight,
92
+ bias,
93
+ input_scale,
94
+ weight_scale,
95
+ output_scale,
96
+ hidden_bf16,
97
+ out_fp8,
98
+ )
99
+ return hidden_bf16, out_fp8
100
+
101
+
102
+ def fp8_gelu_mlp_bf16(
103
+ input: torch.Tensor,
104
+ up_weight: torch.Tensor,
105
+ up_bias: torch.Tensor,
106
+ down_weight: torch.Tensor,
107
+ down_bias: torch.Tensor,
108
+ input_scale: torch.Tensor,
109
+ up_weight_scale: torch.Tensor,
110
+ hidden_scale: torch.Tensor,
111
+ down_weight_scale: torch.Tensor,
112
+ hidden_bf16: torch.Tensor | None = None,
113
+ hidden_fp8: torch.Tensor | None = None,
114
+ out: torch.Tensor | None = None,
115
+ ) -> torch.Tensor:
116
+ """FP8 GELU MLP block with BF16 output.
117
+
118
+ Computes:
119
+
120
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
121
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
122
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
123
+ """
124
+
125
+ if hidden_bf16 is None:
126
+ hidden_bf16 = torch.empty(
127
+ (input.shape[0], up_weight.shape[0]),
128
+ device=input.device,
129
+ dtype=torch.bfloat16,
130
+ )
131
+ if hidden_fp8 is None:
132
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
133
+ if out is None:
134
+ out = torch.empty(
135
+ (input.shape[0], down_weight.shape[0]),
136
+ device=input.device,
137
+ dtype=torch.bfloat16,
138
+ )
139
+ ops.fp8_gelu_mlp_bf16(
140
+ input,
141
+ up_weight,
142
+ up_bias,
143
+ down_weight,
144
+ down_bias,
145
+ input_scale,
146
+ up_weight_scale,
147
+ hidden_scale,
148
+ down_weight_scale,
149
+ hidden_bf16,
150
+ hidden_fp8,
151
+ out,
152
+ )
153
+ return out
154
+
155
+
156
+ __all__ = [
157
+ "fp8_gemm_bf16",
158
+ "fp8_gelu_mlp_bf16",
159
+ "fp8_linear_bias_gelu_quant_bf16",
160
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_e9a1fe0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:364b7d5fdecc653760cee17638abf66b65ce755594c3e56618065c079d7fb9c0
3
+ size 309744
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_e9a1fe0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_e9a1fe0
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_e9a1fe0::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_e9a1fe0",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import ops
35
+
36
+
37
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
38
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
39
+
40
+
41
+ def fp8_gemm_bf16(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor | None = None,
47
+ ) -> torch.Tensor:
48
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
49
+
50
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
51
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
52
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
53
+ """
54
+
55
+ if out is None:
56
+ out = torch.empty(
57
+ (input.shape[0], weight.shape[0]),
58
+ device=input.device,
59
+ dtype=torch.bfloat16,
60
+ )
61
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
62
+ return out
63
+
64
+
65
+ def fp8_linear_bias_gelu_quant_bf16(
66
+ input: torch.Tensor,
67
+ weight: torch.Tensor,
68
+ bias: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ weight_scale: torch.Tensor,
71
+ output_scale: torch.Tensor,
72
+ hidden_bf16: torch.Tensor | None = None,
73
+ out_fp8: torch.Tensor | None = None,
74
+ ) -> tuple[torch.Tensor, torch.Tensor]:
75
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
76
+
77
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
78
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
79
+ """
80
+
81
+ if hidden_bf16 is None:
82
+ hidden_bf16 = torch.empty(
83
+ (input.shape[0], weight.shape[0]),
84
+ device=input.device,
85
+ dtype=torch.bfloat16,
86
+ )
87
+ if out_fp8 is None:
88
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
89
+ ops.fp8_linear_bias_gelu_quant_bf16(
90
+ input,
91
+ weight,
92
+ bias,
93
+ input_scale,
94
+ weight_scale,
95
+ output_scale,
96
+ hidden_bf16,
97
+ out_fp8,
98
+ )
99
+ return hidden_bf16, out_fp8
100
+
101
+
102
+ def fp8_gelu_mlp_bf16(
103
+ input: torch.Tensor,
104
+ up_weight: torch.Tensor,
105
+ up_bias: torch.Tensor,
106
+ down_weight: torch.Tensor,
107
+ down_bias: torch.Tensor,
108
+ input_scale: torch.Tensor,
109
+ up_weight_scale: torch.Tensor,
110
+ hidden_scale: torch.Tensor,
111
+ down_weight_scale: torch.Tensor,
112
+ hidden_bf16: torch.Tensor | None = None,
113
+ hidden_fp8: torch.Tensor | None = None,
114
+ out: torch.Tensor | None = None,
115
+ ) -> torch.Tensor:
116
+ """FP8 GELU MLP block with BF16 output.
117
+
118
+ Computes:
119
+
120
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
121
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
122
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
123
+ """
124
+
125
+ if hidden_bf16 is None:
126
+ hidden_bf16 = torch.empty(
127
+ (input.shape[0], up_weight.shape[0]),
128
+ device=input.device,
129
+ dtype=torch.bfloat16,
130
+ )
131
+ if hidden_fp8 is None:
132
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
133
+ if out is None:
134
+ out = torch.empty(
135
+ (input.shape[0], down_weight.shape[0]),
136
+ device=input.device,
137
+ dtype=torch.bfloat16,
138
+ )
139
+ ops.fp8_gelu_mlp_bf16(
140
+ input,
141
+ up_weight,
142
+ up_bias,
143
+ down_weight,
144
+ down_bias,
145
+ input_scale,
146
+ up_weight_scale,
147
+ hidden_scale,
148
+ down_weight_scale,
149
+ hidden_bf16,
150
+ hidden_fp8,
151
+ out,
152
+ )
153
+ return out
154
+
155
+
156
+ __all__ = [
157
+ "fp8_gemm_bf16",
158
+ "fp8_gelu_mlp_bf16",
159
+ "fp8_linear_bias_gelu_quant_bf16",
160
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_e9a1fe0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:250d27eccb9e7a5231237783665bcbd67871820aa8f7404282032a4329951877
3
+ size 307864
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_e9a1fe0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_e9a1fe0
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_e9a1fe0::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_e9a1fe0",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import ops
35
+
36
+
37
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
38
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
39
+
40
+
41
+ def fp8_gemm_bf16(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor | None = None,
47
+ ) -> torch.Tensor:
48
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
49
+
50
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
51
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
52
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
53
+ """
54
+
55
+ if out is None:
56
+ out = torch.empty(
57
+ (input.shape[0], weight.shape[0]),
58
+ device=input.device,
59
+ dtype=torch.bfloat16,
60
+ )
61
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
62
+ return out
63
+
64
+
65
+ def fp8_linear_bias_gelu_quant_bf16(
66
+ input: torch.Tensor,
67
+ weight: torch.Tensor,
68
+ bias: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ weight_scale: torch.Tensor,
71
+ output_scale: torch.Tensor,
72
+ hidden_bf16: torch.Tensor | None = None,
73
+ out_fp8: torch.Tensor | None = None,
74
+ ) -> tuple[torch.Tensor, torch.Tensor]:
75
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
76
+
77
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
78
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
79
+ """
80
+
81
+ if hidden_bf16 is None:
82
+ hidden_bf16 = torch.empty(
83
+ (input.shape[0], weight.shape[0]),
84
+ device=input.device,
85
+ dtype=torch.bfloat16,
86
+ )
87
+ if out_fp8 is None:
88
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
89
+ ops.fp8_linear_bias_gelu_quant_bf16(
90
+ input,
91
+ weight,
92
+ bias,
93
+ input_scale,
94
+ weight_scale,
95
+ output_scale,
96
+ hidden_bf16,
97
+ out_fp8,
98
+ )
99
+ return hidden_bf16, out_fp8
100
+
101
+
102
+ def fp8_gelu_mlp_bf16(
103
+ input: torch.Tensor,
104
+ up_weight: torch.Tensor,
105
+ up_bias: torch.Tensor,
106
+ down_weight: torch.Tensor,
107
+ down_bias: torch.Tensor,
108
+ input_scale: torch.Tensor,
109
+ up_weight_scale: torch.Tensor,
110
+ hidden_scale: torch.Tensor,
111
+ down_weight_scale: torch.Tensor,
112
+ hidden_bf16: torch.Tensor | None = None,
113
+ hidden_fp8: torch.Tensor | None = None,
114
+ out: torch.Tensor | None = None,
115
+ ) -> torch.Tensor:
116
+ """FP8 GELU MLP block with BF16 output.
117
+
118
+ Computes:
119
+
120
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
121
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
122
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
123
+ """
124
+
125
+ if hidden_bf16 is None:
126
+ hidden_bf16 = torch.empty(
127
+ (input.shape[0], up_weight.shape[0]),
128
+ device=input.device,
129
+ dtype=torch.bfloat16,
130
+ )
131
+ if hidden_fp8 is None:
132
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
133
+ if out is None:
134
+ out = torch.empty(
135
+ (input.shape[0], down_weight.shape[0]),
136
+ device=input.device,
137
+ dtype=torch.bfloat16,
138
+ )
139
+ ops.fp8_gelu_mlp_bf16(
140
+ input,
141
+ up_weight,
142
+ up_bias,
143
+ down_weight,
144
+ down_bias,
145
+ input_scale,
146
+ up_weight_scale,
147
+ hidden_scale,
148
+ down_weight_scale,
149
+ hidden_bf16,
150
+ hidden_fp8,
151
+ out,
152
+ )
153
+ return out
154
+
155
+
156
+ __all__ = [
157
+ "fp8_gemm_bf16",
158
+ "fp8_gelu_mlp_bf16",
159
+ "fp8_linear_bias_gelu_quant_bf16",
160
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_e9a1fe0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bb7793bcc92617d9b7b60d62d78314c368e782632d79e443e0f5fcc2f23b935
3
+ size 314112
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_e9a1fe0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_e9a1fe0
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_e9a1fe0::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_e9a1fe0",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import ops
35
+
36
+
37
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
38
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
39
+
40
+
41
+ def fp8_gemm_bf16(
42
+ input: torch.Tensor,
43
+ weight: torch.Tensor,
44
+ input_scale: torch.Tensor,
45
+ weight_scale: torch.Tensor,
46
+ out: torch.Tensor | None = None,
47
+ ) -> torch.Tensor:
48
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
49
+
50
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
51
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
52
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
53
+ """
54
+
55
+ if out is None:
56
+ out = torch.empty(
57
+ (input.shape[0], weight.shape[0]),
58
+ device=input.device,
59
+ dtype=torch.bfloat16,
60
+ )
61
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
62
+ return out
63
+
64
+
65
+ def fp8_linear_bias_gelu_quant_bf16(
66
+ input: torch.Tensor,
67
+ weight: torch.Tensor,
68
+ bias: torch.Tensor,
69
+ input_scale: torch.Tensor,
70
+ weight_scale: torch.Tensor,
71
+ output_scale: torch.Tensor,
72
+ hidden_bf16: torch.Tensor | None = None,
73
+ out_fp8: torch.Tensor | None = None,
74
+ ) -> tuple[torch.Tensor, torch.Tensor]:
75
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
76
+
77
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
78
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
79
+ """
80
+
81
+ if hidden_bf16 is None:
82
+ hidden_bf16 = torch.empty(
83
+ (input.shape[0], weight.shape[0]),
84
+ device=input.device,
85
+ dtype=torch.bfloat16,
86
+ )
87
+ if out_fp8 is None:
88
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
89
+ ops.fp8_linear_bias_gelu_quant_bf16(
90
+ input,
91
+ weight,
92
+ bias,
93
+ input_scale,
94
+ weight_scale,
95
+ output_scale,
96
+ hidden_bf16,
97
+ out_fp8,
98
+ )
99
+ return hidden_bf16, out_fp8
100
+
101
+
102
+ def fp8_gelu_mlp_bf16(
103
+ input: torch.Tensor,
104
+ up_weight: torch.Tensor,
105
+ up_bias: torch.Tensor,
106
+ down_weight: torch.Tensor,
107
+ down_bias: torch.Tensor,
108
+ input_scale: torch.Tensor,
109
+ up_weight_scale: torch.Tensor,
110
+ hidden_scale: torch.Tensor,
111
+ down_weight_scale: torch.Tensor,
112
+ hidden_bf16: torch.Tensor | None = None,
113
+ hidden_fp8: torch.Tensor | None = None,
114
+ out: torch.Tensor | None = None,
115
+ ) -> torch.Tensor:
116
+ """FP8 GELU MLP block with BF16 output.
117
+
118
+ Computes:
119
+
120
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
121
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
122
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
123
+ """
124
+
125
+ if hidden_bf16 is None:
126
+ hidden_bf16 = torch.empty(
127
+ (input.shape[0], up_weight.shape[0]),
128
+ device=input.device,
129
+ dtype=torch.bfloat16,
130
+ )
131
+ if hidden_fp8 is None:
132
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=torch.float8_e4m3fn)
133
+ if out is None:
134
+ out = torch.empty(
135
+ (input.shape[0], down_weight.shape[0]),
136
+ device=input.device,
137
+ dtype=torch.bfloat16,
138
+ )
139
+ ops.fp8_gelu_mlp_bf16(
140
+ input,
141
+ up_weight,
142
+ up_bias,
143
+ down_weight,
144
+ down_bias,
145
+ input_scale,
146
+ up_weight_scale,
147
+ hidden_scale,
148
+ down_weight_scale,
149
+ hidden_bf16,
150
+ hidden_fp8,
151
+ out,
152
+ )
153
+ return out
154
+
155
+
156
+ __all__ = [
157
+ "fp8_gemm_bf16",
158
+ "fp8_gelu_mlp_bf16",
159
+ "fp8_linear_bias_gelu_quant_bf16",
160
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_e9a1fe0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:842932c5da32e68a4eef4737ab0264d69a4aef0254fceef283333277a9215c94
3
+ size 314112
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_e9a1fe0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_e9a1fe0
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_e9a1fe0::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_e9a1fe0",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }