liangsu9988 commited on
Commit
221ab11
·
verified ·
1 Parent(s): 8329635

Uploaded using `kernel-builder`.

Browse files
Files changed (38) hide show
  1. benchmarks/benchmark.py +50 -0
  2. benchmarks/benchmark_bf16_entry.py +398 -0
  3. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +104 -0
  4. build/torch211-cxx11-cu128-x86_64-linux/{_flashrt_fp8_ffn_cuda_4a17df5.abi3.so → _flashrt_fp8_ffn_cuda_6ec89eb.abi3.so} +2 -2
  5. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  6. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +1 -1
  7. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +104 -0
  8. build/torch211-cxx11-cu130-x86_64-linux/{_flashrt_fp8_ffn_cuda_4a17df5.abi3.so → _flashrt_fp8_ffn_cuda_6ec89eb.abi3.so} +2 -2
  9. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  10. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +1 -1
  11. build/torch211-cxx11-rocm71-x86_64-linux/__init__.py +104 -0
  12. build/torch211-cxx11-rocm71-x86_64-linux/{_flashrt_fp8_ffn_rocm_4a17df5.abi3.so → _flashrt_fp8_ffn_rocm_6ec89eb.abi3.so} +2 -2
  13. build/torch211-cxx11-rocm71-x86_64-linux/_ops.py +3 -3
  14. build/torch211-cxx11-rocm71-x86_64-linux/metadata.json +1 -1
  15. build/torch211-cxx11-rocm72-x86_64-linux/__init__.py +104 -0
  16. build/torch211-cxx11-rocm72-x86_64-linux/{_flashrt_fp8_ffn_rocm_4a17df5.abi3.so → _flashrt_fp8_ffn_rocm_6ec89eb.abi3.so} +2 -2
  17. build/torch211-cxx11-rocm72-x86_64-linux/_ops.py +3 -3
  18. build/torch211-cxx11-rocm72-x86_64-linux/metadata.json +1 -1
  19. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +104 -0
  20. build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +0 -3
  21. build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_6ec89eb.abi3.so +3 -0
  22. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  23. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +1 -1
  24. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +104 -0
  25. build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so +0 -3
  26. build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_6ec89eb.abi3.so +3 -0
  27. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  28. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +1 -1
  29. build/torch212-cxx11-rocm71-x86_64-linux/__init__.py +104 -0
  30. build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so +0 -3
  31. build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_6ec89eb.abi3.so +3 -0
  32. build/torch212-cxx11-rocm71-x86_64-linux/_ops.py +3 -3
  33. build/torch212-cxx11-rocm71-x86_64-linux/metadata.json +1 -1
  34. build/torch212-cxx11-rocm72-x86_64-linux/__init__.py +104 -0
  35. build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so +0 -3
  36. build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_6ec89eb.abi3.so +3 -0
  37. build/torch212-cxx11-rocm72-x86_64-linux/_ops.py +3 -3
  38. build/torch212-cxx11-rocm72-x86_64-linux/metadata.json +1 -1
benchmarks/benchmark.py CHANGED
@@ -152,6 +152,56 @@ class SourceOps:
152
  )
153
  return out
154
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
 
156
  def _preload_cublaslt() -> None:
157
  for parent in Path(torch.__file__).resolve().parents:
 
152
  )
153
  return out
154
 
155
+ def bf16_fp8_gelu_mlp_bf16(
156
+ self,
157
+ x,
158
+ up_w,
159
+ up_b,
160
+ down_w,
161
+ down_b,
162
+ x_scale,
163
+ up_w_scale,
164
+ hidden_scale,
165
+ down_w_scale,
166
+ input_fp8=None,
167
+ hidden_bf16=None,
168
+ hidden_fp8=None,
169
+ out=None,
170
+ *,
171
+ pad_to=None,
172
+ ):
173
+ padded_m = x.shape[0] if pad_to is None else pad_to
174
+ if input_fp8 is None:
175
+ input_fp8 = torch.empty(
176
+ (padded_m, x.shape[1]), device=x.device, dtype=fp8_dtype()
177
+ )
178
+ if hidden_bf16 is None:
179
+ hidden_bf16 = torch.empty(
180
+ (padded_m, up_w.shape[0]), device=x.device, dtype=torch.bfloat16
181
+ )
182
+ if hidden_fp8 is None:
183
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=fp8_dtype())
184
+ if out is None:
185
+ out = torch.empty(
186
+ (padded_m, down_w.shape[0]), device=x.device, dtype=torch.bfloat16
187
+ )
188
+ self._ops.bf16_fp8_gelu_mlp_bf16(
189
+ x,
190
+ up_w,
191
+ up_b,
192
+ down_w,
193
+ down_b,
194
+ x_scale,
195
+ up_w_scale,
196
+ hidden_scale,
197
+ down_w_scale,
198
+ input_fp8,
199
+ hidden_bf16,
200
+ hidden_fp8,
201
+ out,
202
+ )
203
+ return out[: x.shape[0]]
204
+
205
 
206
  def _preload_cublaslt() -> None:
207
  for parent in Path(torch.__file__).resolve().parents:
benchmarks/benchmark_bf16_entry.py ADDED
@@ -0,0 +1,398 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Benchmark the BF16-to-FP8 GELU FFN region boundary."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ from dataclasses import asdict, dataclass
9
+ from pathlib import Path
10
+
11
+ import torch
12
+ import torch.nn.functional as F
13
+
14
+ import benchmark as base
15
+
16
+
17
+ SHAPES = {
18
+ "siglip_m8": (8, 1152, 4304, 1152),
19
+ "siglip_m51": (51, 1152, 4304, 1152),
20
+ "siglip_m64": (64, 1152, 4304, 1152),
21
+ "siglip_m105": (105, 1152, 4304, 1152),
22
+ "siglip_m128": (128, 1152, 4304, 1152),
23
+ "dit_m8": (8, 1536, 6144, 1536),
24
+ "dit_m51": (51, 1536, 6144, 1536),
25
+ "dit_m64": (64, 1536, 6144, 1536),
26
+ "dit_m105": (105, 1536, 6144, 1536),
27
+ "dit_m128": (128, 1536, 6144, 1536),
28
+ }
29
+
30
+
31
+ @dataclass
32
+ class Result:
33
+ shape: str
34
+ M: int
35
+ K: int
36
+ H: int
37
+ N: int
38
+ flashrt_bf16_entry_us: float
39
+ flashrt_cuda_graph_us: float
40
+ separate_quant_us: float
41
+ fp8_kernel_only_us: float
42
+ torch_bf16_eager_us: float
43
+ torch_bf16_compile_us: float | None
44
+ speedup_vs_separate_quant: float
45
+ speedup_vs_eager: float
46
+ speedup_vs_compile: float | None
47
+ compile_status: str
48
+ flashrt_compile_status: str
49
+ input_quant_exact: bool
50
+ output_dtype: str
51
+ staged_max_abs: float
52
+ staged_mean_abs: float
53
+ staged_p99_abs: float
54
+ staged_cosine: float
55
+ bf16_max_abs: float
56
+ bf16_mean_abs: float
57
+ bf16_p99_abs: float
58
+ bf16_cosine: float
59
+ performance_status: str
60
+ status: str
61
+
62
+
63
+ def percentile(x: torch.Tensor, q: float) -> float:
64
+ flat = x.flatten()
65
+ k = max(1, min(flat.numel(), int(q * flat.numel() + 0.999999)))
66
+ return float(flat.kthvalue(k).values.item())
67
+
68
+
69
+ def metrics(got: torch.Tensor, expected: torch.Tensor) -> dict[str, float]:
70
+ diff = (got.float() - expected.float()).abs().flatten()
71
+ cosine = F.cosine_similarity(
72
+ got.float().flatten(), expected.float().flatten(), dim=0
73
+ )
74
+ return {
75
+ "max_abs": float(diff.max().item()),
76
+ "mean_abs": float(diff.mean().item()),
77
+ "p99_abs": percentile(diff, 0.99),
78
+ "cosine": float(cosine.item()),
79
+ }
80
+
81
+
82
+ def quantize_input(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
83
+ inv_scale = 1.0 / scale.float()
84
+ return torch.clamp(
85
+ x.float() * inv_scale, -base.fp8_max(), base.fp8_max()
86
+ ).to(base.fp8_dtype())
87
+
88
+
89
+ def midm_padded_rows(rows: int) -> int:
90
+ if (
91
+ torch.version.hip is None
92
+ and torch.cuda.get_device_capability(0) == (11, 0)
93
+ and 9 <= rows <= 128
94
+ ):
95
+ return ((rows + 63) // 64) * 64
96
+ return rows
97
+
98
+
99
+ def make_case(M: int, K: int, H: int, N: int):
100
+ x = torch.randn((M, K), device="cuda", dtype=torch.bfloat16) * 0.25
101
+ up = torch.randn((H, K), device="cuda", dtype=torch.bfloat16) * (K**-0.5)
102
+ down = torch.randn((N, H), device="cuda", dtype=torch.bfloat16) * (H**-0.5)
103
+ up_bias = torch.randn((H,), device="cuda", dtype=torch.bfloat16) * 0.01
104
+ down_bias = torch.randn((N,), device="cuda", dtype=torch.bfloat16) * 0.01
105
+
106
+ def scale_for(tensor: torch.Tensor) -> torch.Tensor:
107
+ return (
108
+ tensor.float().abs().max() / (0.9 * base.fp8_max())
109
+ ).clamp_min(1e-6).reshape(1)
110
+
111
+ x_scale = scale_for(x)
112
+ up_scale = scale_for(up)
113
+ down_scale = scale_for(down)
114
+ x_fp8 = quantize_input(x, x_scale)
115
+ up_fp8 = base.quantize_fp8(up, up_scale)
116
+ down_fp8 = base.quantize_fp8(down, down_scale)
117
+ calibrated_hidden = F.gelu(
118
+ (x_fp8.float() * x_scale) @ (up_fp8.float() * up_scale).T
119
+ + up_bias.float(),
120
+ approximate="tanh",
121
+ )
122
+ hidden_scale = scale_for(calibrated_hidden)
123
+ return (
124
+ x,
125
+ up,
126
+ up_bias,
127
+ down,
128
+ down_bias,
129
+ x_fp8,
130
+ up_fp8,
131
+ down_fp8,
132
+ x_scale,
133
+ up_scale,
134
+ hidden_scale,
135
+ down_scale,
136
+ )
137
+
138
+
139
+ def run_shape(ops, name: str, shape, args) -> Result:
140
+ M, K, H, N = shape
141
+ (
142
+ x,
143
+ up,
144
+ up_bias,
145
+ down,
146
+ down_bias,
147
+ x_fp8,
148
+ up_fp8,
149
+ down_fp8,
150
+ x_scale,
151
+ up_scale,
152
+ hidden_scale,
153
+ down_scale,
154
+ ) = make_case(M, K, H, N)
155
+
156
+ padded_m = midm_padded_rows(M)
157
+ input_fp8 = torch.empty((padded_m, K), device="cuda", dtype=base.fp8_dtype())
158
+ hidden_bf16 = torch.empty((padded_m, H), device="cuda", dtype=torch.bfloat16)
159
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=base.fp8_dtype())
160
+ out = torch.empty((padded_m, N), device="cuda", dtype=torch.bfloat16)
161
+ exact_hidden = torch.empty_like(hidden_bf16)
162
+ exact_hidden_fp8 = torch.empty_like(hidden_fp8)
163
+ exact_out = torch.empty_like(out)
164
+ staged_hidden = torch.empty((M, H), device="cuda", dtype=torch.bfloat16)
165
+ staged_hidden_fp8 = torch.empty_like(staged_hidden, dtype=base.fp8_dtype())
166
+ staged_out = torch.empty((M, N), device="cuda", dtype=torch.bfloat16)
167
+
168
+ def flashrt_call():
169
+ return ops.bf16_fp8_gelu_mlp_bf16(
170
+ x,
171
+ up_fp8,
172
+ up_bias,
173
+ down_fp8,
174
+ down_bias,
175
+ x_scale,
176
+ up_scale,
177
+ hidden_scale,
178
+ down_scale,
179
+ input_fp8=input_fp8,
180
+ hidden_bf16=hidden_bf16,
181
+ hidden_fp8=hidden_fp8,
182
+ out=out,
183
+ pad_to=padded_m,
184
+ )
185
+
186
+ def staged_call(input_arg=x_fp8):
187
+ return ops.fp8_gelu_mlp_bf16(
188
+ input_arg,
189
+ up_fp8,
190
+ up_bias,
191
+ down_fp8,
192
+ down_bias,
193
+ x_scale,
194
+ up_scale,
195
+ hidden_scale,
196
+ down_scale,
197
+ hidden=staged_hidden,
198
+ hidden_fp8=staged_hidden_fp8,
199
+ out=staged_out,
200
+ )
201
+
202
+ def separate_quant_call():
203
+ return staged_call(quantize_input(x, x_scale))
204
+
205
+ def exact_staged_call():
206
+ return ops.fp8_gelu_mlp_bf16(
207
+ input_fp8,
208
+ up_fp8,
209
+ up_bias,
210
+ down_fp8,
211
+ down_bias,
212
+ x_scale,
213
+ up_scale,
214
+ hidden_scale,
215
+ down_scale,
216
+ hidden=exact_hidden,
217
+ hidden_fp8=exact_hidden_fp8,
218
+ out=exact_out,
219
+ )[:M]
220
+
221
+ def torch_bf16_reference():
222
+ hidden = F.gelu(F.linear(x, up, up_bias), approximate="tanh")
223
+ return F.linear(hidden, down, down_bias)
224
+
225
+ got = flashrt_call().clone()
226
+ staged = exact_staged_call().clone()
227
+ torch.cuda.synchronize()
228
+ quant_exact = bool(
229
+ torch.equal(input_fp8[:M], x_fp8)
230
+ and (padded_m == M or torch.count_nonzero(input_fp8[M:]).item() == 0)
231
+ )
232
+ staged_metrics = metrics(got, staged)
233
+ bf16_expected = torch_bf16_reference()
234
+ bf16_metrics = metrics(got, bf16_expected)
235
+ staged_compatible = quant_exact and staged_metrics["max_abs"] == 0.0
236
+
237
+ flashrt_us = base.time_us(flashrt_call, warmup=args.warmup, iters=args.iters)
238
+ graph = torch.cuda.CUDAGraph()
239
+ flashrt_call()
240
+ torch.cuda.synchronize()
241
+ with torch.cuda.graph(graph):
242
+ flashrt_call()
243
+ graph_us = base.time_us(graph.replay, warmup=args.warmup, iters=args.iters)
244
+ separate_us = base.time_us(
245
+ separate_quant_call, warmup=args.warmup, iters=args.iters
246
+ )
247
+ kernel_us = base.time_us(
248
+ staged_call, warmup=args.warmup, iters=args.iters
249
+ )
250
+ eager_us = base.time_us(
251
+ torch_bf16_reference, warmup=args.warmup, iters=args.iters
252
+ )
253
+
254
+ compile_us = None
255
+ compile_status = "not_requested"
256
+ flashrt_compile_status = "not_requested"
257
+ if args.compile_baseline:
258
+ try:
259
+ compiled = torch.compile(
260
+ torch_bf16_reference, fullgraph=True, mode="reduce-overhead"
261
+ )
262
+ compiled_out = compiled()
263
+ torch.cuda.synchronize()
264
+ compiled_metrics = metrics(compiled_out, bf16_expected)
265
+ if compiled_metrics["cosine"] < 0.9999:
266
+ compile_status = (
267
+ "mismatch: cosine="
268
+ f"{compiled_metrics['cosine']:.8f}"
269
+ )
270
+ else:
271
+ compile_us = base.time_us(
272
+ compiled, warmup=args.warmup, iters=args.iters
273
+ )
274
+ compile_status = "fullgraph-ok"
275
+ except Exception as exc: # noqa: BLE001
276
+ compile_status = f"failed: {type(exc).__name__}: {exc}"
277
+ try:
278
+ compiled_flashrt = torch.compile(
279
+ flashrt_call, fullgraph=True, mode="reduce-overhead"
280
+ )
281
+ compiled_got = compiled_flashrt().clone()
282
+ torch.cuda.synchronize()
283
+ flashrt_compile_status = (
284
+ "fullgraph-ok"
285
+ if metrics(compiled_got, got)["max_abs"] == 0.0
286
+ else "mismatch"
287
+ )
288
+ except Exception as exc: # noqa: BLE001
289
+ flashrt_compile_status = f"failed: {type(exc).__name__}: {exc}"
290
+
291
+ speedup_eager = eager_us / flashrt_us
292
+ speedup_separate = separate_us / flashrt_us
293
+ perf_status = (
294
+ ("PASS" if speedup_eager >= 1.3 else "FAIL")
295
+ if M == 51
296
+ else "DIAGNOSTIC"
297
+ )
298
+ status = "PASS" if staged_compatible and perf_status != "FAIL" else "FAIL"
299
+ return Result(
300
+ shape=name,
301
+ M=M,
302
+ K=K,
303
+ H=H,
304
+ N=N,
305
+ flashrt_bf16_entry_us=flashrt_us,
306
+ flashrt_cuda_graph_us=graph_us,
307
+ separate_quant_us=separate_us,
308
+ fp8_kernel_only_us=kernel_us,
309
+ torch_bf16_eager_us=eager_us,
310
+ torch_bf16_compile_us=compile_us,
311
+ speedup_vs_separate_quant=speedup_separate,
312
+ speedup_vs_eager=speedup_eager,
313
+ speedup_vs_compile=compile_us / flashrt_us if compile_us else None,
314
+ compile_status=compile_status,
315
+ flashrt_compile_status=flashrt_compile_status,
316
+ input_quant_exact=quant_exact,
317
+ output_dtype=str(got.dtype),
318
+ staged_max_abs=staged_metrics["max_abs"],
319
+ staged_mean_abs=staged_metrics["mean_abs"],
320
+ staged_p99_abs=staged_metrics["p99_abs"],
321
+ staged_cosine=staged_metrics["cosine"],
322
+ bf16_max_abs=bf16_metrics["max_abs"],
323
+ bf16_mean_abs=bf16_metrics["mean_abs"],
324
+ bf16_p99_abs=bf16_metrics["p99_abs"],
325
+ bf16_cosine=bf16_metrics["cosine"],
326
+ performance_status=perf_status,
327
+ status=status,
328
+ )
329
+
330
+
331
+ def main() -> None:
332
+ parser = argparse.ArgumentParser()
333
+ parser.add_argument("--backend", choices=["source", "installed", "hub"], default="source")
334
+ parser.add_argument("--artifact")
335
+ parser.add_argument("--repo-id", default="flashrt/flashrt-fp8-ffn")
336
+ parser.add_argument("--version", type=int, default=1)
337
+ parser.add_argument("--shapes", default="all")
338
+ parser.add_argument("--warmup", type=int, default=20)
339
+ parser.add_argument("--iters", type=int, default=100)
340
+ parser.add_argument("--compile-baseline", action="store_true")
341
+ parser.add_argument("--output", type=Path)
342
+ args = parser.parse_args()
343
+
344
+ if not torch.cuda.is_available():
345
+ raise SystemExit("CUDA is required")
346
+ torch.manual_seed(19)
347
+ if args.backend == "source":
348
+ ops = base.load_source_ops()
349
+ elif args.backend == "installed":
350
+ ops = base.load_installed_ops(args.artifact)
351
+ else:
352
+ ops = base.load_hub_ops(args.repo_id, args.version)
353
+ names = list(SHAPES) if args.shapes == "all" else args.shapes.split(",")
354
+ unknown = [name for name in names if name not in SHAPES]
355
+ if unknown:
356
+ raise SystemExit(f"unknown shapes: {unknown}")
357
+
358
+ results = []
359
+ for name in names:
360
+ result = run_shape(ops, name, SHAPES[name], args)
361
+ results.append(result)
362
+ compile_text = (
363
+ f"{result.torch_bf16_compile_us:.3f}us"
364
+ if result.torch_bf16_compile_us is not None
365
+ else result.compile_status
366
+ )
367
+ print(
368
+ f"{result.status} {name}: flashrt={result.flashrt_bf16_entry_us:.3f}us "
369
+ f"graph={result.flashrt_cuda_graph_us:.3f}us "
370
+ f"separate={result.separate_quant_us:.3f}us "
371
+ f"kernel_only={result.fp8_kernel_only_us:.3f}us "
372
+ f"eager={result.torch_bf16_eager_us:.3f}us "
373
+ f"compile={compile_text} vs_eager={result.speedup_vs_eager:.2f}x "
374
+ f"vs_separate={result.speedup_vs_separate_quant:.2f}x "
375
+ f"staged_max={result.staged_max_abs:.6f} "
376
+ f"bf16_cos={result.bf16_cosine:.8f} "
377
+ f"perf={result.performance_status} "
378
+ f"op_compile={result.flashrt_compile_status}"
379
+ )
380
+ torch.cuda.empty_cache()
381
+
382
+ payload = {
383
+ "backend": args.backend,
384
+ "device": torch.cuda.get_device_name(0),
385
+ "torch": torch.__version__,
386
+ "warmup": args.warmup,
387
+ "iters": args.iters,
388
+ "results": [asdict(result) for result in results],
389
+ }
390
+ if args.output:
391
+ args.output.parent.mkdir(parents=True, exist_ok=True)
392
+ args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
393
+ if any(result.status != "PASS" for result in results):
394
+ raise SystemExit(1)
395
+
396
+
397
+ if __name__ == "__main__":
398
+ main()
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch211-cxx11-cu128-x86_64-linux/{_flashrt_fp8_ffn_cuda_4a17df5.abi3.so → _flashrt_fp8_ffn_cuda_6ec89eb.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:49d584720e4739dbf34f58426b49bd60f9ce619de6956132177812c850af7479
3
- size 300056
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45fdee1e129815abb6aa0bec654917ee8bb84450f526fb6ab1b4ad7a48f5520e
3
+ size 381256
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_6ec89eb::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch211-cxx11-cu130-x86_64-linux/{_flashrt_fp8_ffn_cuda_4a17df5.abi3.so → _flashrt_fp8_ffn_cuda_6ec89eb.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3113678b5434234f55790ee0a215dffc4da6936faa0af7f6e1822211a720372f
3
- size 298176
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa986eb7f49ab4f3c56d8e28e77f1eb288ca04d65d8355a94e25b767b4b3feed
3
+ size 376384
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_6ec89eb::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch211-cxx11-rocm71-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch211-cxx11-rocm71-x86_64-linux/{_flashrt_fp8_ffn_rocm_4a17df5.abi3.so → _flashrt_fp8_ffn_rocm_6ec89eb.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9d56b2ca87eae36419a8aeee3fe444a1894f7c1024c70eaabdea7848201374e3
3
- size 145192
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:450bf9e293e7e6631e071ed4c346f3d1f899413567e6342cd60a08744a0da70f
3
+ size 167328
build/torch211-cxx11-rocm71-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_6ec89eb::{op_name}"
build/torch211-cxx11-rocm71-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch211-cxx11-rocm72-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch211-cxx11-rocm72-x86_64-linux/{_flashrt_fp8_ffn_rocm_4a17df5.abi3.so → _flashrt_fp8_ffn_rocm_6ec89eb.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:74b3c142dc669c0e4354b31e05cdf0b080617a74cdf451761aa4c5520d86f651
3
- size 143848
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69400d755c2f5b7f75d42381f1663da28d04b52d176f4d989ae1422a6b254c9f
3
+ size 165760
build/torch211-cxx11-rocm72-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_6ec89eb::{op_name}"
build/torch211-cxx11-rocm72-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f30f0cd5e5d36d0c5eb7ee59f26230eaa452d9eb4e346853a39965ad5ed5a7a6
3
- size 298032
 
 
 
 
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_6ec89eb.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6d9361c58816342dc79636e88c950840bb1fbdbeb5ccf690ea125c838851ae7
3
+ size 376296
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_6ec89eb::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_4a17df5.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:14e82facb9b426efcf66b0855d0f281192c806d20d4e4bd00ef63c1a94ea4d48
3
- size 298032
 
 
 
 
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_6ec89eb.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:602c56052c4e10d26275d245c2325b57dd4ebb5f845fb560985363b279ddaaa6
3
+ size 376296
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_6ec89eb::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-rocm71-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3dc2093b25e985a0af258cd2702fb3e28ac98e4ff04a406c423ce48dd66cea11
3
- size 146416
 
 
 
 
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_6ec89eb.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2c1341cdf4e22c362d63a08687803fe9bd3a94f15b1e2c4e6c06048962acb07
3
+ size 167224
build/torch212-cxx11-rocm71-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_6ec89eb::{op_name}"
build/torch212-cxx11-rocm71-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-rocm72-x86_64-linux/__init__.py CHANGED
@@ -101,6 +101,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -220,7 +248,83 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
226
  "fp8_linear_bias_gelu_quant_bf16",
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
105
+ def _bf16_fp8_gelu_mlp_bf16_fake(
106
+ input: torch.Tensor,
107
+ up_weight: torch.Tensor,
108
+ up_bias: torch.Tensor,
109
+ down_weight: torch.Tensor,
110
+ down_bias: torch.Tensor,
111
+ input_scale: torch.Tensor,
112
+ up_weight_scale: torch.Tensor,
113
+ hidden_scale: torch.Tensor,
114
+ down_weight_scale: torch.Tensor,
115
+ input_fp8: torch.Tensor,
116
+ hidden_bf16: torch.Tensor,
117
+ hidden_fp8: torch.Tensor,
118
+ out: torch.Tensor,
119
+ ) -> None:
120
+ padded_m = input_fp8.shape[0]
121
+ if input_fp8.shape != (padded_m, input.shape[1]):
122
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
123
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
124
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
125
+ if hidden_fp8.shape != hidden_bf16.shape:
126
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
127
+ if out.shape != (padded_m, down_weight.shape[0]):
128
+ raise RuntimeError("out has an invalid padded shape")
129
+ return None
130
+
131
+
132
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
133
  return torch.tensor([value], device=input.device, dtype=torch.float32)
134
 
 
248
  return out
249
 
250
 
251
+ def _midm_padded_rows(input: torch.Tensor) -> int:
252
+ rows = input.shape[0]
253
+ if (
254
+ torch.version.hip is None
255
+ and input.is_cuda
256
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
257
+ and 9 <= rows <= 128
258
+ ):
259
+ return ((rows + 63) // 64) * 64
260
+ return rows
261
+
262
+
263
+ def bf16_fp8_gelu_mlp_bf16(
264
+ input: torch.Tensor,
265
+ up_weight: torch.Tensor,
266
+ up_bias: torch.Tensor,
267
+ down_weight: torch.Tensor,
268
+ down_bias: torch.Tensor,
269
+ input_scale: torch.Tensor,
270
+ up_weight_scale: torch.Tensor,
271
+ hidden_scale: torch.Tensor,
272
+ down_weight_scale: torch.Tensor,
273
+ input_fp8: torch.Tensor | None = None,
274
+ hidden_bf16: torch.Tensor | None = None,
275
+ hidden_fp8: torch.Tensor | None = None,
276
+ out: torch.Tensor | None = None,
277
+ *,
278
+ pad_to: int | None = None,
279
+ ) -> torch.Tensor:
280
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
281
+
282
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
283
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
284
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
285
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
286
+ custom-op boundary, not a single CUDA launch.
287
+ """
288
+
289
+ logical_m = input.shape[0]
290
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
291
+ if padded_m < logical_m:
292
+ raise ValueError("pad_to must be >= input.shape[0]")
293
+ device = input.device
294
+ if input_fp8 is None:
295
+ input_fp8 = torch.empty(
296
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
297
+ )
298
+ if hidden_bf16 is None:
299
+ hidden_bf16 = torch.empty(
300
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
301
+ )
302
+ if hidden_fp8 is None:
303
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
304
+ if out is None:
305
+ out = torch.empty(
306
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
307
+ )
308
+ ops.bf16_fp8_gelu_mlp_bf16(
309
+ input,
310
+ up_weight,
311
+ up_bias,
312
+ down_weight,
313
+ down_bias,
314
+ input_scale,
315
+ up_weight_scale,
316
+ hidden_scale,
317
+ down_weight_scale,
318
+ input_fp8,
319
+ hidden_bf16,
320
+ hidden_fp8,
321
+ out,
322
+ )
323
+ return out[:logical_m]
324
+
325
+
326
  __all__ = [
327
+ "bf16_fp8_gelu_mlp_bf16",
328
  "fp8_gemm_bf16",
329
  "fp8_gelu_mlp_bf16",
330
  "fp8_linear_bias_gelu_quant_bf16",
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_4a17df5.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:387239194294371025c6fa91fd16e3ac550052f1632b73ec37b7879371cc8b9d
3
- size 145040
 
 
 
 
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_6ec89eb.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee538a560b24aea9f86af508f4f271f522a499fe70c3700f23be7daa91146332
3
+ size 165656
build/torch212-cxx11-rocm72-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_6ec89eb
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_6ec89eb
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_6ec89eb::{op_name}"
build/torch212-cxx11-rocm72-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_6ec89eb",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],