liangsu9988 commited on
Commit
ef6d21e
·
verified ·
1 Parent(s): 7b9c0f6

Promote latest kernel artifacts to main

Browse files
README.md DELETED
@@ -1,9 +0,0 @@
1
- # flashrt/flashrt-fp8-ffn
2
-
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
-
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-fp8-ffn
7
-
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
build/torch213-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,601 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP8 FFN kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import ctypes.util
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import torch
11
+
12
+
13
+ def _torch_bundled_cublaslt() -> Optional[Path]:
14
+ for parent in Path(torch.__file__).resolve().parents:
15
+ candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
16
+ if candidate.exists():
17
+ return candidate
18
+ return None
19
+
20
+
21
+ def _preload_cublaslt() -> None:
22
+ bundled = _torch_bundled_cublaslt()
23
+ library = str(bundled) if bundled is not None else (
24
+ ctypes.util.find_library("cublasLt") or "libcublasLt.so"
25
+ )
26
+ try:
27
+ ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
28
+ except OSError:
29
+ pass
30
+
31
+
32
+ _preload_cublaslt()
33
+
34
+ from ._ops import add_op_namespace_prefix, ops
35
+
36
+
37
+ def _fp8_dtype() -> torch.dtype:
38
+ if torch.version.hip is not None and hasattr(torch, "float8_e4m3fnuz"):
39
+ return torch.float8_e4m3fnuz
40
+ return torch.float8_e4m3fn
41
+
42
+
43
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gemm_bf16"))
44
+ def _fp8_gemm_bf16_fake(
45
+ input: torch.Tensor,
46
+ weight: torch.Tensor,
47
+ input_scale: torch.Tensor,
48
+ weight_scale: torch.Tensor,
49
+ out: torch.Tensor,
50
+ ) -> None:
51
+ if input.dim() != 2 or weight.dim() != 2:
52
+ raise RuntimeError("input and weight must be rank-2 tensors")
53
+ if out.shape != (input.shape[0], weight.shape[0]):
54
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
55
+ return None
56
+
57
+
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_bf16"))
59
+ def _fp8_linear_bias_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ out: torch.Tensor,
66
+ ) -> None:
67
+ if input.dim() != 2 or weight.dim() != 2:
68
+ raise RuntimeError("input and weight must be rank-2 tensors")
69
+ if bias.shape != (weight.shape[0],):
70
+ raise RuntimeError("bias shape must be (weight.shape[0],)")
71
+ if out.shape != (input.shape[0], weight.shape[0]):
72
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
73
+ return None
74
+
75
+
76
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_linear_bias_bf16"))
77
+ def _bf16_fp8_linear_bias_bf16_fake(
78
+ input: torch.Tensor,
79
+ weight: torch.Tensor,
80
+ bias: torch.Tensor,
81
+ input_scale: torch.Tensor,
82
+ weight_scale: torch.Tensor,
83
+ input_fp8: torch.Tensor,
84
+ out: torch.Tensor,
85
+ ) -> None:
86
+ if input.dim() != 2 or weight.dim() != 2:
87
+ raise RuntimeError("input and weight must be rank-2 tensors")
88
+ if input.shape[1] != weight.shape[1]:
89
+ raise RuntimeError("input.shape[1] must match weight.shape[1]")
90
+ padded_m = input_fp8.shape[0]
91
+ if padded_m < input.shape[0]:
92
+ raise RuntimeError("input_fp8 padded rows must cover input rows")
93
+ if input_fp8.shape != (padded_m, input.shape[1]):
94
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
95
+ if bias.shape != (weight.shape[0],):
96
+ raise RuntimeError("bias shape must be (weight.shape[0],)")
97
+ if out.shape != (padded_m, weight.shape[0]):
98
+ raise RuntimeError("out must have shape (padded_m, weight.shape[0])")
99
+ return None
100
+
101
+
102
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
103
+ def _fp8_linear_bias_gelu_quant_bf16_fake(
104
+ input: torch.Tensor,
105
+ weight: torch.Tensor,
106
+ bias: torch.Tensor,
107
+ input_scale: torch.Tensor,
108
+ weight_scale: torch.Tensor,
109
+ output_scale: torch.Tensor,
110
+ hidden_bf16: torch.Tensor,
111
+ out_fp8: torch.Tensor,
112
+ ) -> None:
113
+ expected = (input.shape[0], weight.shape[0])
114
+ if hidden_bf16.shape != expected or out_fp8.shape != expected:
115
+ raise RuntimeError(
116
+ "hidden_bf16 and out_fp8 shapes must be "
117
+ "(input.shape[0], weight.shape[0])"
118
+ )
119
+ return None
120
+
121
+
122
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_bf16"))
123
+ def _fp8_gelu_mlp_bf16_fake(
124
+ input: torch.Tensor,
125
+ up_weight: torch.Tensor,
126
+ up_bias: torch.Tensor,
127
+ down_weight: torch.Tensor,
128
+ down_bias: torch.Tensor,
129
+ input_scale: torch.Tensor,
130
+ up_weight_scale: torch.Tensor,
131
+ hidden_scale: torch.Tensor,
132
+ down_weight_scale: torch.Tensor,
133
+ hidden_bf16: torch.Tensor,
134
+ hidden_fp8: torch.Tensor,
135
+ out: torch.Tensor,
136
+ ) -> None:
137
+ hidden_shape = (input.shape[0], up_weight.shape[0])
138
+ out_shape = (input.shape[0], down_weight.shape[0])
139
+ if hidden_bf16.shape != hidden_shape or hidden_fp8.shape != hidden_shape:
140
+ raise RuntimeError(
141
+ "hidden buffers must be (input.shape[0], up_weight.shape[0])"
142
+ )
143
+ if out.shape != out_shape:
144
+ raise RuntimeError("out shape must be (input.shape[0], down_weight.shape[0])")
145
+ return None
146
+
147
+
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
+ def _bf16_fp8_gelu_mlp_bf16_fake(
171
+ input: torch.Tensor,
172
+ up_weight: torch.Tensor,
173
+ up_bias: torch.Tensor,
174
+ down_weight: torch.Tensor,
175
+ down_bias: torch.Tensor,
176
+ input_scale: torch.Tensor,
177
+ up_weight_scale: torch.Tensor,
178
+ hidden_scale: torch.Tensor,
179
+ down_weight_scale: torch.Tensor,
180
+ input_fp8: torch.Tensor,
181
+ hidden_bf16: torch.Tensor,
182
+ hidden_fp8: torch.Tensor,
183
+ out: torch.Tensor,
184
+ ) -> None:
185
+ padded_m = input_fp8.shape[0]
186
+ if input_fp8.shape != (padded_m, input.shape[1]):
187
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
188
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
189
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
190
+ if hidden_fp8.shape != hidden_bf16.shape:
191
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
192
+ if out.shape != (padded_m, down_weight.shape[0]):
193
+ raise RuntimeError("out has an invalid padded shape")
194
+ return None
195
+
196
+
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
+ def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
+ return torch.tensor([value], device=input.device, dtype=torch.float32)
222
+
223
+
224
+ def fp8_gemm_bf16(
225
+ input: torch.Tensor,
226
+ weight: torch.Tensor,
227
+ input_scale: torch.Tensor,
228
+ weight_scale: torch.Tensor,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ """Compute ``(input * input_scale) @ (weight * weight_scale).T``.
232
+
233
+ ``input`` is FP8 E4M3 with shape ``(M, K)``. ``weight`` is FP8 E4M3 with
234
+ shape ``(N, K)``. ``input_scale`` and ``weight_scale`` are CUDA float32
235
+ scalar tensors. Output is BF16 with shape ``(M, N)``.
236
+ """
237
+
238
+ if out is None:
239
+ out = torch.empty(
240
+ (input.shape[0], weight.shape[0]),
241
+ device=input.device,
242
+ dtype=torch.bfloat16,
243
+ )
244
+ ops.fp8_gemm_bf16(input, weight, input_scale, weight_scale, out)
245
+ return out
246
+
247
+
248
+ def fp8_linear_bias_bf16(
249
+ input: torch.Tensor,
250
+ weight: torch.Tensor,
251
+ bias: torch.Tensor,
252
+ input_scale: torch.Tensor,
253
+ weight_scale: torch.Tensor,
254
+ out: torch.Tensor | None = None,
255
+ ) -> torch.Tensor:
256
+ """Run an FP8-weight linear projection with BF16 bias and output.
257
+
258
+ ``input`` and ``weight`` use FP8 E4M3 with shapes ``(M, K)`` and
259
+ ``(N, K)``. Scales are device float32 scalar tensors. Pass ``out`` to keep
260
+ the call allocation-free for CUDA Graph capture.
261
+ """
262
+
263
+ if out is None:
264
+ out = torch.empty(
265
+ (input.shape[0], weight.shape[0]),
266
+ device=input.device,
267
+ dtype=torch.bfloat16,
268
+ )
269
+ ops.fp8_linear_bias_bf16(
270
+ input, weight, bias, input_scale, weight_scale, out
271
+ )
272
+ return out
273
+
274
+
275
+ def fp8_linear_bias_gelu_quant_bf16(
276
+ input: torch.Tensor,
277
+ weight: torch.Tensor,
278
+ bias: torch.Tensor,
279
+ input_scale: torch.Tensor,
280
+ weight_scale: torch.Tensor,
281
+ output_scale: torch.Tensor,
282
+ hidden_bf16: torch.Tensor | None = None,
283
+ out_fp8: torch.Tensor | None = None,
284
+ ) -> tuple[torch.Tensor, torch.Tensor]:
285
+ """FP8 linear + BF16 bias/GELU + FP8 quantized output.
286
+
287
+ Returns ``(hidden_bf16, out_fp8)``. ``hidden_bf16`` is the post-GEMM
288
+ pre-activation scratch; ``out_fp8`` is the quantized activation.
289
+ """
290
+
291
+ if hidden_bf16 is None:
292
+ hidden_bf16 = torch.empty(
293
+ (input.shape[0], weight.shape[0]),
294
+ device=input.device,
295
+ dtype=torch.bfloat16,
296
+ )
297
+ if out_fp8 is None:
298
+ out_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
299
+ ops.fp8_linear_bias_gelu_quant_bf16(
300
+ input,
301
+ weight,
302
+ bias,
303
+ input_scale,
304
+ weight_scale,
305
+ output_scale,
306
+ hidden_bf16,
307
+ out_fp8,
308
+ )
309
+ return hidden_bf16, out_fp8
310
+
311
+
312
+ def fp8_gelu_mlp_bf16(
313
+ input: torch.Tensor,
314
+ up_weight: torch.Tensor,
315
+ up_bias: torch.Tensor,
316
+ down_weight: torch.Tensor,
317
+ down_bias: torch.Tensor,
318
+ input_scale: torch.Tensor,
319
+ up_weight_scale: torch.Tensor,
320
+ hidden_scale: torch.Tensor,
321
+ down_weight_scale: torch.Tensor,
322
+ hidden_bf16: torch.Tensor | None = None,
323
+ hidden_fp8: torch.Tensor | None = None,
324
+ out: torch.Tensor | None = None,
325
+ ) -> torch.Tensor:
326
+ """FP8 GELU MLP block with BF16 output.
327
+
328
+ Computes:
329
+
330
+ ``hidden = gelu(fp8_gemm(input, up_weight) + up_bias)``
331
+ ``hidden_fp8 = quantize_fp8(hidden, hidden_scale)``
332
+ ``out = fp8_gemm(hidden_fp8, down_weight) + down_bias``
333
+ """
334
+
335
+ if hidden_bf16 is None:
336
+ hidden_bf16 = torch.empty(
337
+ (input.shape[0], up_weight.shape[0]),
338
+ device=input.device,
339
+ dtype=torch.bfloat16,
340
+ )
341
+ if hidden_fp8 is None:
342
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
343
+ if out is None:
344
+ out = torch.empty(
345
+ (input.shape[0], down_weight.shape[0]),
346
+ device=input.device,
347
+ dtype=torch.bfloat16,
348
+ )
349
+ ops.fp8_gelu_mlp_bf16(
350
+ input,
351
+ up_weight,
352
+ up_bias,
353
+ down_weight,
354
+ down_bias,
355
+ input_scale,
356
+ up_weight_scale,
357
+ hidden_scale,
358
+ down_weight_scale,
359
+ hidden_bf16,
360
+ hidden_fp8,
361
+ out,
362
+ )
363
+ return out
364
+
365
+
366
+ def _midm_padded_rows(input: torch.Tensor) -> int:
367
+ rows = input.shape[0]
368
+ if (
369
+ torch.version.hip is None
370
+ and input.is_cuda
371
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
372
+ and 9 <= rows <= 128
373
+ ):
374
+ return ((rows + 63) // 64) * 64
375
+ return rows
376
+
377
+
378
+ def bf16_fp8_linear_bias_bf16(
379
+ input: torch.Tensor,
380
+ weight: torch.Tensor,
381
+ bias: torch.Tensor,
382
+ input_scale: torch.Tensor,
383
+ weight_scale: torch.Tensor,
384
+ input_fp8: torch.Tensor | None = None,
385
+ out: torch.Tensor | None = None,
386
+ *,
387
+ pad_to: int | None = None,
388
+ ) -> torch.Tensor:
389
+ """Run a static-scale FP8 linear projection from a BF16 boundary.
390
+
391
+ This traceable region entry quantizes ``input`` and computes the FP8
392
+ projection with BF16 bias/output. Thor SM110 pads M=9..128 to a multiple of
393
+ 64 by default. Preallocate ``input_fp8`` and ``out`` for allocation-free
394
+ CUDA Graph replay.
395
+ """
396
+
397
+ logical_m = input.shape[0]
398
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
399
+ if padded_m < logical_m:
400
+ raise ValueError("pad_to must be >= input.shape[0]")
401
+ if input_fp8 is None:
402
+ input_fp8 = torch.empty(
403
+ (padded_m, input.shape[1]), device=input.device, dtype=_fp8_dtype()
404
+ )
405
+ if out is None:
406
+ out = torch.empty(
407
+ (padded_m, weight.shape[0]),
408
+ device=input.device,
409
+ dtype=torch.bfloat16,
410
+ )
411
+ ops.bf16_fp8_linear_bias_bf16(
412
+ input, weight, bias, input_scale, weight_scale, input_fp8, out
413
+ )
414
+ return out[:logical_m]
415
+
416
+
417
+ def bf16_fp8_gelu_mlp_bf16(
418
+ input: torch.Tensor,
419
+ up_weight: torch.Tensor,
420
+ up_bias: torch.Tensor,
421
+ down_weight: torch.Tensor,
422
+ down_bias: torch.Tensor,
423
+ input_scale: torch.Tensor,
424
+ up_weight_scale: torch.Tensor,
425
+ hidden_scale: torch.Tensor,
426
+ down_weight_scale: torch.Tensor,
427
+ input_fp8: torch.Tensor | None = None,
428
+ hidden_bf16: torch.Tensor | None = None,
429
+ hidden_fp8: torch.Tensor | None = None,
430
+ out: torch.Tensor | None = None,
431
+ *,
432
+ pad_to: int | None = None,
433
+ ) -> torch.Tensor:
434
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
435
+
436
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
437
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
438
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
439
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
440
+ custom-op boundary, not a single CUDA launch.
441
+ """
442
+
443
+ logical_m = input.shape[0]
444
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
445
+ if padded_m < logical_m:
446
+ raise ValueError("pad_to must be >= input.shape[0]")
447
+ device = input.device
448
+ if input_fp8 is None:
449
+ input_fp8 = torch.empty(
450
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
451
+ )
452
+ if hidden_bf16 is None:
453
+ hidden_bf16 = torch.empty(
454
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
455
+ )
456
+ if hidden_fp8 is None:
457
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
458
+ if out is None:
459
+ out = torch.empty(
460
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
461
+ )
462
+ ops.bf16_fp8_gelu_mlp_bf16(
463
+ input,
464
+ up_weight,
465
+ up_bias,
466
+ down_weight,
467
+ down_bias,
468
+ input_scale,
469
+ up_weight_scale,
470
+ hidden_scale,
471
+ down_weight_scale,
472
+ input_fp8,
473
+ hidden_bf16,
474
+ hidden_fp8,
475
+ out,
476
+ )
477
+ return out[:logical_m]
478
+
479
+
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
+ __all__ = [
593
+ "bf16_fp8_linear_bias_bf16",
594
+ "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
+ "fp8_gemm_bf16",
597
+ "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
+ "fp8_linear_bias_bf16",
600
+ "fp8_linear_bias_gelu_quant_bf16",
601
+ ]
build/torch213-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_83e3875.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ccc3ee278136473f39af8ff00ccc9cf91355d4d01b0c8ea587c1ec128b34a18c
3
+ size 672992
build/torch213-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_fp8_ffn_cuda_83e3875
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_83e3875
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_fp8_ffn_cuda_83e3875::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/flashrt_fp8_ffn/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch213-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_83e3875",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ },
22
+ "digest": {
23
+ "algorithm": "sha256",
24
+ "files": {
25
+ "__init__.py": "O5OvnWgzbfzKB83CtQwQy60BSAa83vizBgNuMnx6D6A=",
26
+ "_flashrt_fp8_ffn_cuda_83e3875.abi3.so": "zMPuJ4E2Rz85r4/wDMyc+RNV1NAbDI6lh8HsEos0oYw=",
27
+ "_ops.py": "z6MKuuWVqLRmms4H9SHn7FX0eBgGrwfRHHsvyj2Vxn0=",
28
+ "flashrt_fp8_ffn/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
29
+ }
30
+ },
31
+ "provenance": {
32
+ "kernel-builder": {
33
+ "version": "0.17.0-dev0",
34
+ "sha": "b39ca23f1b36383df00b27b3ffe1276cd5dbea85",
35
+ "dirty": false
36
+ },
37
+ "kernel": {
38
+ "sha": "83e3875c62f7d0b1af0c21f128cf69e59e9d6e1f",
39
+ "dirty": false
40
+ }
41
+ }
42
+ }