liangsu9988 commited on
Commit
1db1507
·
verified ·
1 Parent(s): 285c1c2

Promote latest kernel artifacts to main

Browse files
Files changed (39) hide show
  1. README.md +0 -9
  2. benchmarks/benchmark.py +4 -1
  3. benchmarks/benchmark_mlp_v2.py +151 -0
  4. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +158 -0
  5. build/{torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so} +2 -2
  6. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  7. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +1 -1
  8. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +158 -0
  9. build/{torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so} +2 -2
  10. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  11. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +1 -1
  12. build/torch211-cxx11-rocm71-x86_64-linux/__init__.py +158 -0
  13. build/{torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so} +2 -2
  14. build/torch211-cxx11-rocm71-x86_64-linux/_ops.py +3 -3
  15. build/torch211-cxx11-rocm71-x86_64-linux/metadata.json +1 -1
  16. build/torch211-cxx11-rocm72-x86_64-linux/__init__.py +158 -0
  17. build/{torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so} +2 -2
  18. build/torch211-cxx11-rocm72-x86_64-linux/_ops.py +3 -3
  19. build/torch211-cxx11-rocm72-x86_64-linux/metadata.json +1 -1
  20. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +158 -0
  21. build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so +0 -3
  22. build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so +3 -0
  23. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  24. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +1 -1
  25. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +158 -0
  26. build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so +0 -3
  27. build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so +3 -0
  28. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  29. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +1 -1
  30. build/torch212-cxx11-rocm71-x86_64-linux/__init__.py +158 -0
  31. build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so +0 -3
  32. build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so +3 -0
  33. build/torch212-cxx11-rocm71-x86_64-linux/_ops.py +3 -3
  34. build/torch212-cxx11-rocm71-x86_64-linux/metadata.json +1 -1
  35. build/torch212-cxx11-rocm72-x86_64-linux/__init__.py +158 -0
  36. build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so +0 -3
  37. build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so +3 -0
  38. build/torch212-cxx11-rocm72-x86_64-linux/_ops.py +3 -3
  39. build/torch212-cxx11-rocm72-x86_64-linux/metadata.json +1 -1
README.md DELETED
@@ -1,9 +0,0 @@
1
- # flashrt/flashrt-fp8-ffn
2
-
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
-
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-fp8-ffn
7
-
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
benchmarks/benchmark.py CHANGED
@@ -254,7 +254,10 @@ def load_installed_ops(artifact: str | None):
254
  def load_hub_ops(repo_id: str, version: int):
255
  from kernels import get_kernel
256
 
257
- return get_kernel(repo_id, version=version)
 
 
 
258
 
259
 
260
  def fp8_dtype() -> torch.dtype:
 
254
  def load_hub_ops(repo_id: str, version: int):
255
  from kernels import get_kernel
256
 
257
+ try:
258
+ return get_kernel(repo_id, version=version, trust_remote_code=True)
259
+ except TypeError:
260
+ return get_kernel(repo_id, version=version)
261
 
262
 
263
  def fp8_dtype() -> torch.dtype:
benchmarks/benchmark_mlp_v2.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Compare staged-bias v1 and fused-bias v2 FP8 GELU MLP entries."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import importlib
8
+ import importlib.util
9
+ import json
10
+ import os
11
+ import statistics
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ import torch
16
+
17
+
18
+ ROOT = Path(__file__).resolve().parents[2]
19
+ TEST_FILE = ROOT / "flashrt-fp8-ffn" / "tests" / "test_fp8_ffn.py"
20
+ SHAPES = {
21
+ "groot_vit": (128, 1024, 4096, 1024),
22
+ "groot_deepstack": (128, 4096, 4096, 2048),
23
+ "groot_action_dit": (41, 1536, 6144, 1536),
24
+ "pi05_decoder": (10, 1024, 4096, 1024),
25
+ }
26
+
27
+
28
+ def helpers_module():
29
+ spec = importlib.util.spec_from_file_location("fp8_ffn_helpers", TEST_FILE)
30
+ if spec is None or spec.loader is None:
31
+ raise RuntimeError(f"cannot load {TEST_FILE}")
32
+ module = importlib.util.module_from_spec(spec)
33
+ spec.loader.exec_module(module)
34
+ return module
35
+
36
+
37
+ def time_us(fn, warmup: int, iterations: int, rounds: int = 5) -> float:
38
+ samples = []
39
+ for _ in range(rounds):
40
+ for _ in range(warmup):
41
+ fn()
42
+ torch.cuda.synchronize()
43
+ start = torch.cuda.Event(enable_timing=True)
44
+ end = torch.cuda.Event(enable_timing=True)
45
+ start.record()
46
+ for _ in range(iterations):
47
+ fn()
48
+ end.record()
49
+ end.synchronize()
50
+ samples.append(start.elapsed_time(end) * 1000.0 / iterations)
51
+ return float(statistics.median(samples))
52
+
53
+
54
+ def main() -> int:
55
+ parser = argparse.ArgumentParser()
56
+ parser.add_argument("--backend", choices=("source", "installed"), default="source")
57
+ parser.add_argument("--artifact")
58
+ parser.add_argument("--warmup", type=int, default=20)
59
+ parser.add_argument("--iterations", type=int, default=100)
60
+ parser.add_argument("--json-out")
61
+ args = parser.parse_args()
62
+ helpers = helpers_module()
63
+ if args.backend == "source":
64
+ ops = helpers.load_source_ops()
65
+ else:
66
+ if args.artifact:
67
+ sys.path.insert(0, args.artifact)
68
+ try:
69
+ ops = importlib.import_module("flashrt_fp8_ffn")
70
+ finally:
71
+ if args.artifact:
72
+ sys.path.remove(args.artifact)
73
+
74
+ rows = []
75
+ for name, shape in SHAPES.items():
76
+ tensors = helpers.make_case(*shape)
77
+ m, _, h, n = shape
78
+ hidden_v1 = torch.empty((m, h), device="cuda", dtype=torch.bfloat16)
79
+ hidden_fp8_v1 = torch.empty_like(hidden_v1, dtype=helpers.fp8_dtype())
80
+ out_v1_buf = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
81
+ hidden_v2 = torch.empty_like(hidden_v1)
82
+ hidden_fp8_v2 = torch.empty_like(hidden_fp8_v1)
83
+ out_v2_buf = torch.empty_like(out_v1_buf)
84
+ if args.backend == "source":
85
+ v1 = lambda: ops.fp8_gelu_mlp_bf16(
86
+ *tensors, hidden=hidden_v1, hidden_fp8=hidden_fp8_v1,
87
+ out=out_v1_buf,
88
+ )
89
+ v2 = lambda: ops.fp8_gelu_mlp_v2_bf16(
90
+ *tensors, hidden=hidden_v2, hidden_fp8=hidden_fp8_v2,
91
+ out=out_v2_buf,
92
+ )
93
+ else:
94
+ v1 = lambda: ops.fp8_gelu_mlp_bf16(
95
+ *tensors, hidden_bf16=hidden_v1, hidden_fp8=hidden_fp8_v1,
96
+ out=out_v1_buf,
97
+ )
98
+ v2 = lambda: ops.fp8_gelu_mlp_v2_bf16(
99
+ *tensors, hidden_bf16=hidden_v2, hidden_fp8=hidden_fp8_v2,
100
+ out=out_v2_buf,
101
+ )
102
+ out_v1 = v1()
103
+ out_v2 = v2()
104
+ torch.cuda.synchronize()
105
+ old = os.environ.get("FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE")
106
+ os.environ["FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE"] = "1"
107
+ try:
108
+ v2()
109
+ torch.cuda.synchronize()
110
+ fused_hit = True
111
+ except RuntimeError as error:
112
+ if "BIAS epilogue was required" not in str(error):
113
+ raise
114
+ fused_hit = False
115
+ finally:
116
+ if old is None:
117
+ os.environ.pop("FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE", None)
118
+ else:
119
+ os.environ["FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE"] = old
120
+ metrics = helpers.distribution_metrics(out_v2, out_v1)
121
+ v1_us = time_us(v1, args.warmup, args.iterations)
122
+ v2_us = time_us(v2, args.warmup, args.iterations)
123
+ row = {
124
+ "shape": name,
125
+ "M": shape[0],
126
+ "K": shape[1],
127
+ "H": shape[2],
128
+ "N": shape[3],
129
+ "fused_bias_epilogue": fused_hit,
130
+ "v1_us": v1_us,
131
+ "v2_us": v2_us,
132
+ "speedup": v1_us / v2_us,
133
+ **metrics,
134
+ }
135
+ rows.append(row)
136
+ print(json.dumps(row, sort_keys=True), flush=True)
137
+
138
+ result = {
139
+ "device": torch.cuda.get_device_name(),
140
+ "capability": list(torch.cuda.get_device_capability()),
141
+ "torch": torch.__version__,
142
+ "cuda": torch.version.cuda,
143
+ "rows": rows,
144
+ }
145
+ if args.json_out:
146
+ Path(args.json_out).write_text(json.dumps(result, indent=2) + "\n")
147
+ return 0
148
+
149
+
150
+ if __name__ == "__main__":
151
+ raise SystemExit(main())
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/{torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:01f36636ddf259f85c94157b09cb5267b9cd4b371b148acd5150f4f3c13f3b78
3
- size 178992
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46aff28a95fa0aa348046c6845ed289baa67c10c39c1b9936afcab4a709f3d07
3
+ size 703520
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/{torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3d274681fba749480e852c3da2670c29041914b6edfb234ebce6e46bec1afa8c
3
- size 698640
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f6a225f67637c94184bc64dfb3c93c23a0e5e0f3e7957ba20a1a01874d21f5f
3
+ size 673200
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch211-cxx11-rocm71-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/{torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2636112b16bd21bec7d249cf99d86206b873df2f8672fe47a560e57171b55cd2
3
- size 668320
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:409804cf1498f2d98af1c3cf5ba70dd4d3e8a8c6e1ab8725668371d329797711
3
+ size 187352
build/torch211-cxx11-rocm71-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
build/torch211-cxx11-rocm71-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch211-cxx11-rocm72-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/{torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b0a6f1d91521ad7db13bb2f702ce4e74c4f54aa5e3666c6a11c52bd4ecc1c31d
3
- size 180528
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6f65669c49255f4bce57e5bcd24f0298fe67147bdf0f495ea827d83fb4e9eaf
3
+ size 185816
build/torch211-cxx11-rocm72-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
build/torch211-cxx11-rocm72-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9cf974f0a6585860d33d00d4269205d2046cbaa8e74442d4e289ec8ea7cddc6e
3
- size 668544
 
 
 
 
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a95a3b6e1f95822e181bd4a01f425ef5fbe8dcadcabe684f59bd33c76987ed9
3
+ size 673144
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:92f9e9ba898c08f78f8ea086de134a0f1dcfb1201ce9d04ca932d4ed30844919
3
- size 668544
 
 
 
 
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81094023a0b91b9bd217f3bce768851bcf0923776bb17b6e8d2f192bd20ef063
3
+ size 673144
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-rocm71-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6203acdac17af6b44944b57bb831a4584a39b8ad401f0e8cd58a7c2b5760862e
3
- size 180000
 
 
 
 
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8dc1bfc079ae1d9ee4a6f535ad36ee88fcc380f6b59ec075fab1116e49ef8b0a
3
+ size 187584
build/torch212-cxx11-rocm71-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
build/torch212-cxx11-rocm71-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
build/torch212-cxx11-rocm72-x86_64-linux/__init__.py CHANGED
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
145
  return None
146
 
147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
  def _bf16_fp8_gelu_mlp_bf16_fake(
150
  input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
173
  return None
174
 
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
433
  return out[:logical_m]
434
 
435
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  __all__ = [
437
  "bf16_fp8_linear_bias_bf16",
438
  "bf16_fp8_gelu_mlp_bf16",
 
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
 
441
  "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
149
+ def _fp8_gelu_mlp_v2_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ hidden_bf16: torch.Tensor,
160
+ hidden_fp8: torch.Tensor,
161
+ out: torch.Tensor,
162
+ ) -> None:
163
+ return _fp8_gelu_mlp_bf16_fake(
164
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
165
+ up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
166
+ hidden_fp8, out)
167
+
168
+
169
  @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
170
  def _bf16_fp8_gelu_mlp_bf16_fake(
171
  input: torch.Tensor,
 
194
  return None
195
 
196
 
197
+ @torch.library.register_fake(
198
+ add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
199
+ def _bf16_fp8_gelu_mlp_v2_bf16_fake(
200
+ input: torch.Tensor,
201
+ up_weight: torch.Tensor,
202
+ up_bias: torch.Tensor,
203
+ down_weight: torch.Tensor,
204
+ down_bias: torch.Tensor,
205
+ input_scale: torch.Tensor,
206
+ up_weight_scale: torch.Tensor,
207
+ hidden_scale: torch.Tensor,
208
+ down_weight_scale: torch.Tensor,
209
+ input_fp8: torch.Tensor,
210
+ hidden_bf16: torch.Tensor,
211
+ hidden_fp8: torch.Tensor,
212
+ out: torch.Tensor,
213
+ ) -> None:
214
+ return _bf16_fp8_gelu_mlp_bf16_fake(
215
+ input, up_weight, up_bias, down_weight, down_bias, input_scale,
216
+ up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
217
+ hidden_bf16, hidden_fp8, out)
218
+
219
+
220
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
221
  return torch.tensor([value], device=input.device, dtype=torch.float32)
222
 
 
477
  return out[:logical_m]
478
 
479
 
480
+ def fp8_gelu_mlp_v2_bf16(
481
+ input: torch.Tensor,
482
+ up_weight: torch.Tensor,
483
+ up_bias: torch.Tensor,
484
+ down_weight: torch.Tensor,
485
+ down_bias: torch.Tensor,
486
+ input_scale: torch.Tensor,
487
+ up_weight_scale: torch.Tensor,
488
+ hidden_scale: torch.Tensor,
489
+ down_weight_scale: torch.Tensor,
490
+ hidden_bf16: torch.Tensor | None = None,
491
+ hidden_fp8: torch.Tensor | None = None,
492
+ out: torch.Tensor | None = None,
493
+ ) -> torch.Tensor:
494
+ """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
495
+
496
+ Same contract and buffers as the v1 entry; one launch and one full
497
+ write of ``out`` less per call on targets whose cublasLt path
498
+ accepts the BIAS epilogue (elsewhere the runtime falls back to the
499
+ v1 two-launch tail, still correct).
500
+ """
501
+
502
+ if hidden_bf16 is None:
503
+ hidden_bf16 = torch.empty(
504
+ (input.shape[0], up_weight.shape[0]),
505
+ device=input.device,
506
+ dtype=torch.bfloat16,
507
+ )
508
+ if hidden_fp8 is None:
509
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
510
+ if out is None:
511
+ out = torch.empty(
512
+ (input.shape[0], down_weight.shape[0]),
513
+ device=input.device,
514
+ dtype=torch.bfloat16,
515
+ )
516
+ ops.fp8_gelu_mlp_v2_bf16(
517
+ input,
518
+ up_weight,
519
+ up_bias,
520
+ down_weight,
521
+ down_bias,
522
+ input_scale,
523
+ up_weight_scale,
524
+ hidden_scale,
525
+ down_weight_scale,
526
+ hidden_bf16,
527
+ hidden_fp8,
528
+ out,
529
+ )
530
+ return out
531
+
532
+
533
+ def bf16_fp8_gelu_mlp_v2_bf16(
534
+ input: torch.Tensor,
535
+ up_weight: torch.Tensor,
536
+ up_bias: torch.Tensor,
537
+ down_weight: torch.Tensor,
538
+ down_bias: torch.Tensor,
539
+ input_scale: torch.Tensor,
540
+ up_weight_scale: torch.Tensor,
541
+ hidden_scale: torch.Tensor,
542
+ down_weight_scale: torch.Tensor,
543
+ input_fp8: torch.Tensor | None = None,
544
+ hidden_bf16: torch.Tensor | None = None,
545
+ hidden_fp8: torch.Tensor | None = None,
546
+ out: torch.Tensor | None = None,
547
+ *,
548
+ pad_to: int | None = None,
549
+ ) -> torch.Tensor:
550
+ """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
551
+
552
+ Identical padding and buffer contract to the v1 entry.
553
+ """
554
+
555
+ logical_m = input.shape[0]
556
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
557
+ if padded_m < logical_m:
558
+ raise ValueError("pad_to must be >= input.shape[0]")
559
+ device = input.device
560
+ if input_fp8 is None:
561
+ input_fp8 = torch.empty(
562
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
563
+ )
564
+ if hidden_bf16 is None:
565
+ hidden_bf16 = torch.empty(
566
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
567
+ )
568
+ if hidden_fp8 is None:
569
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
570
+ if out is None:
571
+ out = torch.empty(
572
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
573
+ )
574
+ ops.bf16_fp8_gelu_mlp_v2_bf16(
575
+ input,
576
+ up_weight,
577
+ up_bias,
578
+ down_weight,
579
+ down_bias,
580
+ input_scale,
581
+ up_weight_scale,
582
+ hidden_scale,
583
+ down_weight_scale,
584
+ input_fp8,
585
+ hidden_bf16,
586
+ hidden_fp8,
587
+ out,
588
+ )
589
+ return out[:logical_m]
590
+
591
+
592
  __all__ = [
593
  "bf16_fp8_linear_bias_bf16",
594
  "bf16_fp8_gelu_mlp_bf16",
595
+ "bf16_fp8_gelu_mlp_v2_bf16",
596
  "fp8_gemm_bf16",
597
  "fp8_gelu_mlp_bf16",
598
+ "fp8_gelu_mlp_v2_bf16",
599
  "fp8_linear_bias_bf16",
600
  "fp8_linear_bias_gelu_quant_bf16",
601
  ]
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:eb09062fde7313db39fe558ff36f8b902b04ac06fade78310ccd1eb27ad3d90b
3
- size 178424
 
 
 
 
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:860c6f79cb15466319a99fc5df4f1651d1040a8d074d9df37ecdf8f86b1985b7
3
+ size 186040
build/torch212-cxx11-rocm72-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_rocm_255584f
3
- ops = torch.ops._flashrt_fp8_ffn_rocm_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_flashrt_fp8_ffn_rocm_255584f::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_rocm_b2eeca0
3
+ ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
build/torch212-cxx11-rocm72-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_rocm_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_rocm_b2eeca0",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],