Promote latest kernel artifacts to main
Browse files- README.md +0 -9
- benchmarks/benchmark.py +4 -1
- benchmarks/benchmark_mlp_v2.py +151 -0
- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +158 -0
- build/{torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so} +2 -2
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +1 -1
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +158 -0
- build/{torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so} +2 -2
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +1 -1
- build/torch211-cxx11-rocm71-x86_64-linux/__init__.py +158 -0
- build/{torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so} +2 -2
- build/torch211-cxx11-rocm71-x86_64-linux/_ops.py +3 -3
- build/torch211-cxx11-rocm71-x86_64-linux/metadata.json +1 -1
- build/torch211-cxx11-rocm72-x86_64-linux/__init__.py +158 -0
- build/{torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so} +2 -2
- build/torch211-cxx11-rocm72-x86_64-linux/_ops.py +3 -3
- build/torch211-cxx11-rocm72-x86_64-linux/metadata.json +1 -1
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +158 -0
- build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so +0 -3
- build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so +3 -0
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +1 -1
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +158 -0
- build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so +0 -3
- build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so +3 -0
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +1 -1
- build/torch212-cxx11-rocm71-x86_64-linux/__init__.py +158 -0
- build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so +0 -3
- build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so +3 -0
- build/torch212-cxx11-rocm71-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-rocm71-x86_64-linux/metadata.json +1 -1
- build/torch212-cxx11-rocm72-x86_64-linux/__init__.py +158 -0
- build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so +0 -3
- build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so +3 -0
- build/torch212-cxx11-rocm72-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-rocm72-x86_64-linux/metadata.json +1 -1
README.md
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
# flashrt/flashrt-fp8-ffn
|
| 2 |
-
|
| 3 |
-
This repository is a compatibility mirror for older `kernels` clients
|
| 4 |
-
that resolve repositories through the default Hugging Face model repo API.
|
| 5 |
-
|
| 6 |
-
Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-fp8-ffn
|
| 7 |
-
|
| 8 |
-
Do not edit this mirror by hand. It is generated from the Kernel Hub
|
| 9 |
-
`vN` branches and contains the same `build/**` artifacts.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
benchmarks/benchmark.py
CHANGED
|
@@ -254,7 +254,10 @@ def load_installed_ops(artifact: str | None):
|
|
| 254 |
def load_hub_ops(repo_id: str, version: int):
|
| 255 |
from kernels import get_kernel
|
| 256 |
|
| 257 |
-
|
|
|
|
|
|
|
|
|
|
| 258 |
|
| 259 |
|
| 260 |
def fp8_dtype() -> torch.dtype:
|
|
|
|
| 254 |
def load_hub_ops(repo_id: str, version: int):
|
| 255 |
from kernels import get_kernel
|
| 256 |
|
| 257 |
+
try:
|
| 258 |
+
return get_kernel(repo_id, version=version, trust_remote_code=True)
|
| 259 |
+
except TypeError:
|
| 260 |
+
return get_kernel(repo_id, version=version)
|
| 261 |
|
| 262 |
|
| 263 |
def fp8_dtype() -> torch.dtype:
|
benchmarks/benchmark_mlp_v2.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Compare staged-bias v1 and fused-bias v2 FP8 GELU MLP entries."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import importlib
|
| 8 |
+
import importlib.util
|
| 9 |
+
import json
|
| 10 |
+
import os
|
| 11 |
+
import statistics
|
| 12 |
+
import sys
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 19 |
+
TEST_FILE = ROOT / "flashrt-fp8-ffn" / "tests" / "test_fp8_ffn.py"
|
| 20 |
+
SHAPES = {
|
| 21 |
+
"groot_vit": (128, 1024, 4096, 1024),
|
| 22 |
+
"groot_deepstack": (128, 4096, 4096, 2048),
|
| 23 |
+
"groot_action_dit": (41, 1536, 6144, 1536),
|
| 24 |
+
"pi05_decoder": (10, 1024, 4096, 1024),
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def helpers_module():
|
| 29 |
+
spec = importlib.util.spec_from_file_location("fp8_ffn_helpers", TEST_FILE)
|
| 30 |
+
if spec is None or spec.loader is None:
|
| 31 |
+
raise RuntimeError(f"cannot load {TEST_FILE}")
|
| 32 |
+
module = importlib.util.module_from_spec(spec)
|
| 33 |
+
spec.loader.exec_module(module)
|
| 34 |
+
return module
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def time_us(fn, warmup: int, iterations: int, rounds: int = 5) -> float:
|
| 38 |
+
samples = []
|
| 39 |
+
for _ in range(rounds):
|
| 40 |
+
for _ in range(warmup):
|
| 41 |
+
fn()
|
| 42 |
+
torch.cuda.synchronize()
|
| 43 |
+
start = torch.cuda.Event(enable_timing=True)
|
| 44 |
+
end = torch.cuda.Event(enable_timing=True)
|
| 45 |
+
start.record()
|
| 46 |
+
for _ in range(iterations):
|
| 47 |
+
fn()
|
| 48 |
+
end.record()
|
| 49 |
+
end.synchronize()
|
| 50 |
+
samples.append(start.elapsed_time(end) * 1000.0 / iterations)
|
| 51 |
+
return float(statistics.median(samples))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def main() -> int:
|
| 55 |
+
parser = argparse.ArgumentParser()
|
| 56 |
+
parser.add_argument("--backend", choices=("source", "installed"), default="source")
|
| 57 |
+
parser.add_argument("--artifact")
|
| 58 |
+
parser.add_argument("--warmup", type=int, default=20)
|
| 59 |
+
parser.add_argument("--iterations", type=int, default=100)
|
| 60 |
+
parser.add_argument("--json-out")
|
| 61 |
+
args = parser.parse_args()
|
| 62 |
+
helpers = helpers_module()
|
| 63 |
+
if args.backend == "source":
|
| 64 |
+
ops = helpers.load_source_ops()
|
| 65 |
+
else:
|
| 66 |
+
if args.artifact:
|
| 67 |
+
sys.path.insert(0, args.artifact)
|
| 68 |
+
try:
|
| 69 |
+
ops = importlib.import_module("flashrt_fp8_ffn")
|
| 70 |
+
finally:
|
| 71 |
+
if args.artifact:
|
| 72 |
+
sys.path.remove(args.artifact)
|
| 73 |
+
|
| 74 |
+
rows = []
|
| 75 |
+
for name, shape in SHAPES.items():
|
| 76 |
+
tensors = helpers.make_case(*shape)
|
| 77 |
+
m, _, h, n = shape
|
| 78 |
+
hidden_v1 = torch.empty((m, h), device="cuda", dtype=torch.bfloat16)
|
| 79 |
+
hidden_fp8_v1 = torch.empty_like(hidden_v1, dtype=helpers.fp8_dtype())
|
| 80 |
+
out_v1_buf = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
|
| 81 |
+
hidden_v2 = torch.empty_like(hidden_v1)
|
| 82 |
+
hidden_fp8_v2 = torch.empty_like(hidden_fp8_v1)
|
| 83 |
+
out_v2_buf = torch.empty_like(out_v1_buf)
|
| 84 |
+
if args.backend == "source":
|
| 85 |
+
v1 = lambda: ops.fp8_gelu_mlp_bf16(
|
| 86 |
+
*tensors, hidden=hidden_v1, hidden_fp8=hidden_fp8_v1,
|
| 87 |
+
out=out_v1_buf,
|
| 88 |
+
)
|
| 89 |
+
v2 = lambda: ops.fp8_gelu_mlp_v2_bf16(
|
| 90 |
+
*tensors, hidden=hidden_v2, hidden_fp8=hidden_fp8_v2,
|
| 91 |
+
out=out_v2_buf,
|
| 92 |
+
)
|
| 93 |
+
else:
|
| 94 |
+
v1 = lambda: ops.fp8_gelu_mlp_bf16(
|
| 95 |
+
*tensors, hidden_bf16=hidden_v1, hidden_fp8=hidden_fp8_v1,
|
| 96 |
+
out=out_v1_buf,
|
| 97 |
+
)
|
| 98 |
+
v2 = lambda: ops.fp8_gelu_mlp_v2_bf16(
|
| 99 |
+
*tensors, hidden_bf16=hidden_v2, hidden_fp8=hidden_fp8_v2,
|
| 100 |
+
out=out_v2_buf,
|
| 101 |
+
)
|
| 102 |
+
out_v1 = v1()
|
| 103 |
+
out_v2 = v2()
|
| 104 |
+
torch.cuda.synchronize()
|
| 105 |
+
old = os.environ.get("FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE")
|
| 106 |
+
os.environ["FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE"] = "1"
|
| 107 |
+
try:
|
| 108 |
+
v2()
|
| 109 |
+
torch.cuda.synchronize()
|
| 110 |
+
fused_hit = True
|
| 111 |
+
except RuntimeError as error:
|
| 112 |
+
if "BIAS epilogue was required" not in str(error):
|
| 113 |
+
raise
|
| 114 |
+
fused_hit = False
|
| 115 |
+
finally:
|
| 116 |
+
if old is None:
|
| 117 |
+
os.environ.pop("FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE", None)
|
| 118 |
+
else:
|
| 119 |
+
os.environ["FLASHRT_FP8_FFN_REQUIRE_BIAS_EPILOGUE"] = old
|
| 120 |
+
metrics = helpers.distribution_metrics(out_v2, out_v1)
|
| 121 |
+
v1_us = time_us(v1, args.warmup, args.iterations)
|
| 122 |
+
v2_us = time_us(v2, args.warmup, args.iterations)
|
| 123 |
+
row = {
|
| 124 |
+
"shape": name,
|
| 125 |
+
"M": shape[0],
|
| 126 |
+
"K": shape[1],
|
| 127 |
+
"H": shape[2],
|
| 128 |
+
"N": shape[3],
|
| 129 |
+
"fused_bias_epilogue": fused_hit,
|
| 130 |
+
"v1_us": v1_us,
|
| 131 |
+
"v2_us": v2_us,
|
| 132 |
+
"speedup": v1_us / v2_us,
|
| 133 |
+
**metrics,
|
| 134 |
+
}
|
| 135 |
+
rows.append(row)
|
| 136 |
+
print(json.dumps(row, sort_keys=True), flush=True)
|
| 137 |
+
|
| 138 |
+
result = {
|
| 139 |
+
"device": torch.cuda.get_device_name(),
|
| 140 |
+
"capability": list(torch.cuda.get_device_capability()),
|
| 141 |
+
"torch": torch.__version__,
|
| 142 |
+
"cuda": torch.version.cuda,
|
| 143 |
+
"rows": rows,
|
| 144 |
+
}
|
| 145 |
+
if args.json_out:
|
| 146 |
+
Path(args.json_out).write_text(json.dumps(result, indent=2) + "\n")
|
| 147 |
+
return 0
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
if __name__ == "__main__":
|
| 151 |
+
raise SystemExit(main())
|
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/{torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:46aff28a95fa0aa348046c6845ed289baa67c10c39c1b9936afcab4a709f3d07
|
| 3 |
+
size 703520
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_cuda_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_cuda_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/{torch211-cxx11-cu128-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1f6a225f67637c94184bc64dfb3c93c23a0e5e0f3e7957ba20a1a01874d21f5f
|
| 3 |
+
size 673200
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_cuda_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_cuda_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch211-cxx11-rocm71-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/{torch211-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so → torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:409804cf1498f2d98af1c3cf5ba70dd4d3e8a8c6e1ab8725668371d329797711
|
| 3 |
+
size 187352
|
build/torch211-cxx11-rocm71-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_rocm_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
|
build/torch211-cxx11-rocm71-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_rocm_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch211-cxx11-rocm72-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/{torch211-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so → torch211-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c6f65669c49255f4bce57e5bcd24f0298fe67147bdf0f495ea827d83fb4e9eaf
|
| 3 |
+
size 185816
|
build/torch211-cxx11-rocm72-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_rocm_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
|
build/torch211-cxx11-rocm72-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_rocm_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:9cf974f0a6585860d33d00d4269205d2046cbaa8e74442d4e289ec8ea7cddc6e
|
| 3 |
-
size 668544
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0a95a3b6e1f95822e181bd4a01f425ef5fbe8dcadcabe684f59bd33c76987ed9
|
| 3 |
+
size 673144
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_cuda_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_cuda_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:92f9e9ba898c08f78f8ea086de134a0f1dcfb1201ce9d04ca932d4ed30844919
|
| 3 |
-
size 668544
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_fp8_ffn_cuda_b2eeca0.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:81094023a0b91b9bd217f3bce768851bcf0923776bb17b6e8d2f192bd20ef063
|
| 3 |
+
size 673144
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_cuda_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_cuda_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_cuda_b2eeca0::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_cuda_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch212-cxx11-rocm71-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6203acdac17af6b44944b57bb831a4584a39b8ad401f0e8cd58a7c2b5760862e
|
| 3 |
-
size 180000
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-cxx11-rocm71-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8dc1bfc079ae1d9ee4a6f535ad36ee88fcc380f6b59ec075fab1116e49ef8b0a
|
| 3 |
+
size 187584
|
build/torch212-cxx11-rocm71-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_rocm_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
|
build/torch212-cxx11-rocm71-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_rocm_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
build/torch212-cxx11-rocm72-x86_64-linux/__init__.py
CHANGED
|
@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
input: torch.Tensor,
|
|
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
|
|
| 173 |
return None
|
| 174 |
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
|
|
| 433 |
return out[:logical_m]
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
__all__ = [
|
| 437 |
"bf16_fp8_linear_bias_bf16",
|
| 438 |
"bf16_fp8_gelu_mlp_bf16",
|
|
|
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 441 |
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
|
| 149 |
+
def _fp8_gelu_mlp_v2_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
hidden_bf16: torch.Tensor,
|
| 160 |
+
hidden_fp8: torch.Tensor,
|
| 161 |
+
out: torch.Tensor,
|
| 162 |
+
) -> None:
|
| 163 |
+
return _fp8_gelu_mlp_bf16_fake(
|
| 164 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 165 |
+
up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
|
| 166 |
+
hidden_fp8, out)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 170 |
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 171 |
input: torch.Tensor,
|
|
|
|
| 194 |
return None
|
| 195 |
|
| 196 |
|
| 197 |
+
@torch.library.register_fake(
|
| 198 |
+
add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
|
| 199 |
+
def _bf16_fp8_gelu_mlp_v2_bf16_fake(
|
| 200 |
+
input: torch.Tensor,
|
| 201 |
+
up_weight: torch.Tensor,
|
| 202 |
+
up_bias: torch.Tensor,
|
| 203 |
+
down_weight: torch.Tensor,
|
| 204 |
+
down_bias: torch.Tensor,
|
| 205 |
+
input_scale: torch.Tensor,
|
| 206 |
+
up_weight_scale: torch.Tensor,
|
| 207 |
+
hidden_scale: torch.Tensor,
|
| 208 |
+
down_weight_scale: torch.Tensor,
|
| 209 |
+
input_fp8: torch.Tensor,
|
| 210 |
+
hidden_bf16: torch.Tensor,
|
| 211 |
+
hidden_fp8: torch.Tensor,
|
| 212 |
+
out: torch.Tensor,
|
| 213 |
+
) -> None:
|
| 214 |
+
return _bf16_fp8_gelu_mlp_bf16_fake(
|
| 215 |
+
input, up_weight, up_bias, down_weight, down_bias, input_scale,
|
| 216 |
+
up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
|
| 217 |
+
hidden_bf16, hidden_fp8, out)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 221 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 222 |
|
|
|
|
| 477 |
return out[:logical_m]
|
| 478 |
|
| 479 |
|
| 480 |
+
def fp8_gelu_mlp_v2_bf16(
|
| 481 |
+
input: torch.Tensor,
|
| 482 |
+
up_weight: torch.Tensor,
|
| 483 |
+
up_bias: torch.Tensor,
|
| 484 |
+
down_weight: torch.Tensor,
|
| 485 |
+
down_bias: torch.Tensor,
|
| 486 |
+
input_scale: torch.Tensor,
|
| 487 |
+
up_weight_scale: torch.Tensor,
|
| 488 |
+
hidden_scale: torch.Tensor,
|
| 489 |
+
down_weight_scale: torch.Tensor,
|
| 490 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 491 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 492 |
+
out: torch.Tensor | None = None,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
"""``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
|
| 495 |
+
|
| 496 |
+
Same contract and buffers as the v1 entry; one launch and one full
|
| 497 |
+
write of ``out`` less per call on targets whose cublasLt path
|
| 498 |
+
accepts the BIAS epilogue (elsewhere the runtime falls back to the
|
| 499 |
+
v1 two-launch tail, still correct).
|
| 500 |
+
"""
|
| 501 |
+
|
| 502 |
+
if hidden_bf16 is None:
|
| 503 |
+
hidden_bf16 = torch.empty(
|
| 504 |
+
(input.shape[0], up_weight.shape[0]),
|
| 505 |
+
device=input.device,
|
| 506 |
+
dtype=torch.bfloat16,
|
| 507 |
+
)
|
| 508 |
+
if hidden_fp8 is None:
|
| 509 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 510 |
+
if out is None:
|
| 511 |
+
out = torch.empty(
|
| 512 |
+
(input.shape[0], down_weight.shape[0]),
|
| 513 |
+
device=input.device,
|
| 514 |
+
dtype=torch.bfloat16,
|
| 515 |
+
)
|
| 516 |
+
ops.fp8_gelu_mlp_v2_bf16(
|
| 517 |
+
input,
|
| 518 |
+
up_weight,
|
| 519 |
+
up_bias,
|
| 520 |
+
down_weight,
|
| 521 |
+
down_bias,
|
| 522 |
+
input_scale,
|
| 523 |
+
up_weight_scale,
|
| 524 |
+
hidden_scale,
|
| 525 |
+
down_weight_scale,
|
| 526 |
+
hidden_bf16,
|
| 527 |
+
hidden_fp8,
|
| 528 |
+
out,
|
| 529 |
+
)
|
| 530 |
+
return out
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def bf16_fp8_gelu_mlp_v2_bf16(
|
| 534 |
+
input: torch.Tensor,
|
| 535 |
+
up_weight: torch.Tensor,
|
| 536 |
+
up_bias: torch.Tensor,
|
| 537 |
+
down_weight: torch.Tensor,
|
| 538 |
+
down_bias: torch.Tensor,
|
| 539 |
+
input_scale: torch.Tensor,
|
| 540 |
+
up_weight_scale: torch.Tensor,
|
| 541 |
+
hidden_scale: torch.Tensor,
|
| 542 |
+
down_weight_scale: torch.Tensor,
|
| 543 |
+
input_fp8: torch.Tensor | None = None,
|
| 544 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 545 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 546 |
+
out: torch.Tensor | None = None,
|
| 547 |
+
*,
|
| 548 |
+
pad_to: int | None = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
"""``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
|
| 551 |
+
|
| 552 |
+
Identical padding and buffer contract to the v1 entry.
|
| 553 |
+
"""
|
| 554 |
+
|
| 555 |
+
logical_m = input.shape[0]
|
| 556 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 557 |
+
if padded_m < logical_m:
|
| 558 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 559 |
+
device = input.device
|
| 560 |
+
if input_fp8 is None:
|
| 561 |
+
input_fp8 = torch.empty(
|
| 562 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 563 |
+
)
|
| 564 |
+
if hidden_bf16 is None:
|
| 565 |
+
hidden_bf16 = torch.empty(
|
| 566 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 567 |
+
)
|
| 568 |
+
if hidden_fp8 is None:
|
| 569 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 570 |
+
if out is None:
|
| 571 |
+
out = torch.empty(
|
| 572 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 573 |
+
)
|
| 574 |
+
ops.bf16_fp8_gelu_mlp_v2_bf16(
|
| 575 |
+
input,
|
| 576 |
+
up_weight,
|
| 577 |
+
up_bias,
|
| 578 |
+
down_weight,
|
| 579 |
+
down_bias,
|
| 580 |
+
input_scale,
|
| 581 |
+
up_weight_scale,
|
| 582 |
+
hidden_scale,
|
| 583 |
+
down_weight_scale,
|
| 584 |
+
input_fp8,
|
| 585 |
+
hidden_bf16,
|
| 586 |
+
hidden_fp8,
|
| 587 |
+
out,
|
| 588 |
+
)
|
| 589 |
+
return out[:logical_m]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
__all__ = [
|
| 593 |
"bf16_fp8_linear_bias_bf16",
|
| 594 |
"bf16_fp8_gelu_mlp_bf16",
|
| 595 |
+
"bf16_fp8_gelu_mlp_v2_bf16",
|
| 596 |
"fp8_gemm_bf16",
|
| 597 |
"fp8_gelu_mlp_bf16",
|
| 598 |
+
"fp8_gelu_mlp_v2_bf16",
|
| 599 |
"fp8_linear_bias_bf16",
|
| 600 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 601 |
]
|
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_255584f.abi3.so
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:eb09062fde7313db39fe558ff36f8b902b04ac06fade78310ccd1eb27ad3d90b
|
| 3 |
-
size 178424
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-cxx11-rocm72-x86_64-linux/_flashrt_fp8_ffn_rocm_b2eeca0.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:860c6f79cb15466319a99fc5df4f1651d1040a8d074d9df37ecdf8f86b1985b7
|
| 3 |
+
size 186040
|
build/torch212-cxx11-rocm72-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_rocm_b2eeca0
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_rocm_b2eeca0
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_flashrt_fp8_ffn_rocm_b2eeca0::{op_name}"
|
build/torch212-cxx11-rocm72-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_rocm_b2eeca0",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|