Promote latest kernel artifacts to main

README.md

@@ -1,9 +0,0 @@
-# flashrt/flashrt-fp8-ffn
-
-This repository is a compatibility mirror for older `kernels` clients
-that resolve repositories through the default Hugging Face model repo API.
-
-Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-fp8-ffn
-
-Do not edit this mirror by hand. It is generated from the Kernel Hub
-`vN` branches and contains the same `build/**` artifacts.

build/torch211-cxx11-cu130-aarch64-linux/__init__.py

@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
     return None
 
 
+@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
+def _fp8_gelu_mlp_v2_bf16_fake(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    hidden_bf16: torch.Tensor,
+    hidden_fp8: torch.Tensor,
+    out: torch.Tensor,
+) -> None:
+    return _fp8_gelu_mlp_bf16_fake(
+        input, up_weight, up_bias, down_weight, down_bias, input_scale,
+        up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
+        hidden_fp8, out)
+
+
 @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
 def _bf16_fp8_gelu_mlp_bf16_fake(
     input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
     return None
 
 
+@torch.library.register_fake(
+    add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
+def _bf16_fp8_gelu_mlp_v2_bf16_fake(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    input_fp8: torch.Tensor,
+    hidden_bf16: torch.Tensor,
+    hidden_fp8: torch.Tensor,
+    out: torch.Tensor,
+) -> None:
+    return _bf16_fp8_gelu_mlp_bf16_fake(
+        input, up_weight, up_bias, down_weight, down_bias, input_scale,
+        up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
+        hidden_bf16, hidden_fp8, out)
+
+
 def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
     return torch.tensor([value], device=input.device, dtype=torch.float32)
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
     return out[:logical_m]
 
 
+def fp8_gelu_mlp_v2_bf16(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    hidden_bf16: torch.Tensor | None = None,
+    hidden_fp8: torch.Tensor | None = None,
+    out: torch.Tensor | None = None,
+) -> torch.Tensor:
+    """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
+
+    Same contract and buffers as the v1 entry; one launch and one full
+    write of ``out`` less per call on targets whose cublasLt path
+    accepts the BIAS epilogue (elsewhere the runtime falls back to the
+    v1 two-launch tail, still correct).
+    """
+
+    if hidden_bf16 is None:
+        hidden_bf16 = torch.empty(
+            (input.shape[0], up_weight.shape[0]),
+            device=input.device,
+            dtype=torch.bfloat16,
+        )
+    if hidden_fp8 is None:
+        hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
+    if out is None:
+        out = torch.empty(
+            (input.shape[0], down_weight.shape[0]),
+            device=input.device,
+            dtype=torch.bfloat16,
+        )
+    ops.fp8_gelu_mlp_v2_bf16(
+        input,
+        up_weight,
+        up_bias,
+        down_weight,
+        down_bias,
+        input_scale,
+        up_weight_scale,
+        hidden_scale,
+        down_weight_scale,
+        hidden_bf16,
+        hidden_fp8,
+        out,
+    )
+    return out
+
+
+def bf16_fp8_gelu_mlp_v2_bf16(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    input_fp8: torch.Tensor | None = None,
+    hidden_bf16: torch.Tensor | None = None,
+    hidden_fp8: torch.Tensor | None = None,
+    out: torch.Tensor | None = None,
+    *,
+    pad_to: int | None = None,
+) -> torch.Tensor:
+    """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
+
+    Identical padding and buffer contract to the v1 entry.
+    """
+
+    logical_m = input.shape[0]
+    padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
+    if padded_m < logical_m:
+        raise ValueError("pad_to must be >= input.shape[0]")
+    device = input.device
+    if input_fp8 is None:
+        input_fp8 = torch.empty(
+            (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
+        )
+    if hidden_bf16 is None:
+        hidden_bf16 = torch.empty(
+            (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
+        )
+    if hidden_fp8 is None:
+        hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
+    if out is None:
+        out = torch.empty(
+            (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
+        )
+    ops.bf16_fp8_gelu_mlp_v2_bf16(
+        input,
+        up_weight,
+        up_bias,
+        down_weight,
+        down_bias,
+        input_scale,
+        up_weight_scale,
+        hidden_scale,
+        down_weight_scale,
+        input_fp8,
+        hidden_bf16,
+        hidden_fp8,
+        out,
+    )
+    return out[:logical_m]
+
+
 __all__ = [
     "bf16_fp8_linear_bias_bf16",
     "bf16_fp8_gelu_mlp_bf16",
+    "bf16_fp8_gelu_mlp_v2_bf16",
     "fp8_gemm_bf16",
     "fp8_gelu_mlp_bf16",
+    "fp8_gelu_mlp_v2_bf16",
     "fp8_linear_bias_bf16",
     "fp8_linear_bias_gelu_quant_bf16",
 ]

build/torch211-cxx11-cu130-aarch64-linux/{_flashrt_fp8_ffn_cuda_255584f.abi3.so → _flashrt_fp8_ffn_cuda_230e467.abi3.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec0844b1d17575d1e324ef9aac70c4ba78d464615bee9a1a9f3ae4bca51e8a41
-size 256720
+oid sha256:36c3c74bc022516143f5c906cfa9a8dbf17cff61569d1ca8474fda55b73ce76c
+size 257080

build/torch211-cxx11-cu130-aarch64-linux/_ops.py

@@ -1,6 +1,6 @@
 import torch
-from . import _flashrt_fp8_ffn_cuda_255584f
-ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
+from . import _flashrt_fp8_ffn_cuda_230e467
+ops = torch.ops._flashrt_fp8_ffn_cuda_230e467
 
 def add_op_namespace_prefix(op_name: str):
-    return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
+    return f"_flashrt_fp8_ffn_cuda_230e467::{op_name}"

build/torch211-cxx11-cu130-aarch64-linux/metadata.json

@@ -1,22 +1,28 @@
 {
   "name": "flashrt-fp8-ffn",
-  "id": "_flashrt_fp8_ffn_cuda_255584f",
+  "id": "_flashrt_fp8_ffn_cuda_230e467",
   "version": 1,
   "license": "Apache-2.0",
   "python-depends": [],
   "backend": {
     "type": "cuda",
     "archs": [
-      "11.0"
+      "11.0a"
     ]
   },
   "digest": {
     "algorithm": "sha256",
     "files": {
-      "__init__.py": "Lfpi4OMG3jS0AMFqtyCpL5H/nNuAzqtpwFaUnv2wgqo=",
-      "_flashrt_fp8_ffn_cuda_255584f.abi3.so": "7AhEsdF1ddHjJO+arHDEunjUZGFb7poanzrkvKUeikE=",
-      "_ops.py": "DjRVHDqNFmTPfak1/oS9Kq0Gncu1ODIrvbLkhemqRzI=",
+      "__init__.py": "O5OvnWgzbfzKB83CtQwQy60BSAa83vizBgNuMnx6D6A=",
+      "_flashrt_fp8_ffn_cuda_230e467.abi3.so": "NsPHS8AiUWFD9ckGz6mo2/F8/2FWnRyoR0/aVbc852w=",
+      "_ops.py": "MmNuiXB4ECVn/fPAOD6Abcdco5Bb4KAyVc26dklNvvc=",
       "flashrt_fp8_ffn/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
     }
+  },
+  "provenance": {
+    "kernel": {
+      "sha": "230e467",
+      "dirty": false
+    }
   }
 }

build/torch213-cxx11-cu130-aarch64-linux/__init__.py

@@ -145,6 +145,27 @@ def _fp8_gelu_mlp_bf16_fake(
     return None
 
 
+@torch.library.register_fake(add_op_namespace_prefix("fp8_gelu_mlp_v2_bf16"))
+def _fp8_gelu_mlp_v2_bf16_fake(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    hidden_bf16: torch.Tensor,
+    hidden_fp8: torch.Tensor,
+    out: torch.Tensor,
+) -> None:
+    return _fp8_gelu_mlp_bf16_fake(
+        input, up_weight, up_bias, down_weight, down_bias, input_scale,
+        up_weight_scale, hidden_scale, down_weight_scale, hidden_bf16,
+        hidden_fp8, out)
+
+
 @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
 def _bf16_fp8_gelu_mlp_bf16_fake(
     input: torch.Tensor,
@@ -173,6 +194,29 @@ def _bf16_fp8_gelu_mlp_bf16_fake(
     return None
 
 
+@torch.library.register_fake(
+    add_op_namespace_prefix("bf16_fp8_gelu_mlp_v2_bf16"))
+def _bf16_fp8_gelu_mlp_v2_bf16_fake(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    input_fp8: torch.Tensor,
+    hidden_bf16: torch.Tensor,
+    hidden_fp8: torch.Tensor,
+    out: torch.Tensor,
+) -> None:
+    return _bf16_fp8_gelu_mlp_bf16_fake(
+        input, up_weight, up_bias, down_weight, down_bias, input_scale,
+        up_weight_scale, hidden_scale, down_weight_scale, input_fp8,
+        hidden_bf16, hidden_fp8, out)
+
+
 def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
     return torch.tensor([value], device=input.device, dtype=torch.float32)
 
@@ -433,11 +477,125 @@ def bf16_fp8_gelu_mlp_bf16(
     return out[:logical_m]
 
 
+def fp8_gelu_mlp_v2_bf16(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    hidden_bf16: torch.Tensor | None = None,
+    hidden_fp8: torch.Tensor | None = None,
+    out: torch.Tensor | None = None,
+) -> torch.Tensor:
+    """``fp8_gelu_mlp_bf16`` with the down bias in the GEMM epilogue.
+
+    Same contract and buffers as the v1 entry; one launch and one full
+    write of ``out`` less per call on targets whose cublasLt path
+    accepts the BIAS epilogue (elsewhere the runtime falls back to the
+    v1 two-launch tail, still correct).
+    """
+
+    if hidden_bf16 is None:
+        hidden_bf16 = torch.empty(
+            (input.shape[0], up_weight.shape[0]),
+            device=input.device,
+            dtype=torch.bfloat16,
+        )
+    if hidden_fp8 is None:
+        hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
+    if out is None:
+        out = torch.empty(
+            (input.shape[0], down_weight.shape[0]),
+            device=input.device,
+            dtype=torch.bfloat16,
+        )
+    ops.fp8_gelu_mlp_v2_bf16(
+        input,
+        up_weight,
+        up_bias,
+        down_weight,
+        down_bias,
+        input_scale,
+        up_weight_scale,
+        hidden_scale,
+        down_weight_scale,
+        hidden_bf16,
+        hidden_fp8,
+        out,
+    )
+    return out
+
+
+def bf16_fp8_gelu_mlp_v2_bf16(
+    input: torch.Tensor,
+    up_weight: torch.Tensor,
+    up_bias: torch.Tensor,
+    down_weight: torch.Tensor,
+    down_bias: torch.Tensor,
+    input_scale: torch.Tensor,
+    up_weight_scale: torch.Tensor,
+    hidden_scale: torch.Tensor,
+    down_weight_scale: torch.Tensor,
+    input_fp8: torch.Tensor | None = None,
+    hidden_bf16: torch.Tensor | None = None,
+    hidden_fp8: torch.Tensor | None = None,
+    out: torch.Tensor | None = None,
+    *,
+    pad_to: int | None = None,
+) -> torch.Tensor:
+    """``bf16_fp8_gelu_mlp_bf16`` with the down bias in the epilogue.
+
+    Identical padding and buffer contract to the v1 entry.
+    """
+
+    logical_m = input.shape[0]
+    padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
+    if padded_m < logical_m:
+        raise ValueError("pad_to must be >= input.shape[0]")
+    device = input.device
+    if input_fp8 is None:
+        input_fp8 = torch.empty(
+            (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
+        )
+    if hidden_bf16 is None:
+        hidden_bf16 = torch.empty(
+            (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
+        )
+    if hidden_fp8 is None:
+        hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
+    if out is None:
+        out = torch.empty(
+            (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
+        )
+    ops.bf16_fp8_gelu_mlp_v2_bf16(
+        input,
+        up_weight,
+        up_bias,
+        down_weight,
+        down_bias,
+        input_scale,
+        up_weight_scale,
+        hidden_scale,
+        down_weight_scale,
+        input_fp8,
+        hidden_bf16,
+        hidden_fp8,
+        out,
+    )
+    return out[:logical_m]
+
+
 __all__ = [
     "bf16_fp8_linear_bias_bf16",
     "bf16_fp8_gelu_mlp_bf16",
+    "bf16_fp8_gelu_mlp_v2_bf16",
     "fp8_gemm_bf16",
     "fp8_gelu_mlp_bf16",
+    "fp8_gelu_mlp_v2_bf16",
     "fp8_linear_bias_bf16",
     "fp8_linear_bias_gelu_quant_bf16",
 ]

build/torch213-cxx11-cu130-aarch64-linux/{flashrt_fp8_ffn_test.abi3.so → _flashrt_fp8_ffn_cuda_230e467.abi3.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d680e168126517a113e7d6412e1d9c3595d57e1388809ff55d5a366e8bf2857
-size 256376
+oid sha256:15af85e5b0676f35438b6918df8e0808d6dda1a3eb7ccee0e1253391f1ff2f1b
+size 257080

build/torch213-cxx11-cu130-aarch64-linux/_ops.py

@@ -1,6 +1,6 @@
 import torch
-from . import flashrt_fp8_ffn_test
-ops = torch.ops.flashrt_fp8_ffn_test
+from . import _flashrt_fp8_ffn_cuda_230e467
+ops = torch.ops._flashrt_fp8_ffn_cuda_230e467
 
 def add_op_namespace_prefix(op_name: str):
-    return f"flashrt_fp8_ffn_test::{op_name}"
+    return f"_flashrt_fp8_ffn_cuda_230e467::{op_name}"

build/torch213-cxx11-cu130-aarch64-linux/metadata.json

@@ -1,32 +1,28 @@
 {
   "name": "flashrt-fp8-ffn",
-  "id": "flashrt_fp8_ffn_test",
+  "id": "_flashrt_fp8_ffn_cuda_230e467",
   "version": 1,
   "license": "Apache-2.0",
   "python-depends": [],
   "backend": {
     "type": "cuda",
     "archs": [
-      "11.0"
+      "11.0a"
     ]
   },
   "digest": {
     "algorithm": "sha256",
     "files": {
-      "__init__.py": "Lfpi4OMG3jS0AMFqtyCpL5H/nNuAzqtpwFaUnv2wgqo=",
-      "flashrt_fp8_ffn_test.abi3.so": "jWgOFoEmUXoRPn1kEuHZw1ldV+E4iAn/VdWjZui/KFc=",
-      "_ops.py": "F8Tu59YGRLKuw0Q2GWUMHzFHDDk3uAor7D3+cCubz5E=",
+      "__init__.py": "O5OvnWgzbfzKB83CtQwQy60BSAa83vizBgNuMnx6D6A=",
+      "_flashrt_fp8_ffn_cuda_230e467.abi3.so": "Fa+F5bBnbzVDi2kY344ICNbdoaPrfM7g4SUzkfH/Lxs=",
+      "_ops.py": "MmNuiXB4ECVn/fPAOD6Abcdco5Bb4KAyVc26dklNvvc=",
       "flashrt_fp8_ffn/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
     }
   },
   "provenance": {
     "kernel": {
-      "sha": "456d297",
+      "sha": "230e467",
       "dirty": false
-    },
-    "validation": {
-      "torch": "2.13.0+cu130",
-      "cuda": "13.0"
     }
   }
 }