Rebuild CUDA variants with the current builder (provenance, digest, no shim)

.gitattributes

@@ -50,3 +50,9 @@ build/torch212-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so filte
 build/torch212-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch212-metal-aarch64-darwin/_gguf_kernels_metal_0becb0a.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch213-metal-aarch64-darwin/_gguf_kernels_metal_0becb0a.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu126-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-cxx11-cu126-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so filter=lfs diff=lfs merge=lfs -text

build/torch211-cxx11-cu126-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bc09a45aabd708b54f9ce83b5ec9627ed185826b64aeb2b5673f46cad4c508e
-size 24799632
+oid sha256:b13c3193a71b1801478effbc94a8007b706c1ec6f140f9567349d97bcef68ace
+size 133

build/torch211-cxx11-cu128-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b71394062f24b9601e17ee704b313494487bbe77b8d20f54adbfd52665d9f300
-size 37124624
+oid sha256:039423d3e913e348225f13e3764e50c4cd8feb79617f045003a447e63bafe609
+size 133

build/torch211-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b19aec224f81561100e3133d414393779bacfabf2c703040e64b95fe0b69cef
-size 37378504
+oid sha256:e8af7132e7c0e5ee536b2537b538b880325d8b00ac1fa8e53aa3914f537d0149
+size 133

build/torch212-cxx11-cu126-x86_64-linux/__init__.py

@@ -20,9 +20,16 @@ __all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
 MAX_GEMV_ROWS = 8
 
-# ggml type ids `mul_mat_vec` implements: Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, and the IQ quants.
-# `dequantize` covers more, so check this one before choosing the fused path.
-GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
 
 
 def dequantize(

build/torch212-cxx11-cu126-x86_64-linux/{_gguf_kernels_cuda_e4e2164.abi3.so → _gguf_kernels_cuda_1df3b99.abi3.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0176cb70a486d747de1e4b4aa785760580e7b4dc8afc9efc897621d5fd307639
-size 24805280
+oid sha256:a0dcdaf3e54554158476036dad0d2e6412aa42d4d1bfefa8d68e019e841283da
+size 24822056

build/torch212-cxx11-cu126-x86_64-linux/_ops.py

@@ -1,9 +1,9 @@
 import torch
-from . import _gguf_kernels_cuda_e4e2164
-ops = torch.ops._gguf_kernels_cuda_e4e2164
+from . import _gguf_kernels_cuda_1df3b99
+ops = torch.ops._gguf_kernels_cuda_1df3b99
 
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_gguf_kernels_cuda_e4e2164::{op_name}"
+    return f"_gguf_kernels_cuda_1df3b99::{op_name}"

build/torch212-cxx11-cu126-x86_64-linux/gguf_kernels/__init__.py

@@ -1,26 +0,0 @@
-import ctypes
-import importlib.util
-import sys
-from pathlib import Path
-from types import ModuleType
-
-
-def _import_from_path(file_path: Path) -> ModuleType:
-    # We cannot use the module name as-is, after adding it to `sys.modules`,
-    # it would also be used for other imports. So, we make a module name that
-    # depends on the path for it to be unique using the hex-encoded hash of
-    # the path.
-    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
-    module_name = path_hash
-    spec = importlib.util.spec_from_file_location(module_name, file_path)
-    if spec is None:
-        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
-    module = importlib.util.module_from_spec(spec)
-    if module is None:
-        raise ImportError(f"Cannot load module {module_name} from spec")
-    sys.modules[module_name] = module
-    spec.loader.exec_module(module)  # type: ignore
-    return module
-
-
-globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu126-x86_64-linux/metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "gguf-kernels",
-  "id": "_gguf_kernels_cuda_e4e2164",
+  "id": "_gguf_kernels_cuda_1df3b99",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -13,5 +13,24 @@
       "8.9",
       "9.0"
     ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_cuda_1df3b99.abi3.so": "oNza8+VFVBWEdgNtrQ0uZBKqQtTRv++o1o4BnoQSg9o=",
+      "_ops.py": "dRIZkil9x8IS6KhIwXlRzO6qzQoGYUQkGL87Z3gpEzY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "1df3b999686b77ef298ae13d066a39958503164c",
+      "dirty": false
+    }
   }
-}
+}

build/torch212-cxx11-cu130-x86_64-linux/__init__.py

@@ -20,9 +20,16 @@ __all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
 MAX_GEMV_ROWS = 8
 
-# ggml type ids `mul_mat_vec` implements: Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, and the IQ quants.
-# `dequantize` covers more, so check this one before choosing the fused path.
-GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
 
 
 def dequantize(

build/{torch212-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so → torch212-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:998f0c3e2e8a77d5e7dbd43ed817a98d46f519e1bcd626cd58d6365d2666edb4
-size 37398744
+oid sha256:7355d69df74e88b5163248a7ada198f95c5310b2b182268e951ee7da67f4a537
+size 37396976

build/torch212-cxx11-cu130-x86_64-linux/_ops.py

@@ -1,9 +1,9 @@
 import torch
-from . import _gguf_kernels_cuda_e4e2164
-ops = torch.ops._gguf_kernels_cuda_e4e2164
+from . import _gguf_kernels_cuda_1df3b99
+ops = torch.ops._gguf_kernels_cuda_1df3b99
 
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_gguf_kernels_cuda_e4e2164::{op_name}"
+    return f"_gguf_kernels_cuda_1df3b99::{op_name}"

build/torch212-cxx11-cu130-x86_64-linux/gguf_kernels/__init__.py

@@ -1,26 +0,0 @@
-import ctypes
-import importlib.util
-import sys
-from pathlib import Path
-from types import ModuleType
-
-
-def _import_from_path(file_path: Path) -> ModuleType:
-    # We cannot use the module name as-is, after adding it to `sys.modules`,
-    # it would also be used for other imports. So, we make a module name that
-    # depends on the path for it to be unique using the hex-encoded hash of
-    # the path.
-    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
-    module_name = path_hash
-    spec = importlib.util.spec_from_file_location(module_name, file_path)
-    if spec is None:
-        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
-    module = importlib.util.module_from_spec(spec)
-    if module is None:
-        raise ImportError(f"Cannot load module {module_name} from spec")
-    sys.modules[module_name] = module
-    spec.loader.exec_module(module)  # type: ignore
-    return module
-
-
-globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu130-x86_64-linux/metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "gguf-kernels",
-  "id": "_gguf_kernels_cuda_e4e2164",
+  "id": "_gguf_kernels_cuda_1df3b99",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -15,5 +15,24 @@
       "8.9",
       "9.0"
     ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_cuda_1df3b99.abi3.so": "c1XWnfdOiLUWMkinraGY+VxTELKxgiaOlR7n2mf0pTc=",
+      "_ops.py": "dRIZkil9x8IS6KhIwXlRzO6qzQoGYUQkGL87Z3gpEzY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "1df3b999686b77ef298ae13d066a39958503164c",
+      "dirty": false
+    }
   }
-}
+}

build/torch212-cxx11-cu132-x86_64-linux/__init__.py

@@ -20,9 +20,16 @@ __all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
 MAX_GEMV_ROWS = 8
 
-# ggml type ids `mul_mat_vec` implements: Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, and the IQ quants.
-# `dequantize` covers more, so check this one before choosing the fused path.
-GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
 
 
 def dequantize(

build/{torch212-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so → torch212-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d7e366d102d2194c5df8ab2b96f2c741e4738798e4ce913c5f980eefe0f96d4
-size 37380240
+oid sha256:50a408d91abc14355e1b1666ac6aa591f61271f8333dc863f1e9395822f0b1a1
+size 37415488

build/torch212-cxx11-cu132-x86_64-linux/_ops.py

@@ -1,9 +1,9 @@
 import torch
-from . import _gguf_kernels_cuda_e4e2164
-ops = torch.ops._gguf_kernels_cuda_e4e2164
+from . import _gguf_kernels_cuda_1df3b99
+ops = torch.ops._gguf_kernels_cuda_1df3b99
 
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_gguf_kernels_cuda_e4e2164::{op_name}"
+    return f"_gguf_kernels_cuda_1df3b99::{op_name}"

build/torch212-cxx11-cu132-x86_64-linux/gguf_kernels/__init__.py

@@ -1,26 +0,0 @@
-import ctypes
-import importlib.util
-import sys
-from pathlib import Path
-from types import ModuleType
-
-
-def _import_from_path(file_path: Path) -> ModuleType:
-    # We cannot use the module name as-is, after adding it to `sys.modules`,
-    # it would also be used for other imports. So, we make a module name that
-    # depends on the path for it to be unique using the hex-encoded hash of
-    # the path.
-    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
-    module_name = path_hash
-    spec = importlib.util.spec_from_file_location(module_name, file_path)
-    if spec is None:
-        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
-    module = importlib.util.module_from_spec(spec)
-    if module is None:
-        raise ImportError(f"Cannot load module {module_name} from spec")
-    sys.modules[module_name] = module
-    spec.loader.exec_module(module)  # type: ignore
-    return module
-
-
-globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu132-x86_64-linux/metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "gguf-kernels",
-  "id": "_gguf_kernels_cuda_e4e2164",
+  "id": "_gguf_kernels_cuda_1df3b99",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -15,5 +15,24 @@
       "8.9",
       "9.0"
     ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_cuda_1df3b99.abi3.so": "UKQI2Rq8FDVeGxZmrGqlkfYScfgzPchj8ek5WCLwsaE=",
+      "_ops.py": "dRIZkil9x8IS6KhIwXlRzO6qzQoGYUQkGL87Z3gpEzY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "1df3b999686b77ef298ae13d066a39958503164c",
+      "dirty": false
+    }
   }
-}
+}

build/torch212-metal-aarch64-darwin/_gguf_kernels_metal_0becb0a.abi3.so

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b369cc5b468e2fd3b9ac84f5c63897a52bc25e0381c37104d11d0956f5ad4dd9
-size 7403104
+oid sha256:b15d061c06a3be61c1b7039d153b6a791d8bb2f7bc972a44fa33e575642ecbd0
+size 132

build/torch213-cxx11-cu126-x86_64-linux/__init__.py

@@ -0,0 +1,62 @@
+"""Compute directly on the packed blocks of a GGUF checkpoint.
+
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+
+import torch
+
+from ._ops import add_op_namespace_prefix, ops
+
+
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+
+
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+
+
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+
+
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+
+
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch213-cxx11-cu126-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a7b5651497660db8a6b4cbe857610fc5ac448d52c36ad44b16c6e199d0d09e6
+size 24821896

build/torch213-cxx11-cu126-x86_64-linux/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _gguf_kernels_cuda_1df3b99
+ops = torch.ops._gguf_kernels_cuda_1df3b99
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_cuda_1df3b99::{op_name}"

build/torch213-cxx11-cu126-x86_64-linux/metadata.json

@@ -0,0 +1,36 @@
+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_cuda_1df3b99",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_cuda_1df3b99.abi3.so": "mntWUUl2YNuKa0y+hXYQ/FrESNUsNq1EsWxuGZ0NCeY=",
+      "_ops.py": "dRIZkil9x8IS6KhIwXlRzO6qzQoGYUQkGL87Z3gpEzY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "1df3b999686b77ef298ae13d066a39958503164c",
+      "dirty": false
+    }
+  }
+}

build/torch213-cxx11-cu130-x86_64-linux/__init__.py

@@ -0,0 +1,62 @@
+"""Compute directly on the packed blocks of a GGUF checkpoint.
+
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+
+import torch
+
+from ._ops import add_op_namespace_prefix, ops
+
+
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+
+
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+
+
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+
+
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+
+
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch213-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa7b02e5b0e1976c3d244e697710bf692943a598e9a56fee5c475cbd0d63015c
+size 37396824

build/torch213-cxx11-cu130-x86_64-linux/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _gguf_kernels_cuda_1df3b99
+ops = torch.ops._gguf_kernels_cuda_1df3b99
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_cuda_1df3b99::{op_name}"

build/torch213-cxx11-cu130-x86_64-linux/metadata.json

@@ -0,0 +1,38 @@
+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_cuda_1df3b99",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_cuda_1df3b99.abi3.so": "qnsC5bDhl2w9JE5pdxC/aSlDpZjppW/uXEdcvQ1jAVw=",
+      "_ops.py": "dRIZkil9x8IS6KhIwXlRzO6qzQoGYUQkGL87Z3gpEzY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "1df3b999686b77ef298ae13d066a39958503164c",
+      "dirty": false
+    }
+  }
+}

build/torch213-cxx11-cu132-x86_64-linux/__init__.py

@@ -0,0 +1,62 @@
+"""Compute directly on the packed blocks of a GGUF checkpoint.
+
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+
+import torch
+
+from ._ops import add_op_namespace_prefix, ops
+
+
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+
+
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+
+
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+
+
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+
+
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch213-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_1df3b99.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:732504ad19d3eb0692fca0a1085cd27eeb838c068c6f27b657d9b8224504f629
+size 37415328

build/torch213-cxx11-cu132-x86_64-linux/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _gguf_kernels_cuda_1df3b99
+ops = torch.ops._gguf_kernels_cuda_1df3b99
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_cuda_1df3b99::{op_name}"

build/torch213-cxx11-cu132-x86_64-linux/metadata.json

@@ -0,0 +1,38 @@
+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_cuda_1df3b99",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_cuda_1df3b99.abi3.so": "cyUErRnT6waS/KChCFzSfuuDjAaMbye2V9m4IkUE9ik=",
+      "_ops.py": "dRIZkil9x8IS6KhIwXlRzO6qzQoGYUQkGL87Z3gpEzY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "1df3b999686b77ef298ae13d066a39958503164c",
+      "dirty": false
+    }
+  }
+}

build/torch213-metal-aarch64-darwin/_gguf_kernels_metal_0becb0a.abi3.so

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59d0c2717f44308bc965b82e520cce45b0a6f7e496bc9e95dfc2367dbc75f441
-size 7403104
+oid sha256:6346a66092b922568c84e4ed636157542f3b9b529d6b5680a03a79bc81e34712
+size 132