Uploaded using `kernel-builder`.

Browse files

Files changed (8) hide show

build/torch212-metal-aarch64-darwin/__init__.py +62 -0
build/torch212-metal-aarch64-darwin/_gguf_kernels_metal_5d3209c_dirty.abi3.so +3 -0
build/torch212-metal-aarch64-darwin/_ops.py +9 -0
build/torch212-metal-aarch64-darwin/metadata.json +29 -0
build/torch213-metal-aarch64-darwin/__init__.py +62 -0
build/torch213-metal-aarch64-darwin/_gguf_kernels_metal_5d3209c_dirty.abi3.so +3 -0
build/torch213-metal-aarch64-darwin/_ops.py +9 -0
build/torch213-metal-aarch64-darwin/metadata.json +29 -0

build/torch212-metal-aarch64-darwin/__init__.py ADDED Viewed

	@@ -0,0 +1,62 @@

+"""Compute directly on the packed blocks of a GGUF checkpoint.
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+import torch
+from ._ops import add_op_namespace_prefix, ops
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch212-metal-aarch64-darwin/_gguf_kernels_metal_5d3209c_dirty.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:416d3b6d1969700652047a0d373f6f3025d312a1f044cdf83bdeb96d99b04246
+size 7403160

build/torch212-metal-aarch64-darwin/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _gguf_kernels_metal_5d3209c_dirty
+ops = torch.ops._gguf_kernels_metal_5d3209c_dirty
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_metal_5d3209c_dirty::{op_name}"

build/torch212-metal-aarch64-darwin/metadata.json ADDED Viewed

	@@ -0,0 +1,29 @@

+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_metal_5d3209c_dirty",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "metal"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_metal_5d3209c_dirty.abi3.so": "QW07bRlpcAZSBHoNNz9vMCXTEqHwRM34O965bZmwQkY=",
+      "_ops.py": "P0YXhjRkGKAKtdyb3TiOjCFsI4YDDGkzttoDI2JiJHg="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "5d3209c5797b32001022de342dad3b792a94e921",
+      "dirty": true
+    }
+  }
+}

build/torch213-metal-aarch64-darwin/__init__.py ADDED Viewed

	@@ -0,0 +1,62 @@

+"""Compute directly on the packed blocks of a GGUF checkpoint.
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+import torch
+from ._ops import add_op_namespace_prefix, ops
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
+# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
+# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
+# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
+# check this before choosing the fused path.
+try:
+    GEMV_TYPES = frozenset(ops.gemv_types())
+except AttributeError:
+    # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
+    GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch213-metal-aarch64-darwin/_gguf_kernels_metal_5d3209c_dirty.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2033fc4c4d2457754641c8a895601968c998314db1a11913846215047b80f50c
+size 7403160

build/torch213-metal-aarch64-darwin/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _gguf_kernels_metal_5d3209c_dirty
+ops = torch.ops._gguf_kernels_metal_5d3209c_dirty
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_metal_5d3209c_dirty::{op_name}"

build/torch213-metal-aarch64-darwin/metadata.json ADDED Viewed

	@@ -0,0 +1,29 @@

+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_metal_5d3209c_dirty",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "metal"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "GoFcHJbbJ9C8b40XNznMxsTcC+CV9kaFE8a1DP763kU=",
+      "_gguf_kernels_metal_5d3209c_dirty.abi3.so": "IDP8TE0kV3VGQciolWAZaMmYMU2xoRkThGIVBHuA9Qw=",
+      "_ops.py": "P0YXhjRkGKAKtdyb3TiOjCFsI4YDDGkzttoDI2JiJHg="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "5d3209c5797b32001022de342dad3b792a94e921",
+      "dirty": true
+    }
+  }
+}