Uploaded using `kernel-builder`.

Browse files

Files changed (8) hide show

build/torch212-metal-aarch64-darwin/__init__.py +21 -1
build/torch212-metal-aarch64-darwin/{_ggml_quantization_metal_091adac.abi3.so → _ggml_quantization_metal_9534e0b.abi3.so} +2 -2
build/torch212-metal-aarch64-darwin/_ops.py +3 -3
build/torch212-metal-aarch64-darwin/metadata.json +5 -5
build/torch213-metal-aarch64-darwin/__init__.py +21 -1
build/torch213-metal-aarch64-darwin/{_ggml_quantization_metal_091adac.abi3.so → _ggml_quantization_metal_9534e0b.abi3.so} +2 -2
build/torch213-metal-aarch64-darwin/_ops.py +3 -3
build/torch213-metal-aarch64-darwin/metadata.json +5 -5

build/torch212-metal-aarch64-darwin/__init__.py CHANGED Viewed

@@ -14,7 +14,7 @@ import torch
 from ._ops import add_op_namespace_prefix, ops
-__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_vec"]
 # Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
@@ -62,6 +62,21 @@ def mul_mat_vec(
     return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
 # Without these, torch.compile cannot trace the ops and breaks the graph at every call.
 @torch.library.register_fake(add_op_namespace_prefix("get_rows"))
 def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
@@ -76,3 +91,8 @@ def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
 @torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
 def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
     return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

 from ._ops import add_op_namespace_prefix, ops
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_id", "mul_mat_vec"]
 # Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
     return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+def mul_mat_id(
+    blocks: torch.Tensor, x: torch.Tensor, ids: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """One dispatch for a bank of routed experts: ggml's `mul_mv_id`.
+    `blocks` is `(n_experts, out_features, bytes_per_row)`, `x` is `(n_tokens, in_features)`, and
+    `ids` is `(n_tokens, n_used)` naming the expert each of a token's slots picked. The result is
+    `(n_tokens, n_used, out_features)` f32, one row per slot.
+    The alternative is a gemv per expert per layer, whose arithmetic is dwarfed by the dispatch
+    around it -- which is most of what a MoE decode step costs.
+    """
+    return ops.mul_mat_id(blocks, x, ids, ggml_type, out_features)
 # Without these, torch.compile cannot trace the ops and breaks the graph at every call.
 @torch.library.register_fake(add_op_namespace_prefix("get_rows"))
 def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
 @torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
 def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
     return x.new_empty((x.shape[0], out_features), dtype=torch.float32)
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_id"))
+def _mul_mat_id_fake(blocks, x, ids, ggml_type, out_features):
+    return x.new_empty((x.shape[0], ids.shape[1], out_features), dtype=torch.float32)

build/torch212-metal-aarch64-darwin/{_ggml_quantization_metal_091adac.abi3.so → _ggml_quantization_metal_9534e0b.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b61bbc3b9ec41e3e18079b49c09829cb30fc1d754937a2ce9dd2c14113169846
-size 817976

 version https://git-lfs.github.com/spec/v1
+oid sha256:cb93e4485ad772dddca1ee0510c8b2c50ddacae22914c6a6164ae37deb9430a6
+size 3347416

build/torch212-metal-aarch64-darwin/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _ggml_quantization_metal_091adac
-ops = torch.ops._ggml_quantization_metal_091adac
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_ggml_quantization_metal_091adac::{op_name}"

 import torch
+from . import _ggml_quantization_metal_9534e0b
+ops = torch.ops._ggml_quantization_metal_9534e0b
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_ggml_quantization_metal_9534e0b::{op_name}"

build/torch212-metal-aarch64-darwin/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "ggml-quantization",
-  "id": "_ggml_quantization_metal_091adac",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -10,9 +10,9 @@
   "digest": {
     "algorithm": "sha256",
     "files": {
-      "__init__.py": "68gDnPDp55W5ZSVOVBPJqML9O/YHq9mzcdCvNs4rnEg=",
-      "_ggml_quantization_metal_091adac.abi3.so": "thu8O57EHj4YB5tJwJgpyzD8HXVJN6LOndLBQRMWmEY=",
-      "_ops.py": "pPsLI0u39Io1wZY3MyuneW/N/5Jz9X7/leL+zhg5U+4="
     }
   },
   "provenance": {
@@ -22,7 +22,7 @@
       "dirty": false
     },
     "kernel": {
-      "sha": "091adacf825b75e297440a367f476b71d7136086",
       "dirty": false
     }
   }

 {
   "name": "ggml-quantization",
+  "id": "_ggml_quantization_metal_9534e0b",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
   "digest": {
     "algorithm": "sha256",
     "files": {
+      "__init__.py": "iPXgZXBgY3CzpNUSjmCovMVNPLBol1KBaNoTzkXp5mM=",
+      "_ggml_quantization_metal_9534e0b.abi3.so": "y5PkSFrXct3coe4FEMiyxQ3ayuIpFMamFkrjfeuUMKY=",
+      "_ops.py": "sK+sDWDDfYKJPFFj+YTyiancoGDCqiBXKU7B6dK+xvg="
     }
   },
   "provenance": {
       "dirty": false
     },
     "kernel": {
+      "sha": "9534e0b885d397dae61825fd15c158b94e815650",
       "dirty": false
     }
   }

build/torch213-metal-aarch64-darwin/__init__.py CHANGED Viewed

@@ -14,7 +14,7 @@ import torch
 from ._ops import add_op_namespace_prefix, ops
-__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_vec"]
 # Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
@@ -62,6 +62,21 @@ def mul_mat_vec(
     return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
 # Without these, torch.compile cannot trace the ops and breaks the graph at every call.
 @torch.library.register_fake(add_op_namespace_prefix("get_rows"))
 def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
@@ -76,3 +91,8 @@ def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
 @torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
 def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
     return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

 from ._ops import add_op_namespace_prefix, ops
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_id", "mul_mat_vec"]
 # Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
 # caller with more (prefill) dequantizes and uses an ordinary matmul.
     return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+def mul_mat_id(
+    blocks: torch.Tensor, x: torch.Tensor, ids: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """One dispatch for a bank of routed experts: ggml's `mul_mv_id`.
+    `blocks` is `(n_experts, out_features, bytes_per_row)`, `x` is `(n_tokens, in_features)`, and
+    `ids` is `(n_tokens, n_used)` naming the expert each of a token's slots picked. The result is
+    `(n_tokens, n_used, out_features)` f32, one row per slot.
+    The alternative is a gemv per expert per layer, whose arithmetic is dwarfed by the dispatch
+    around it -- which is most of what a MoE decode step costs.
+    """
+    return ops.mul_mat_id(blocks, x, ids, ggml_type, out_features)
 # Without these, torch.compile cannot trace the ops and breaks the graph at every call.
 @torch.library.register_fake(add_op_namespace_prefix("get_rows"))
 def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
 @torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
 def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
     return x.new_empty((x.shape[0], out_features), dtype=torch.float32)
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_id"))
+def _mul_mat_id_fake(blocks, x, ids, ggml_type, out_features):
+    return x.new_empty((x.shape[0], ids.shape[1], out_features), dtype=torch.float32)

build/torch213-metal-aarch64-darwin/{_ggml_quantization_metal_091adac.abi3.so → _ggml_quantization_metal_9534e0b.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d846ff14beeb1526e9b6248bf13543a1ce982c03be4b8904e2e02bb3d06fc03
-size 817976

 version https://git-lfs.github.com/spec/v1
+oid sha256:1b6a747eb385d07f816173e4506930633a671e56a68cf52cd59c164344847381
+size 3347416

build/torch213-metal-aarch64-darwin/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _ggml_quantization_metal_091adac
-ops = torch.ops._ggml_quantization_metal_091adac
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_ggml_quantization_metal_091adac::{op_name}"

 import torch
+from . import _ggml_quantization_metal_9534e0b
+ops = torch.ops._ggml_quantization_metal_9534e0b
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_ggml_quantization_metal_9534e0b::{op_name}"

build/torch213-metal-aarch64-darwin/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "ggml-quantization",
-  "id": "_ggml_quantization_metal_091adac",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -10,9 +10,9 @@
   "digest": {
     "algorithm": "sha256",
     "files": {
-      "__init__.py": "68gDnPDp55W5ZSVOVBPJqML9O/YHq9mzcdCvNs4rnEg=",
-      "_ggml_quantization_metal_091adac.abi3.so": "PYRv8UvusVJum2JIvxNUOhzpgsA75LiQTi4Cuz0G/AM=",
-      "_ops.py": "pPsLI0u39Io1wZY3MyuneW/N/5Jz9X7/leL+zhg5U+4="
     }
   },
   "provenance": {
@@ -22,7 +22,7 @@
       "dirty": false
     },
     "kernel": {
-      "sha": "091adacf825b75e297440a367f476b71d7136086",
       "dirty": false
     }
   }

 {
   "name": "ggml-quantization",
+  "id": "_ggml_quantization_metal_9534e0b",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
   "digest": {
     "algorithm": "sha256",
     "files": {
+      "__init__.py": "iPXgZXBgY3CzpNUSjmCovMVNPLBol1KBaNoTzkXp5mM=",
+      "_ggml_quantization_metal_9534e0b.abi3.so": "G2p0frOF0H+BYXPkUGkwYzpnHlamjPUs1ZwWQ0SEc4E=",
+      "_ops.py": "sK+sDWDDfYKJPFFj+YTyiancoGDCqiBXKU7B6dK+xvg="
     }
   },
   "provenance": {
       "dirty": false
     },
     "kernel": {
+      "sha": "9534e0b885d397dae61825fd15c158b94e815650",
       "dirty": false
     }
   }