dispatch ggml's f32 gemv for unquantized weights

build/torch212-metal-aarch64-darwin/_ggml_quantization_metal_9534e0b_dirty.abi3.so

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5cfbd7846fed56165a528fdb29d0c3245bfd8a3d3f45c659232ffc92592b3f57
-size 3347472

build/torch212-metal-aarch64-darwin/_ops.py

@@ -1,9 +1,9 @@
 import torch
-from . import _ggml_quantization_metal_9534e0b_dirty
-ops = torch.ops._ggml_quantization_metal_9534e0b_dirty
+from . import _ggml_quantization_metal_e1c26fb_dirty
+ops = torch.ops._ggml_quantization_metal_e1c26fb_dirty
 
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_ggml_quantization_metal_9534e0b_dirty::{op_name}"
+    return f"_ggml_quantization_metal_e1c26fb_dirty::{op_name}"

build/torch212-metal-aarch64-darwin/metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "ggml-quantization",
-  "id": "_ggml_quantization_metal_9534e0b_dirty",
+  "id": "_ggml_quantization_metal_e1c26fb_dirty",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -11,8 +11,8 @@
     "algorithm": "sha256",
     "files": {
       "__init__.py": "iPXgZXBgY3CzpNUSjmCovMVNPLBol1KBaNoTzkXp5mM=",
-      "_ggml_quantization_metal_9534e0b_dirty.abi3.so": "XPvXhG/tVhZaUo/bKdDDJFv9ij0/RcZZIy/8klkrP1c=",
-      "_ops.py": "e/VERCOzF+fbeuBbr0rD/FwcaQgotDQRKK9XD5G0Sqo="
+      "_ggml_quantization_metal_e1c26fb_dirty.abi3.so": "h8e+W6XgHY6Nq5KUfmDnI9LZUxHexKjA/O2yuwuFzko=",
+      "_ops.py": "Z1v1Vem/A3QJA8XQki0lB8D80aXJJ+Zsma16emhbRNw="
     }
   },
   "provenance": {
@@ -22,7 +22,7 @@
       "dirty": false
     },
     "kernel": {
-      "sha": "9534e0b885d397dae61825fd15c158b94e815650",
+      "sha": "e1c26fbaad0bd4b9ef9a3c48435942a4bc560f0a",
       "dirty": true
     }
   }

build/torch213-metal-aarch64-darwin/_ggml_quantization_metal_9534e0b_dirty.abi3.so

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:deae8683c779e9477721573ac7f0b3abccccd11aebef0a49c27d4ae7a8cd7f91
-size 3347472

build/torch213-metal-aarch64-darwin/_ops.py

@@ -1,9 +1,9 @@
 import torch
-from . import _ggml_quantization_metal_9534e0b_dirty
-ops = torch.ops._ggml_quantization_metal_9534e0b_dirty
+from . import _ggml_quantization_metal_e1c26fb_dirty
+ops = torch.ops._ggml_quantization_metal_e1c26fb_dirty
 
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_ggml_quantization_metal_9534e0b_dirty::{op_name}"
+    return f"_ggml_quantization_metal_e1c26fb_dirty::{op_name}"

build/torch213-metal-aarch64-darwin/metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "ggml-quantization",
-  "id": "_ggml_quantization_metal_9534e0b_dirty",
+  "id": "_ggml_quantization_metal_e1c26fb_dirty",
   "version": 1,
   "license": "MIT",
   "python-depends": [],
@@ -11,8 +11,8 @@
     "algorithm": "sha256",
     "files": {
       "__init__.py": "iPXgZXBgY3CzpNUSjmCovMVNPLBol1KBaNoTzkXp5mM=",
-      "_ggml_quantization_metal_9534e0b_dirty.abi3.so": "3q6Gg8d56Ud3IVc6x/Czq8zM0Rrr7wpJwn1K56jNf5E=",
-      "_ops.py": "e/VERCOzF+fbeuBbr0rD/FwcaQgotDQRKK9XD5G0Sqo="
+      "_ggml_quantization_metal_e1c26fb_dirty.abi3.so": "51EmqssYeOEoRK/C2bQHIY5tLWmK0Rp7GKMboo4w0u4=",
+      "_ops.py": "Z1v1Vem/A3QJA8XQki0lB8D80aXJJ+Zsma16emhbRNw="
     }
   },
   "provenance": {
@@ -22,7 +22,7 @@
       "dirty": false
     },
     "kernel": {
-      "sha": "9534e0b885d397dae61825fd15c158b94e815650",
+      "sha": "e1c26fbaad0bd4b9ef9a3c48435942a4bc560f0a",
       "dirty": true
     }
   }

gguf_metal/ggml_dispatch.mm

@@ -29,7 +29,7 @@
 namespace {
 
 // ggml type ids, matching ggml.h
-enum { GGML_Q4_0 = 2, GGML_Q4_1 = 3, GGML_Q5_0 = 6, GGML_Q5_1 = 7, GGML_Q8_0 = 8,
+enum { GGML_F32 = 0, GGML_Q4_0 = 2, GGML_Q4_1 = 3, GGML_Q5_0 = 6, GGML_Q5_1 = 7, GGML_Q8_0 = 8,
        GGML_Q2_K = 10, GGML_Q3_K = 11, GGML_Q4_K = 12, GGML_Q5_K = 13, GGML_Q6_K = 14,
        GGML_IQ2_XXS = 16, GGML_IQ2_XS = 17, GGML_IQ3_XXS = 18, GGML_IQ1_S = 19,
        GGML_IQ4_NL = 20, GGML_IQ3_S = 21, GGML_IQ2_S = 22, GGML_IQ4_XS = 23,
@@ -59,6 +59,10 @@ struct TypeInfo {
 // which upstream's Metal backend has no kernel for. Block sizes are ggml's GGML_QUANT_SIZES.
 const std::unordered_map<int, TypeInfo> &type_table() {
   static const std::unordered_map<int, TypeInfo> table = {
+      // A dense f32 weight, which a GGUF stores unquantized: the norms, the router, and any tensor
+      // llama.cpp declined to quantize. Upstream picks this kernel's grid from the row length rather
+      // than from a table, so the nsg/nr0/smem below are placeholders that `mv_params` overrides.
+      {GGML_F32, {"f32", 1, 4, 1, 2, true, 0}},
       // legacy quants
       {GGML_Q4_0, {"q4_0", 32, 18, N_SG_Q4_0, N_R0_Q4_0, false, 0}},
       {GGML_Q4_1, {"q4_1", 32, 20, N_SG_Q4_1, N_R0_Q4_1, false, 0}},
@@ -86,6 +90,26 @@ const std::unordered_map<int, TypeInfo> &type_table() {
   return table;
 }
 
+// What `ggml_metal_library_get_pipeline_mul_mv` decides inline for the dense types, where the grid
+// follows the row length instead of a per-type constant. Quantized types keep their table values.
+struct MvParams {
+  int nsg;
+  int nr0;
+  size_t smem;
+  const char *suffix;
+};
+
+MvParams mv_params(const TypeInfo *info, int64_t K) {
+  if (info->block_elems != 1) {
+    return {info->mv_nsg, info->mv_nr0, info->mv_smem, ""};
+  }
+  if (K < 32) {
+    return {1, 32, 0, "_short"};
+  }
+  const int nsg = (int)MIN((int64_t)4, (K + 127) / 128);
+  return {nsg, 2, 32 * sizeof(float) * 2, (K % 4 == 0) ? "_4" : ""};
+}
+
 const TypeInfo *lookup(int ggml_type) {
   auto it = type_table().find(ggml_type);
   return it == type_table().end() ? nullptr : &it->second;
@@ -202,9 +226,10 @@ extern "C" int gguf_metal_mul_mat(void *blocks, size_t blocks_off, void *x, size
 
   if (M <= 8) {
     // gemv: one token (or a handful), the decode path
-    const int nsg = info->mv_nsg, nr0 = info->mv_nr0;
+    const MvParams mv = mv_params(info, K);
+    const int nsg = mv.nsg, nr0 = mv.nr0;
     char fn[128];
-    snprintf(fn, sizeof(fn), "kernel_mul_mv_%s_f32", info->name);
+    snprintf(fn, sizeof(fn), "kernel_mul_mv_%s_f32%s", info->name, mv.suffix);
     std::string key = std::string(fn) + "_nsg=" + std::to_string(nsg);
 
     MTLFunctionConstantValues *cv = [MTLFunctionConstantValues new];
@@ -232,8 +257,8 @@ extern "C" int gguf_metal_mul_mat(void *blocks, size_t blocks_off, void *x, size
       [enc setBuffer:b_blocks offset:blocks_off atIndex:1];
       [enc setBuffer:b_x offset:x_off atIndex:2];
       [enc setBuffer:b_out offset:out_off atIndex:3];
-      if (info->mv_smem) {
-        [enc setThreadgroupMemoryLength:info->mv_smem atIndex:0];
+      if (mv.smem) {
+        [enc setThreadgroupMemoryLength:mv.smem atIndex:0];
       }
       const NSUInteger gx = info->mv_reduce_across_sgs ? (N + nr0 - 1) / nr0
                                                        : (N + nr0 * nsg - 1) / (nr0 * nsg);