Marc Sun commited on
Commit ·
a03d80d
1
Parent(s): a5ee429
stop tracking build/, like the other ggml kernels
Browse files`build/` is already gitignored here, and `ggml-attn` and `ggml-gated-delta-net` never tracked it: the
artifacts a client downloads live on the kernel repo's `v1` branch, which `build-and-upload` writes.
Keeping a second copy in git only creates two things to keep in step, and they had already drifted --
what shipped was stamped 09da59d, what was committed here 1f05408.
- .gitattributes +0 -2
- build/torch212-metal-aarch64-darwin/__init__.py +0 -78
- build/torch212-metal-aarch64-darwin/_ggml_quantization_metal_09da59d.abi3.so +0 -3
- build/torch212-metal-aarch64-darwin/_ops.py +0 -9
- build/torch212-metal-aarch64-darwin/metadata.json +0 -29
- build/torch213-metal-aarch64-darwin/__init__.py +0 -78
- build/torch213-metal-aarch64-darwin/_ggml_quantization_metal_09da59d.abi3.so +0 -3
- build/torch213-metal-aarch64-darwin/_ops.py +0 -9
- build/torch213-metal-aarch64-darwin/metadata.json +0 -29
.gitattributes
CHANGED
|
@@ -33,5 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
# Every built artifact, whatever hash the build stamps into its name.
|
| 37 |
-
build/**/*.abi3.so filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
build/torch212-metal-aarch64-darwin/__init__.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
| 1 |
-
"""Compute directly on the packed blocks of a GGUF checkpoint.
|
| 2 |
-
|
| 3 |
-
A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
|
| 4 |
-
layout as it is, so a quantized model can be loaded and run without ever materializing a dense
|
| 5 |
-
copy — which is the whole memory saving.
|
| 6 |
-
|
| 7 |
-
Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
|
| 8 |
-
they do rather than for a backend: each backend registers its own implementation of the same
|
| 9 |
-
schema, so calls dispatch on the tensor's device.
|
| 10 |
-
"""
|
| 11 |
-
|
| 12 |
-
import torch
|
| 13 |
-
|
| 14 |
-
from ._ops import add_op_namespace_prefix, ops
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_vec"]
|
| 18 |
-
|
| 19 |
-
# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
|
| 20 |
-
# caller with more (prefill) dequantizes and uses an ordinary matmul.
|
| 21 |
-
MAX_GEMV_ROWS = 8
|
| 22 |
-
|
| 23 |
-
# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
|
| 24 |
-
# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
|
| 25 |
-
# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
|
| 26 |
-
# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
|
| 27 |
-
# check this before choosing the fused path.
|
| 28 |
-
try:
|
| 29 |
-
GEMV_TYPES = frozenset(ops.gemv_types())
|
| 30 |
-
except AttributeError:
|
| 31 |
-
# A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
|
| 32 |
-
GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
def get_rows(
|
| 36 |
-
blocks: torch.Tensor, indices: torch.Tensor, ggml_type: int, cols: int, dtype: torch.dtype
|
| 37 |
-
) -> torch.Tensor:
|
| 38 |
-
"""The rows `indices` names, unpacked: `(rows, bytes_per_row)` uint8 -> `(len(indices), cols)`.
|
| 39 |
-
|
| 40 |
-
ggml's `get_rows`, which dequantizes as it gathers -- so reading a few rows out of a large table
|
| 41 |
-
never touches the rest.
|
| 42 |
-
"""
|
| 43 |
-
return ops.get_rows(blocks, indices, ggml_type, cols, dtype)
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def dequantize(
|
| 47 |
-
blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
|
| 48 |
-
) -> torch.Tensor:
|
| 49 |
-
"""`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
|
| 50 |
-
return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def mul_mat_vec(
|
| 54 |
-
blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
|
| 55 |
-
) -> torch.Tensor:
|
| 56 |
-
"""Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
|
| 57 |
-
|
| 58 |
-
`x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
|
| 59 |
-
whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
|
| 60 |
-
where the cast can be fused into whatever consumes it.
|
| 61 |
-
"""
|
| 62 |
-
return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
|
| 66 |
-
@torch.library.register_fake(add_op_namespace_prefix("get_rows"))
|
| 67 |
-
def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
|
| 68 |
-
return blocks.new_empty((indices.numel(), cols), dtype=dtype)
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
|
| 72 |
-
def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
|
| 73 |
-
return blocks.new_empty((rows, cols), dtype=dtype)
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
|
| 77 |
-
def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
|
| 78 |
-
return x.new_empty((x.shape[0], out_features), dtype=torch.float32)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-metal-aarch64-darwin/_ggml_quantization_metal_09da59d.abi3.so
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:f3ec2c0bd762a1cb5855006a816cdb22f18c11e969006d17131460e35e577ebb
|
| 3 |
-
size 817976
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-metal-aarch64-darwin/_ops.py
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
from . import _ggml_quantization_metal_09da59d
|
| 3 |
-
ops = torch.ops._ggml_quantization_metal_09da59d
|
| 4 |
-
|
| 5 |
-
def add_op_namespace_prefix(op_name: str):
|
| 6 |
-
"""
|
| 7 |
-
Prefix op by namespace.
|
| 8 |
-
"""
|
| 9 |
-
return f"_ggml_quantization_metal_09da59d::{op_name}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch212-metal-aarch64-darwin/metadata.json
DELETED
|
@@ -1,29 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"name": "ggml-quantization",
|
| 3 |
-
"id": "_ggml_quantization_metal_09da59d",
|
| 4 |
-
"version": 1,
|
| 5 |
-
"license": "MIT",
|
| 6 |
-
"python-depends": [],
|
| 7 |
-
"backend": {
|
| 8 |
-
"type": "metal"
|
| 9 |
-
},
|
| 10 |
-
"digest": {
|
| 11 |
-
"algorithm": "sha256",
|
| 12 |
-
"files": {
|
| 13 |
-
"__init__.py": "68gDnPDp55W5ZSVOVBPJqML9O/YHq9mzcdCvNs4rnEg=",
|
| 14 |
-
"_ggml_quantization_metal_09da59d.abi3.so": "8+wsC9dioctYVQBqgWzbIvGMEelpAG0XExRg415Xfrs=",
|
| 15 |
-
"_ops.py": "KPScFwQZRvXLLvmzyYkY4+0hIJB+X5ZJ9opUxecMnZs="
|
| 16 |
-
}
|
| 17 |
-
},
|
| 18 |
-
"provenance": {
|
| 19 |
-
"kernel-builder": {
|
| 20 |
-
"version": "0.17.0-dev0",
|
| 21 |
-
"sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
|
| 22 |
-
"dirty": false
|
| 23 |
-
},
|
| 24 |
-
"kernel": {
|
| 25 |
-
"sha": "09da59d51d3f73baa39a941eb07bf6c386223f82",
|
| 26 |
-
"dirty": false
|
| 27 |
-
}
|
| 28 |
-
}
|
| 29 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch213-metal-aarch64-darwin/__init__.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
| 1 |
-
"""Compute directly on the packed blocks of a GGUF checkpoint.
|
| 2 |
-
|
| 3 |
-
A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
|
| 4 |
-
layout as it is, so a quantized model can be loaded and run without ever materializing a dense
|
| 5 |
-
copy — which is the whole memory saving.
|
| 6 |
-
|
| 7 |
-
Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
|
| 8 |
-
they do rather than for a backend: each backend registers its own implementation of the same
|
| 9 |
-
schema, so calls dispatch on the tensor's device.
|
| 10 |
-
"""
|
| 11 |
-
|
| 12 |
-
import torch
|
| 13 |
-
|
| 14 |
-
from ._ops import add_op_namespace_prefix, ops
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_vec"]
|
| 18 |
-
|
| 19 |
-
# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
|
| 20 |
-
# caller with more (prefill) dequantizes and uses an ordinary matmul.
|
| 21 |
-
MAX_GEMV_ROWS = 8
|
| 22 |
-
|
| 23 |
-
# ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
|
| 24 |
-
# a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
|
| 25 |
-
# Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
|
| 26 |
-
# same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
|
| 27 |
-
# check this before choosing the fused path.
|
| 28 |
-
try:
|
| 29 |
-
GEMV_TYPES = frozenset(ops.gemv_types())
|
| 30 |
-
except AttributeError:
|
| 31 |
-
# A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
|
| 32 |
-
GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
def get_rows(
|
| 36 |
-
blocks: torch.Tensor, indices: torch.Tensor, ggml_type: int, cols: int, dtype: torch.dtype
|
| 37 |
-
) -> torch.Tensor:
|
| 38 |
-
"""The rows `indices` names, unpacked: `(rows, bytes_per_row)` uint8 -> `(len(indices), cols)`.
|
| 39 |
-
|
| 40 |
-
ggml's `get_rows`, which dequantizes as it gathers -- so reading a few rows out of a large table
|
| 41 |
-
never touches the rest.
|
| 42 |
-
"""
|
| 43 |
-
return ops.get_rows(blocks, indices, ggml_type, cols, dtype)
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def dequantize(
|
| 47 |
-
blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
|
| 48 |
-
) -> torch.Tensor:
|
| 49 |
-
"""`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
|
| 50 |
-
return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def mul_mat_vec(
|
| 54 |
-
blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
|
| 55 |
-
) -> torch.Tensor:
|
| 56 |
-
"""Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
|
| 57 |
-
|
| 58 |
-
`x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
|
| 59 |
-
whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
|
| 60 |
-
where the cast can be fused into whatever consumes it.
|
| 61 |
-
"""
|
| 62 |
-
return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
|
| 66 |
-
@torch.library.register_fake(add_op_namespace_prefix("get_rows"))
|
| 67 |
-
def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
|
| 68 |
-
return blocks.new_empty((indices.numel(), cols), dtype=dtype)
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
|
| 72 |
-
def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
|
| 73 |
-
return blocks.new_empty((rows, cols), dtype=dtype)
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
|
| 77 |
-
def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
|
| 78 |
-
return x.new_empty((x.shape[0], out_features), dtype=torch.float32)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch213-metal-aarch64-darwin/_ggml_quantization_metal_09da59d.abi3.so
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:391156ef4b8ec8f557c1ecb59a89d716a289e41e1c3853c6f0566330aafe196a
|
| 3 |
-
size 817976
|
|
|
|
|
|
|
|
|
|
|
|
build/torch213-metal-aarch64-darwin/_ops.py
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
from . import _ggml_quantization_metal_09da59d
|
| 3 |
-
ops = torch.ops._ggml_quantization_metal_09da59d
|
| 4 |
-
|
| 5 |
-
def add_op_namespace_prefix(op_name: str):
|
| 6 |
-
"""
|
| 7 |
-
Prefix op by namespace.
|
| 8 |
-
"""
|
| 9 |
-
return f"_ggml_quantization_metal_09da59d::{op_name}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch213-metal-aarch64-darwin/metadata.json
DELETED
|
@@ -1,29 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"name": "ggml-quantization",
|
| 3 |
-
"id": "_ggml_quantization_metal_09da59d",
|
| 4 |
-
"version": 1,
|
| 5 |
-
"license": "MIT",
|
| 6 |
-
"python-depends": [],
|
| 7 |
-
"backend": {
|
| 8 |
-
"type": "metal"
|
| 9 |
-
},
|
| 10 |
-
"digest": {
|
| 11 |
-
"algorithm": "sha256",
|
| 12 |
-
"files": {
|
| 13 |
-
"__init__.py": "68gDnPDp55W5ZSVOVBPJqML9O/YHq9mzcdCvNs4rnEg=",
|
| 14 |
-
"_ggml_quantization_metal_09da59d.abi3.so": "ORFW70uOyPVXwey1monXFqKJ5B4cOFPG8FZjMKr+GWo=",
|
| 15 |
-
"_ops.py": "KPScFwQZRvXLLvmzyYkY4+0hIJB+X5ZJ9opUxecMnZs="
|
| 16 |
-
}
|
| 17 |
-
},
|
| 18 |
-
"provenance": {
|
| 19 |
-
"kernel-builder": {
|
| 20 |
-
"version": "0.17.0-dev0",
|
| 21 |
-
"sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
|
| 22 |
-
"dirty": false
|
| 23 |
-
},
|
| 24 |
-
"kernel": {
|
| 25 |
-
"sha": "09da59d51d3f73baa39a941eb07bf6c386223f82",
|
| 26 |
-
"dirty": false
|
| 27 |
-
}
|
| 28 |
-
}
|
| 29 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|