Marc Sun commited on
Commit
a03d80d
·
1 Parent(s): a5ee429

stop tracking build/, like the other ggml kernels

Browse files

`build/` is already gitignored here, and `ggml-attn` and `ggml-gated-delta-net` never tracked it: the
artifacts a client downloads live on the kernel repo's `v1` branch, which `build-and-upload` writes.
Keeping a second copy in git only creates two things to keep in step, and they had already drifted --
what shipped was stamped 09da59d, what was committed here 1f05408.

.gitattributes CHANGED
@@ -33,5 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
- # Every built artifact, whatever hash the build stamps into its name.
37
- build/**/*.abi3.so filter=lfs diff=lfs merge=lfs -text
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
build/torch212-metal-aarch64-darwin/__init__.py DELETED
@@ -1,78 +0,0 @@
1
- """Compute directly on the packed blocks of a GGUF checkpoint.
2
-
3
- A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
4
- layout as it is, so a quantized model can be loaded and run without ever materializing a dense
5
- copy — which is the whole memory saving.
6
-
7
- Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
8
- they do rather than for a backend: each backend registers its own implementation of the same
9
- schema, so calls dispatch on the tensor's device.
10
- """
11
-
12
- import torch
13
-
14
- from ._ops import add_op_namespace_prefix, ops
15
-
16
-
17
- __all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_vec"]
18
-
19
- # Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
20
- # caller with more (prefill) dequantizes and uses an ordinary matmul.
21
- MAX_GEMV_ROWS = 8
22
-
23
- # ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
24
- # a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
25
- # Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
26
- # same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
27
- # check this before choosing the fused path.
28
- try:
29
- GEMV_TYPES = frozenset(ops.gemv_types())
30
- except AttributeError:
31
- # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
32
- GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
33
-
34
-
35
- def get_rows(
36
- blocks: torch.Tensor, indices: torch.Tensor, ggml_type: int, cols: int, dtype: torch.dtype
37
- ) -> torch.Tensor:
38
- """The rows `indices` names, unpacked: `(rows, bytes_per_row)` uint8 -> `(len(indices), cols)`.
39
-
40
- ggml's `get_rows`, which dequantizes as it gathers -- so reading a few rows out of a large table
41
- never touches the rest.
42
- """
43
- return ops.get_rows(blocks, indices, ggml_type, cols, dtype)
44
-
45
-
46
- def dequantize(
47
- blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
48
- ) -> torch.Tensor:
49
- """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
50
- return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
51
-
52
-
53
- def mul_mat_vec(
54
- blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
55
- ) -> torch.Tensor:
56
- """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
57
-
58
- `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
59
- whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
60
- where the cast can be fused into whatever consumes it.
61
- """
62
- return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
63
-
64
-
65
- # Without these, torch.compile cannot trace the ops and breaks the graph at every call.
66
- @torch.library.register_fake(add_op_namespace_prefix("get_rows"))
67
- def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
68
- return blocks.new_empty((indices.numel(), cols), dtype=dtype)
69
-
70
-
71
- @torch.library.register_fake(add_op_namespace_prefix("dequantize"))
72
- def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
73
- return blocks.new_empty((rows, cols), dtype=dtype)
74
-
75
-
76
- @torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
77
- def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
78
- return x.new_empty((x.shape[0], out_features), dtype=torch.float32)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch212-metal-aarch64-darwin/_ggml_quantization_metal_09da59d.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f3ec2c0bd762a1cb5855006a816cdb22f18c11e969006d17131460e35e577ebb
3
- size 817976
 
 
 
 
build/torch212-metal-aarch64-darwin/_ops.py DELETED
@@ -1,9 +0,0 @@
1
- import torch
2
- from . import _ggml_quantization_metal_09da59d
3
- ops = torch.ops._ggml_quantization_metal_09da59d
4
-
5
- def add_op_namespace_prefix(op_name: str):
6
- """
7
- Prefix op by namespace.
8
- """
9
- return f"_ggml_quantization_metal_09da59d::{op_name}"
 
 
 
 
 
 
 
 
 
 
build/torch212-metal-aarch64-darwin/metadata.json DELETED
@@ -1,29 +0,0 @@
1
- {
2
- "name": "ggml-quantization",
3
- "id": "_ggml_quantization_metal_09da59d",
4
- "version": 1,
5
- "license": "MIT",
6
- "python-depends": [],
7
- "backend": {
8
- "type": "metal"
9
- },
10
- "digest": {
11
- "algorithm": "sha256",
12
- "files": {
13
- "__init__.py": "68gDnPDp55W5ZSVOVBPJqML9O/YHq9mzcdCvNs4rnEg=",
14
- "_ggml_quantization_metal_09da59d.abi3.so": "8+wsC9dioctYVQBqgWzbIvGMEelpAG0XExRg415Xfrs=",
15
- "_ops.py": "KPScFwQZRvXLLvmzyYkY4+0hIJB+X5ZJ9opUxecMnZs="
16
- }
17
- },
18
- "provenance": {
19
- "kernel-builder": {
20
- "version": "0.17.0-dev0",
21
- "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
22
- "dirty": false
23
- },
24
- "kernel": {
25
- "sha": "09da59d51d3f73baa39a941eb07bf6c386223f82",
26
- "dirty": false
27
- }
28
- }
29
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch213-metal-aarch64-darwin/__init__.py DELETED
@@ -1,78 +0,0 @@
1
- """Compute directly on the packed blocks of a GGUF checkpoint.
2
-
3
- A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
4
- layout as it is, so a quantized model can be loaded and run without ever materializing a dense
5
- copy — which is the whole memory saving.
6
-
7
- Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
8
- they do rather than for a backend: each backend registers its own implementation of the same
9
- schema, so calls dispatch on the tensor's device.
10
- """
11
-
12
- import torch
13
-
14
- from ._ops import add_op_namespace_prefix, ops
15
-
16
-
17
- __all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "get_rows", "mul_mat_vec"]
18
-
19
- # Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
20
- # caller with more (prefill) dequantizes and uses an ordinary matmul.
21
- MAX_GEMV_ROWS = 8
22
-
23
- # ggml type ids `mul_mat_vec` implements, as this build reports them -- it is backend-specific, and
24
- # a type routed into a gemv that has no kernel for it is a fault, not a fallback. CUDA covers
25
- # Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, the IQ quants, MXFP4/NVFP4 and Q1_0/Q2_0; Metal is the
26
- # same minus NVFP4/Q1_0/Q2_0, which its metallib has no kernels for. `dequantize` covers more, so
27
- # check this before choosing the fused path.
28
- try:
29
- GEMV_TYPES = frozenset(ops.gemv_types())
30
- except AttributeError:
31
- # A build published before `gemv_types` existed; those are CUDA-only, so its list is theirs.
32
- GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
33
-
34
-
35
- def get_rows(
36
- blocks: torch.Tensor, indices: torch.Tensor, ggml_type: int, cols: int, dtype: torch.dtype
37
- ) -> torch.Tensor:
38
- """The rows `indices` names, unpacked: `(rows, bytes_per_row)` uint8 -> `(len(indices), cols)`.
39
-
40
- ggml's `get_rows`, which dequantizes as it gathers -- so reading a few rows out of a large table
41
- never touches the rest.
42
- """
43
- return ops.get_rows(blocks, indices, ggml_type, cols, dtype)
44
-
45
-
46
- def dequantize(
47
- blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
48
- ) -> torch.Tensor:
49
- """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
50
- return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
51
-
52
-
53
- def mul_mat_vec(
54
- blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
55
- ) -> torch.Tensor:
56
- """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
57
-
58
- `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
59
- whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
60
- where the cast can be fused into whatever consumes it.
61
- """
62
- return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
63
-
64
-
65
- # Without these, torch.compile cannot trace the ops and breaks the graph at every call.
66
- @torch.library.register_fake(add_op_namespace_prefix("get_rows"))
67
- def _get_rows_fake(blocks, indices, ggml_type, cols, dtype):
68
- return blocks.new_empty((indices.numel(), cols), dtype=dtype)
69
-
70
-
71
- @torch.library.register_fake(add_op_namespace_prefix("dequantize"))
72
- def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
73
- return blocks.new_empty((rows, cols), dtype=dtype)
74
-
75
-
76
- @torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
77
- def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
78
- return x.new_empty((x.shape[0], out_features), dtype=torch.float32)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch213-metal-aarch64-darwin/_ggml_quantization_metal_09da59d.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:391156ef4b8ec8f557c1ecb59a89d716a289e41e1c3853c6f0566330aafe196a
3
- size 817976
 
 
 
 
build/torch213-metal-aarch64-darwin/_ops.py DELETED
@@ -1,9 +0,0 @@
1
- import torch
2
- from . import _ggml_quantization_metal_09da59d
3
- ops = torch.ops._ggml_quantization_metal_09da59d
4
-
5
- def add_op_namespace_prefix(op_name: str):
6
- """
7
- Prefix op by namespace.
8
- """
9
- return f"_ggml_quantization_metal_09da59d::{op_name}"
 
 
 
 
 
 
 
 
 
 
build/torch213-metal-aarch64-darwin/metadata.json DELETED
@@ -1,29 +0,0 @@
1
- {
2
- "name": "ggml-quantization",
3
- "id": "_ggml_quantization_metal_09da59d",
4
- "version": 1,
5
- "license": "MIT",
6
- "python-depends": [],
7
- "backend": {
8
- "type": "metal"
9
- },
10
- "digest": {
11
- "algorithm": "sha256",
12
- "files": {
13
- "__init__.py": "68gDnPDp55W5ZSVOVBPJqML9O/YHq9mzcdCvNs4rnEg=",
14
- "_ggml_quantization_metal_09da59d.abi3.so": "ORFW70uOyPVXwey1monXFqKJ5B4cOFPG8FZjMKr+GWo=",
15
- "_ops.py": "KPScFwQZRvXLLvmzyYkY4+0hIJB+X5ZJ9opUxecMnZs="
16
- }
17
- },
18
- "provenance": {
19
- "kernel-builder": {
20
- "version": "0.17.0-dev0",
21
- "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
22
- "dirty": false
23
- },
24
- "kernel": {
25
- "sha": "09da59d51d3f73baa39a941eb07bf6c386223f82",
26
- "dirty": false
27
- }
28
- }
29
- }