liangsu9988 commited on
Commit
efafad7
·
verified ·
1 Parent(s): b7e9346

Uploaded using `kernel-builder`.

Browse files
benchmarks/benchmark.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Benchmark turboquant-kv source or installed artifacts."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ import torch
11
+
12
+
13
+ PACKAGE = Path(__file__).resolve().parents[1]
14
+ sys.path.insert(0, str(PACKAGE / "tests"))
15
+ from test_turboquant_kv import load_installed_ops, load_source_ops, ref_unpack # noqa: E402
16
+
17
+
18
+ def bench(fn, warmup: int, iters: int) -> float:
19
+ for _ in range(warmup):
20
+ fn()
21
+ torch.cuda.synchronize()
22
+ start = torch.cuda.Event(enable_timing=True)
23
+ end = torch.cuda.Event(enable_timing=True)
24
+ start.record()
25
+ for _ in range(iters):
26
+ fn()
27
+ end.record()
28
+ torch.cuda.synchronize()
29
+ return start.elapsed_time(end) * 1000.0 / iters
30
+
31
+
32
+ def make_packed(m: int):
33
+ k_idx = torch.randint(0, 256, (m, 128), device="cuda", dtype=torch.uint8)
34
+ k_qjl = torch.randint(0, 256, (m, 32), device="cuda", dtype=torch.uint8)
35
+ v_idx = torch.randint(0, 256, (m, 128), device="cuda", dtype=torch.uint8)
36
+ cb_k = torch.randn((16,), device="cuda", dtype=torch.float32)
37
+ cb_v = torch.randn((16,), device="cuda", dtype=torch.float32)
38
+ return k_idx, k_qjl, v_idx, cb_k, cb_v
39
+
40
+
41
+ def main() -> int:
42
+ parser = argparse.ArgumentParser()
43
+ parser.add_argument("--backend", choices=["source", "installed"], default="source")
44
+ parser.add_argument("--artifact", default=None)
45
+ parser.add_argument("--warmup", type=int, default=100)
46
+ parser.add_argument("--iters", type=int, default=1000)
47
+ args = parser.parse_args()
48
+
49
+ if not torch.cuda.is_available():
50
+ raise RuntimeError("CUDA is required")
51
+ torch.manual_seed(1234)
52
+ ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
53
+
54
+ print("| Workload | Shape | FlashRT us | PyTorch eager us | Speedup |")
55
+ print("|---|---:|---:|---:|---:|")
56
+
57
+ for m in [1, 4, 128, 1024, 4096, 32768]:
58
+ k_idx, k_qjl, v_idx, cb_k, cb_v = make_packed(m)
59
+ b_k, b_v = 3, 4
60
+ fused = bench(
61
+ lambda: ops.unpack_packed_bf16(k_idx, k_qjl, v_idx, cb_k, cb_v, b_k, b_v),
62
+ args.warmup,
63
+ args.iters,
64
+ )
65
+ eager = bench(
66
+ lambda: ref_unpack(k_idx, k_qjl, v_idx, cb_k, cb_v, b_k, b_v, torch.bfloat16),
67
+ args.warmup,
68
+ args.iters,
69
+ )
70
+ print(f"| unpack_packed_bf16 | M={m}, D=256, bits=3/4 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
71
+
72
+ fused = bench(
73
+ lambda: ops.unpack_packed_mixed(k_idx, k_qjl, v_idx, cb_k, cb_v, b_k, b_v),
74
+ args.warmup,
75
+ args.iters,
76
+ )
77
+ eager = bench(
78
+ lambda: ref_unpack(k_idx, k_qjl, v_idx, cb_k, cb_v, b_k, b_v, torch.float32),
79
+ args.warmup,
80
+ args.iters,
81
+ )
82
+ print(f"| unpack_packed_mixed | M={m}, D=256, bits=3/4 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
83
+
84
+ for m in [1, 4, 128, 1024, 4096, 32768]:
85
+ k_mse = torch.randn((m, 256), device="cuda", dtype=torch.bfloat16)
86
+ k_qjl = torch.randn((m, 256), device="cuda", dtype=torch.bfloat16)
87
+ v_unit = torch.randn((m, 256), device="cuda", dtype=torch.bfloat16)
88
+ k_norm = torch.rand((m,), device="cuda", dtype=torch.float16) + 0.5
89
+ k_rnorm = torch.rand((m,), device="cuda", dtype=torch.float16) + 0.5
90
+ v_norm = torch.rand((m,), device="cuda", dtype=torch.float16) + 0.5
91
+ coef = 0.125
92
+ fused = bench(
93
+ lambda: ops.combine_kv_bf16(k_mse, k_qjl, v_unit, k_norm, k_rnorm, v_norm, coef),
94
+ args.warmup,
95
+ args.iters,
96
+ )
97
+ eager = bench(
98
+ lambda: (
99
+ (k_norm.float().unsqueeze(1) * (k_mse.float() + coef * k_rnorm.float().unsqueeze(1) * k_qjl.float())).to(torch.bfloat16),
100
+ (v_norm.float().unsqueeze(1) * v_unit.float()).to(torch.bfloat16),
101
+ ),
102
+ args.warmup,
103
+ args.iters,
104
+ )
105
+ print(f"| combine_kv_bf16 | M={m}, D=256 | {fused:.3f} | {eager:.3f} | {eager / fused:.2f}x |")
106
+
107
+ return 0
108
+
109
+
110
+ if __name__ == "__main__":
111
+ raise SystemExit(main())
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT TurboQuant KV cache kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ def _check_packed(k_idx: torch.Tensor, k_qjl: torch.Tensor, v_idx: torch.Tensor) -> int:
13
+ if k_idx.dim() != 2 or k_idx.shape[1] != 128:
14
+ raise RuntimeError("k_idx_packed must have shape (M, 128)")
15
+ m = k_idx.shape[0]
16
+ if k_qjl.shape != (m, 32):
17
+ raise RuntimeError("k_qjl_packed must have shape (M, 32)")
18
+ if v_idx.shape != (m, 128):
19
+ raise RuntimeError("v_idx_packed must have shape (M, 128)")
20
+ return m
21
+
22
+
23
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_bf16"))
24
+ def _unpack_packed_bf16_fake(
25
+ k_idx_packed: torch.Tensor,
26
+ k_qjl_packed: torch.Tensor,
27
+ v_idx_packed: torch.Tensor,
28
+ cb_k_mse: torch.Tensor,
29
+ cb_v: torch.Tensor,
30
+ b_k_mse: int,
31
+ b_v: int,
32
+ y_k: torch.Tensor,
33
+ qjl_bf: torch.Tensor,
34
+ y_v: torch.Tensor,
35
+ ) -> None:
36
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
37
+ if y_k.shape != (m, 256) or qjl_bf.shape != (m, 256) or y_v.shape != (m, 256):
38
+ raise RuntimeError("outputs must have shape (M, 256)")
39
+ return None
40
+
41
+
42
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_mixed"))
43
+ def _unpack_packed_mixed_fake(
44
+ k_idx_packed: torch.Tensor,
45
+ k_qjl_packed: torch.Tensor,
46
+ v_idx_packed: torch.Tensor,
47
+ cb_k_mse: torch.Tensor,
48
+ cb_v: torch.Tensor,
49
+ b_k_mse: int,
50
+ b_v: int,
51
+ y_k: torch.Tensor,
52
+ qjl_f: torch.Tensor,
53
+ y_v: torch.Tensor,
54
+ ) -> None:
55
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
56
+ if y_k.shape != (m, 256) or qjl_f.shape != (m, 256) or y_v.shape != (m, 256):
57
+ raise RuntimeError("outputs must have shape (M, 256)")
58
+ return None
59
+
60
+
61
+ @torch.library.register_fake(add_op_namespace_prefix("combine_kv_bf16"))
62
+ def _combine_kv_bf16_fake(
63
+ k_mse: torch.Tensor,
64
+ k_qjl: torch.Tensor,
65
+ v_unit: torch.Tensor,
66
+ k_norm: torch.Tensor,
67
+ k_rnorm: torch.Tensor,
68
+ v_norm: torch.Tensor,
69
+ coef: float,
70
+ k_out: torch.Tensor,
71
+ v_out: torch.Tensor,
72
+ ) -> None:
73
+ if k_mse.dim() != 2 or k_mse.shape[1] != 256:
74
+ raise RuntimeError("k_mse must have shape (M, 256)")
75
+ m = k_mse.shape[0]
76
+ if k_qjl.shape != k_mse.shape or v_unit.shape != k_mse.shape:
77
+ raise RuntimeError("k_qjl and v_unit must have shape (M, 256)")
78
+ if k_norm.shape != (m,) or k_rnorm.shape != (m,) or v_norm.shape != (m,):
79
+ raise RuntimeError("norm tensors must have shape (M,)")
80
+ if k_out.shape != k_mse.shape or v_out.shape != k_mse.shape:
81
+ raise RuntimeError("outputs must have shape (M, 256)")
82
+ return None
83
+
84
+
85
+ def unpack_packed_bf16(
86
+ k_idx_packed: torch.Tensor,
87
+ k_qjl_packed: torch.Tensor,
88
+ v_idx_packed: torch.Tensor,
89
+ cb_k_mse: torch.Tensor,
90
+ cb_v: torch.Tensor,
91
+ b_k_mse: int,
92
+ b_v: int,
93
+ *,
94
+ y_k: Optional[torch.Tensor] = None,
95
+ qjl_bf: Optional[torch.Tensor] = None,
96
+ y_v: Optional[torch.Tensor] = None,
97
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
98
+ """Unpack TurboQuant packed indices to BF16 `(M, 256)` tensors."""
99
+
100
+ m = k_idx_packed.shape[0]
101
+ if y_k is None:
102
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
103
+ if qjl_bf is None:
104
+ qjl_bf = torch.empty_like(y_k)
105
+ if y_v is None:
106
+ y_v = torch.empty_like(y_k)
107
+ ops.unpack_packed_bf16(
108
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
109
+ int(b_k_mse), int(b_v), y_k, qjl_bf, y_v,
110
+ )
111
+ return y_k, qjl_bf, y_v
112
+
113
+
114
+ def unpack_packed_mixed(
115
+ k_idx_packed: torch.Tensor,
116
+ k_qjl_packed: torch.Tensor,
117
+ v_idx_packed: torch.Tensor,
118
+ cb_k_mse: torch.Tensor,
119
+ cb_v: torch.Tensor,
120
+ b_k_mse: int,
121
+ b_v: int,
122
+ *,
123
+ y_k: Optional[torch.Tensor] = None,
124
+ qjl_f: Optional[torch.Tensor] = None,
125
+ y_v: Optional[torch.Tensor] = None,
126
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
127
+ """Unpack packed K/V to BF16 K/V and FP32 QJL signs."""
128
+
129
+ m = k_idx_packed.shape[0]
130
+ if y_k is None:
131
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
132
+ if qjl_f is None:
133
+ qjl_f = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.float32)
134
+ if y_v is None:
135
+ y_v = torch.empty_like(y_k)
136
+ ops.unpack_packed_mixed(
137
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
138
+ int(b_k_mse), int(b_v), y_k, qjl_f, y_v,
139
+ )
140
+ return y_k, qjl_f, y_v
141
+
142
+
143
+ def combine_kv_bf16(
144
+ k_mse: torch.Tensor,
145
+ k_qjl: torch.Tensor,
146
+ v_unit: torch.Tensor,
147
+ k_norm: torch.Tensor,
148
+ k_rnorm: torch.Tensor,
149
+ v_norm: torch.Tensor,
150
+ coef: float,
151
+ *,
152
+ k_out: Optional[torch.Tensor] = None,
153
+ v_out: Optional[torch.Tensor] = None,
154
+ ) -> tuple[torch.Tensor, torch.Tensor]:
155
+ """Combine TurboQuant K/V GEMM outputs into BF16 K/V cache tensors."""
156
+
157
+ if k_out is None:
158
+ k_out = torch.empty_like(k_mse)
159
+ if v_out is None:
160
+ v_out = torch.empty_like(v_unit)
161
+ ops.combine_kv_bf16(k_mse, k_qjl, v_unit, k_norm, k_rnorm, v_norm, float(coef), k_out, v_out)
162
+ return k_out, v_out
163
+
164
+
165
+ __all__ = [
166
+ "combine_kv_bf16",
167
+ "unpack_packed_bf16",
168
+ "unpack_packed_mixed",
169
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _turboquant_kv_cuda_5596053
3
+ ops = torch.ops._turboquant_kv_cuda_5596053
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_turboquant_kv_cuda_5596053::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/_turboquant_kv_cuda_5596053.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6756ba0c92a10809f9384faeb10f5d0dfc08cdbec3f4f74072207973c5171c7
3
+ size 637432
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "turboquant-kv",
3
+ "id": "_turboquant_kv_cuda_5596053",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu128-x86_64-linux/turboquant_kv/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT TurboQuant KV cache kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ def _check_packed(k_idx: torch.Tensor, k_qjl: torch.Tensor, v_idx: torch.Tensor) -> int:
13
+ if k_idx.dim() != 2 or k_idx.shape[1] != 128:
14
+ raise RuntimeError("k_idx_packed must have shape (M, 128)")
15
+ m = k_idx.shape[0]
16
+ if k_qjl.shape != (m, 32):
17
+ raise RuntimeError("k_qjl_packed must have shape (M, 32)")
18
+ if v_idx.shape != (m, 128):
19
+ raise RuntimeError("v_idx_packed must have shape (M, 128)")
20
+ return m
21
+
22
+
23
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_bf16"))
24
+ def _unpack_packed_bf16_fake(
25
+ k_idx_packed: torch.Tensor,
26
+ k_qjl_packed: torch.Tensor,
27
+ v_idx_packed: torch.Tensor,
28
+ cb_k_mse: torch.Tensor,
29
+ cb_v: torch.Tensor,
30
+ b_k_mse: int,
31
+ b_v: int,
32
+ y_k: torch.Tensor,
33
+ qjl_bf: torch.Tensor,
34
+ y_v: torch.Tensor,
35
+ ) -> None:
36
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
37
+ if y_k.shape != (m, 256) or qjl_bf.shape != (m, 256) or y_v.shape != (m, 256):
38
+ raise RuntimeError("outputs must have shape (M, 256)")
39
+ return None
40
+
41
+
42
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_mixed"))
43
+ def _unpack_packed_mixed_fake(
44
+ k_idx_packed: torch.Tensor,
45
+ k_qjl_packed: torch.Tensor,
46
+ v_idx_packed: torch.Tensor,
47
+ cb_k_mse: torch.Tensor,
48
+ cb_v: torch.Tensor,
49
+ b_k_mse: int,
50
+ b_v: int,
51
+ y_k: torch.Tensor,
52
+ qjl_f: torch.Tensor,
53
+ y_v: torch.Tensor,
54
+ ) -> None:
55
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
56
+ if y_k.shape != (m, 256) or qjl_f.shape != (m, 256) or y_v.shape != (m, 256):
57
+ raise RuntimeError("outputs must have shape (M, 256)")
58
+ return None
59
+
60
+
61
+ @torch.library.register_fake(add_op_namespace_prefix("combine_kv_bf16"))
62
+ def _combine_kv_bf16_fake(
63
+ k_mse: torch.Tensor,
64
+ k_qjl: torch.Tensor,
65
+ v_unit: torch.Tensor,
66
+ k_norm: torch.Tensor,
67
+ k_rnorm: torch.Tensor,
68
+ v_norm: torch.Tensor,
69
+ coef: float,
70
+ k_out: torch.Tensor,
71
+ v_out: torch.Tensor,
72
+ ) -> None:
73
+ if k_mse.dim() != 2 or k_mse.shape[1] != 256:
74
+ raise RuntimeError("k_mse must have shape (M, 256)")
75
+ m = k_mse.shape[0]
76
+ if k_qjl.shape != k_mse.shape or v_unit.shape != k_mse.shape:
77
+ raise RuntimeError("k_qjl and v_unit must have shape (M, 256)")
78
+ if k_norm.shape != (m,) or k_rnorm.shape != (m,) or v_norm.shape != (m,):
79
+ raise RuntimeError("norm tensors must have shape (M,)")
80
+ if k_out.shape != k_mse.shape or v_out.shape != k_mse.shape:
81
+ raise RuntimeError("outputs must have shape (M, 256)")
82
+ return None
83
+
84
+
85
+ def unpack_packed_bf16(
86
+ k_idx_packed: torch.Tensor,
87
+ k_qjl_packed: torch.Tensor,
88
+ v_idx_packed: torch.Tensor,
89
+ cb_k_mse: torch.Tensor,
90
+ cb_v: torch.Tensor,
91
+ b_k_mse: int,
92
+ b_v: int,
93
+ *,
94
+ y_k: Optional[torch.Tensor] = None,
95
+ qjl_bf: Optional[torch.Tensor] = None,
96
+ y_v: Optional[torch.Tensor] = None,
97
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
98
+ """Unpack TurboQuant packed indices to BF16 `(M, 256)` tensors."""
99
+
100
+ m = k_idx_packed.shape[0]
101
+ if y_k is None:
102
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
103
+ if qjl_bf is None:
104
+ qjl_bf = torch.empty_like(y_k)
105
+ if y_v is None:
106
+ y_v = torch.empty_like(y_k)
107
+ ops.unpack_packed_bf16(
108
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
109
+ int(b_k_mse), int(b_v), y_k, qjl_bf, y_v,
110
+ )
111
+ return y_k, qjl_bf, y_v
112
+
113
+
114
+ def unpack_packed_mixed(
115
+ k_idx_packed: torch.Tensor,
116
+ k_qjl_packed: torch.Tensor,
117
+ v_idx_packed: torch.Tensor,
118
+ cb_k_mse: torch.Tensor,
119
+ cb_v: torch.Tensor,
120
+ b_k_mse: int,
121
+ b_v: int,
122
+ *,
123
+ y_k: Optional[torch.Tensor] = None,
124
+ qjl_f: Optional[torch.Tensor] = None,
125
+ y_v: Optional[torch.Tensor] = None,
126
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
127
+ """Unpack packed K/V to BF16 K/V and FP32 QJL signs."""
128
+
129
+ m = k_idx_packed.shape[0]
130
+ if y_k is None:
131
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
132
+ if qjl_f is None:
133
+ qjl_f = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.float32)
134
+ if y_v is None:
135
+ y_v = torch.empty_like(y_k)
136
+ ops.unpack_packed_mixed(
137
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
138
+ int(b_k_mse), int(b_v), y_k, qjl_f, y_v,
139
+ )
140
+ return y_k, qjl_f, y_v
141
+
142
+
143
+ def combine_kv_bf16(
144
+ k_mse: torch.Tensor,
145
+ k_qjl: torch.Tensor,
146
+ v_unit: torch.Tensor,
147
+ k_norm: torch.Tensor,
148
+ k_rnorm: torch.Tensor,
149
+ v_norm: torch.Tensor,
150
+ coef: float,
151
+ *,
152
+ k_out: Optional[torch.Tensor] = None,
153
+ v_out: Optional[torch.Tensor] = None,
154
+ ) -> tuple[torch.Tensor, torch.Tensor]:
155
+ """Combine TurboQuant K/V GEMM outputs into BF16 K/V cache tensors."""
156
+
157
+ if k_out is None:
158
+ k_out = torch.empty_like(k_mse)
159
+ if v_out is None:
160
+ v_out = torch.empty_like(v_unit)
161
+ ops.combine_kv_bf16(k_mse, k_qjl, v_unit, k_norm, k_rnorm, v_norm, float(coef), k_out, v_out)
162
+ return k_out, v_out
163
+
164
+
165
+ __all__ = [
166
+ "combine_kv_bf16",
167
+ "unpack_packed_bf16",
168
+ "unpack_packed_mixed",
169
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _turboquant_kv_cuda_5596053
3
+ ops = torch.ops._turboquant_kv_cuda_5596053
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_turboquant_kv_cuda_5596053::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/_turboquant_kv_cuda_5596053.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ad0a2a66d62ae6646e90826c2fd0e0a395eb63c76b3fa26c38a673ec287c77a
3
+ size 637992
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "turboquant-kv",
3
+ "id": "_turboquant_kv_cuda_5596053",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch211-cxx11-cu130-x86_64-linux/turboquant_kv/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT TurboQuant KV cache kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ def _check_packed(k_idx: torch.Tensor, k_qjl: torch.Tensor, v_idx: torch.Tensor) -> int:
13
+ if k_idx.dim() != 2 or k_idx.shape[1] != 128:
14
+ raise RuntimeError("k_idx_packed must have shape (M, 128)")
15
+ m = k_idx.shape[0]
16
+ if k_qjl.shape != (m, 32):
17
+ raise RuntimeError("k_qjl_packed must have shape (M, 32)")
18
+ if v_idx.shape != (m, 128):
19
+ raise RuntimeError("v_idx_packed must have shape (M, 128)")
20
+ return m
21
+
22
+
23
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_bf16"))
24
+ def _unpack_packed_bf16_fake(
25
+ k_idx_packed: torch.Tensor,
26
+ k_qjl_packed: torch.Tensor,
27
+ v_idx_packed: torch.Tensor,
28
+ cb_k_mse: torch.Tensor,
29
+ cb_v: torch.Tensor,
30
+ b_k_mse: int,
31
+ b_v: int,
32
+ y_k: torch.Tensor,
33
+ qjl_bf: torch.Tensor,
34
+ y_v: torch.Tensor,
35
+ ) -> None:
36
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
37
+ if y_k.shape != (m, 256) or qjl_bf.shape != (m, 256) or y_v.shape != (m, 256):
38
+ raise RuntimeError("outputs must have shape (M, 256)")
39
+ return None
40
+
41
+
42
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_mixed"))
43
+ def _unpack_packed_mixed_fake(
44
+ k_idx_packed: torch.Tensor,
45
+ k_qjl_packed: torch.Tensor,
46
+ v_idx_packed: torch.Tensor,
47
+ cb_k_mse: torch.Tensor,
48
+ cb_v: torch.Tensor,
49
+ b_k_mse: int,
50
+ b_v: int,
51
+ y_k: torch.Tensor,
52
+ qjl_f: torch.Tensor,
53
+ y_v: torch.Tensor,
54
+ ) -> None:
55
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
56
+ if y_k.shape != (m, 256) or qjl_f.shape != (m, 256) or y_v.shape != (m, 256):
57
+ raise RuntimeError("outputs must have shape (M, 256)")
58
+ return None
59
+
60
+
61
+ @torch.library.register_fake(add_op_namespace_prefix("combine_kv_bf16"))
62
+ def _combine_kv_bf16_fake(
63
+ k_mse: torch.Tensor,
64
+ k_qjl: torch.Tensor,
65
+ v_unit: torch.Tensor,
66
+ k_norm: torch.Tensor,
67
+ k_rnorm: torch.Tensor,
68
+ v_norm: torch.Tensor,
69
+ coef: float,
70
+ k_out: torch.Tensor,
71
+ v_out: torch.Tensor,
72
+ ) -> None:
73
+ if k_mse.dim() != 2 or k_mse.shape[1] != 256:
74
+ raise RuntimeError("k_mse must have shape (M, 256)")
75
+ m = k_mse.shape[0]
76
+ if k_qjl.shape != k_mse.shape or v_unit.shape != k_mse.shape:
77
+ raise RuntimeError("k_qjl and v_unit must have shape (M, 256)")
78
+ if k_norm.shape != (m,) or k_rnorm.shape != (m,) or v_norm.shape != (m,):
79
+ raise RuntimeError("norm tensors must have shape (M,)")
80
+ if k_out.shape != k_mse.shape or v_out.shape != k_mse.shape:
81
+ raise RuntimeError("outputs must have shape (M, 256)")
82
+ return None
83
+
84
+
85
+ def unpack_packed_bf16(
86
+ k_idx_packed: torch.Tensor,
87
+ k_qjl_packed: torch.Tensor,
88
+ v_idx_packed: torch.Tensor,
89
+ cb_k_mse: torch.Tensor,
90
+ cb_v: torch.Tensor,
91
+ b_k_mse: int,
92
+ b_v: int,
93
+ *,
94
+ y_k: Optional[torch.Tensor] = None,
95
+ qjl_bf: Optional[torch.Tensor] = None,
96
+ y_v: Optional[torch.Tensor] = None,
97
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
98
+ """Unpack TurboQuant packed indices to BF16 `(M, 256)` tensors."""
99
+
100
+ m = k_idx_packed.shape[0]
101
+ if y_k is None:
102
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
103
+ if qjl_bf is None:
104
+ qjl_bf = torch.empty_like(y_k)
105
+ if y_v is None:
106
+ y_v = torch.empty_like(y_k)
107
+ ops.unpack_packed_bf16(
108
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
109
+ int(b_k_mse), int(b_v), y_k, qjl_bf, y_v,
110
+ )
111
+ return y_k, qjl_bf, y_v
112
+
113
+
114
+ def unpack_packed_mixed(
115
+ k_idx_packed: torch.Tensor,
116
+ k_qjl_packed: torch.Tensor,
117
+ v_idx_packed: torch.Tensor,
118
+ cb_k_mse: torch.Tensor,
119
+ cb_v: torch.Tensor,
120
+ b_k_mse: int,
121
+ b_v: int,
122
+ *,
123
+ y_k: Optional[torch.Tensor] = None,
124
+ qjl_f: Optional[torch.Tensor] = None,
125
+ y_v: Optional[torch.Tensor] = None,
126
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
127
+ """Unpack packed K/V to BF16 K/V and FP32 QJL signs."""
128
+
129
+ m = k_idx_packed.shape[0]
130
+ if y_k is None:
131
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
132
+ if qjl_f is None:
133
+ qjl_f = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.float32)
134
+ if y_v is None:
135
+ y_v = torch.empty_like(y_k)
136
+ ops.unpack_packed_mixed(
137
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
138
+ int(b_k_mse), int(b_v), y_k, qjl_f, y_v,
139
+ )
140
+ return y_k, qjl_f, y_v
141
+
142
+
143
+ def combine_kv_bf16(
144
+ k_mse: torch.Tensor,
145
+ k_qjl: torch.Tensor,
146
+ v_unit: torch.Tensor,
147
+ k_norm: torch.Tensor,
148
+ k_rnorm: torch.Tensor,
149
+ v_norm: torch.Tensor,
150
+ coef: float,
151
+ *,
152
+ k_out: Optional[torch.Tensor] = None,
153
+ v_out: Optional[torch.Tensor] = None,
154
+ ) -> tuple[torch.Tensor, torch.Tensor]:
155
+ """Combine TurboQuant K/V GEMM outputs into BF16 K/V cache tensors."""
156
+
157
+ if k_out is None:
158
+ k_out = torch.empty_like(k_mse)
159
+ if v_out is None:
160
+ v_out = torch.empty_like(v_unit)
161
+ ops.combine_kv_bf16(k_mse, k_qjl, v_unit, k_norm, k_rnorm, v_norm, float(coef), k_out, v_out)
162
+ return k_out, v_out
163
+
164
+
165
+ __all__ = [
166
+ "combine_kv_bf16",
167
+ "unpack_packed_bf16",
168
+ "unpack_packed_mixed",
169
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _turboquant_kv_cuda_5596053
3
+ ops = torch.ops._turboquant_kv_cuda_5596053
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_turboquant_kv_cuda_5596053::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/_turboquant_kv_cuda_5596053.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3335ab5b27253d8bcdcde9da4f88531808282c9aa6cef7b41036481e8ddfe4e4
3
+ size 648568
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "turboquant-kv",
3
+ "id": "_turboquant_kv_cuda_5596053",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu130-x86_64-linux/turboquant_kv/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT TurboQuant KV cache kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+
12
+ def _check_packed(k_idx: torch.Tensor, k_qjl: torch.Tensor, v_idx: torch.Tensor) -> int:
13
+ if k_idx.dim() != 2 or k_idx.shape[1] != 128:
14
+ raise RuntimeError("k_idx_packed must have shape (M, 128)")
15
+ m = k_idx.shape[0]
16
+ if k_qjl.shape != (m, 32):
17
+ raise RuntimeError("k_qjl_packed must have shape (M, 32)")
18
+ if v_idx.shape != (m, 128):
19
+ raise RuntimeError("v_idx_packed must have shape (M, 128)")
20
+ return m
21
+
22
+
23
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_bf16"))
24
+ def _unpack_packed_bf16_fake(
25
+ k_idx_packed: torch.Tensor,
26
+ k_qjl_packed: torch.Tensor,
27
+ v_idx_packed: torch.Tensor,
28
+ cb_k_mse: torch.Tensor,
29
+ cb_v: torch.Tensor,
30
+ b_k_mse: int,
31
+ b_v: int,
32
+ y_k: torch.Tensor,
33
+ qjl_bf: torch.Tensor,
34
+ y_v: torch.Tensor,
35
+ ) -> None:
36
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
37
+ if y_k.shape != (m, 256) or qjl_bf.shape != (m, 256) or y_v.shape != (m, 256):
38
+ raise RuntimeError("outputs must have shape (M, 256)")
39
+ return None
40
+
41
+
42
+ @torch.library.register_fake(add_op_namespace_prefix("unpack_packed_mixed"))
43
+ def _unpack_packed_mixed_fake(
44
+ k_idx_packed: torch.Tensor,
45
+ k_qjl_packed: torch.Tensor,
46
+ v_idx_packed: torch.Tensor,
47
+ cb_k_mse: torch.Tensor,
48
+ cb_v: torch.Tensor,
49
+ b_k_mse: int,
50
+ b_v: int,
51
+ y_k: torch.Tensor,
52
+ qjl_f: torch.Tensor,
53
+ y_v: torch.Tensor,
54
+ ) -> None:
55
+ m = _check_packed(k_idx_packed, k_qjl_packed, v_idx_packed)
56
+ if y_k.shape != (m, 256) or qjl_f.shape != (m, 256) or y_v.shape != (m, 256):
57
+ raise RuntimeError("outputs must have shape (M, 256)")
58
+ return None
59
+
60
+
61
+ @torch.library.register_fake(add_op_namespace_prefix("combine_kv_bf16"))
62
+ def _combine_kv_bf16_fake(
63
+ k_mse: torch.Tensor,
64
+ k_qjl: torch.Tensor,
65
+ v_unit: torch.Tensor,
66
+ k_norm: torch.Tensor,
67
+ k_rnorm: torch.Tensor,
68
+ v_norm: torch.Tensor,
69
+ coef: float,
70
+ k_out: torch.Tensor,
71
+ v_out: torch.Tensor,
72
+ ) -> None:
73
+ if k_mse.dim() != 2 or k_mse.shape[1] != 256:
74
+ raise RuntimeError("k_mse must have shape (M, 256)")
75
+ m = k_mse.shape[0]
76
+ if k_qjl.shape != k_mse.shape or v_unit.shape != k_mse.shape:
77
+ raise RuntimeError("k_qjl and v_unit must have shape (M, 256)")
78
+ if k_norm.shape != (m,) or k_rnorm.shape != (m,) or v_norm.shape != (m,):
79
+ raise RuntimeError("norm tensors must have shape (M,)")
80
+ if k_out.shape != k_mse.shape or v_out.shape != k_mse.shape:
81
+ raise RuntimeError("outputs must have shape (M, 256)")
82
+ return None
83
+
84
+
85
+ def unpack_packed_bf16(
86
+ k_idx_packed: torch.Tensor,
87
+ k_qjl_packed: torch.Tensor,
88
+ v_idx_packed: torch.Tensor,
89
+ cb_k_mse: torch.Tensor,
90
+ cb_v: torch.Tensor,
91
+ b_k_mse: int,
92
+ b_v: int,
93
+ *,
94
+ y_k: Optional[torch.Tensor] = None,
95
+ qjl_bf: Optional[torch.Tensor] = None,
96
+ y_v: Optional[torch.Tensor] = None,
97
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
98
+ """Unpack TurboQuant packed indices to BF16 `(M, 256)` tensors."""
99
+
100
+ m = k_idx_packed.shape[0]
101
+ if y_k is None:
102
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
103
+ if qjl_bf is None:
104
+ qjl_bf = torch.empty_like(y_k)
105
+ if y_v is None:
106
+ y_v = torch.empty_like(y_k)
107
+ ops.unpack_packed_bf16(
108
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
109
+ int(b_k_mse), int(b_v), y_k, qjl_bf, y_v,
110
+ )
111
+ return y_k, qjl_bf, y_v
112
+
113
+
114
+ def unpack_packed_mixed(
115
+ k_idx_packed: torch.Tensor,
116
+ k_qjl_packed: torch.Tensor,
117
+ v_idx_packed: torch.Tensor,
118
+ cb_k_mse: torch.Tensor,
119
+ cb_v: torch.Tensor,
120
+ b_k_mse: int,
121
+ b_v: int,
122
+ *,
123
+ y_k: Optional[torch.Tensor] = None,
124
+ qjl_f: Optional[torch.Tensor] = None,
125
+ y_v: Optional[torch.Tensor] = None,
126
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
127
+ """Unpack packed K/V to BF16 K/V and FP32 QJL signs."""
128
+
129
+ m = k_idx_packed.shape[0]
130
+ if y_k is None:
131
+ y_k = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.bfloat16)
132
+ if qjl_f is None:
133
+ qjl_f = torch.empty((m, 256), device=k_idx_packed.device, dtype=torch.float32)
134
+ if y_v is None:
135
+ y_v = torch.empty_like(y_k)
136
+ ops.unpack_packed_mixed(
137
+ k_idx_packed, k_qjl_packed, v_idx_packed, cb_k_mse, cb_v,
138
+ int(b_k_mse), int(b_v), y_k, qjl_f, y_v,
139
+ )
140
+ return y_k, qjl_f, y_v
141
+
142
+
143
+ def combine_kv_bf16(
144
+ k_mse: torch.Tensor,
145
+ k_qjl: torch.Tensor,
146
+ v_unit: torch.Tensor,
147
+ k_norm: torch.Tensor,
148
+ k_rnorm: torch.Tensor,
149
+ v_norm: torch.Tensor,
150
+ coef: float,
151
+ *,
152
+ k_out: Optional[torch.Tensor] = None,
153
+ v_out: Optional[torch.Tensor] = None,
154
+ ) -> tuple[torch.Tensor, torch.Tensor]:
155
+ """Combine TurboQuant K/V GEMM outputs into BF16 K/V cache tensors."""
156
+
157
+ if k_out is None:
158
+ k_out = torch.empty_like(k_mse)
159
+ if v_out is None:
160
+ v_out = torch.empty_like(v_unit)
161
+ ops.combine_kv_bf16(k_mse, k_qjl, v_unit, k_norm, k_rnorm, v_norm, float(coef), k_out, v_out)
162
+ return k_out, v_out
163
+
164
+
165
+ __all__ = [
166
+ "combine_kv_bf16",
167
+ "unpack_packed_bf16",
168
+ "unpack_packed_mixed",
169
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _turboquant_kv_cuda_5596053
3
+ ops = torch.ops._turboquant_kv_cuda_5596053
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_turboquant_kv_cuda_5596053::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/_turboquant_kv_cuda_5596053.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e79461b0097df9d343ebe54cd876cbe59ac242fc50575f2d52a13bd257fdb49
3
+ size 644472
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "turboquant-kv",
3
+ "id": "_turboquant_kv_cuda_5596053",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/turboquant_kv/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))