CompressedGemma commited on
Commit
d3c3c83
·
verified ·
1 Parent(s): b5315cd

Upload patch_llamacpp_hpcq_seg.py

Browse files
Files changed (1) hide show
  1. patch_llamacpp_hpcq_seg.py +872 -0
patch_llamacpp_hpcq_seg.py ADDED
@@ -0,0 +1,872 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Patch llama.cpp so HPCQ.SEG.V1 band-major Q2_K/Q3_K files decode exactly.
3
+
4
+ What this does
5
+ --------------
6
+ HPC-Quantize can emit tensors whose 256-weight superblocks are stored in
7
+ "band-major" slot order (each 16-slot sub-block holds one value band). The
8
+ per-position inverse permutation is stored in a GGUF key-value array:
9
+
10
+ HPCQ.SEG.V1.<tensor> : uint8[] (HQS1 codec, one band id per original
11
+ position, nibble-packed for k <= 16)
12
+
13
+ Stock llama.cpp decodes Q2_K/Q3_K positionally, so band-major payloads are
14
+ garbage without help. This patch makes the ggml dequant paths consult the
15
+ label channel and scatter each block back to original positions:
16
+
17
+ * CPU : the reference ``dequantize_row_q2_K/q3_K`` scatter their writes
18
+ (correct for get_rows / dequantizing ops). Weight matmuls on CPU fall
19
+ back to F16 at load (``--cpu-f16``, default on) because CPU vec_dot is
20
+ per-arch SIMD code.
21
+ * CUDA: the device dequant + mul_mat-vec paths scatter via a device-side
22
+ registry keyed by payload address range, so no kernel signatures or
23
+ launchers change. Large-batch GEMM forcibly uses the (patched)
24
+ convert-to-F16 path for segmented tensors.
25
+
26
+ Usage
27
+ -----
28
+ python3 patch_llamacpp_hpcq_seg.py --llama /path/to/llama.cpp --check
29
+ python3 patch_llamacpp_hpcq_seg.py --llama /path/to/llama.cpp --apply
30
+ python3 patch_llamacpp_hpcq_seg.py --llama /path/to/llama.cpp --revert
31
+ python3 patch_llamacpp_hpcq_seg.py --selftest
32
+
33
+ The patch is idempotent (marker based) and leaves .hpcq-bak backups.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import argparse
39
+ import shutil
40
+ import subprocess
41
+ import sys
42
+ import tempfile
43
+ from pathlib import Path
44
+
45
+ BEGIN = "HPCQ_SEG_PATCH_BEGIN"
46
+ END = "HPCQ_SEG_PATCH_END"
47
+ HEADER_NAME = "hpcq-seg.h"
48
+
49
+ # --------------------------------------------------------------------------
50
+ # 1. New header: shared host/device helpers (no external includes needed)
51
+ # --------------------------------------------------------------------------
52
+
53
+ HEADER = r"""// hpcq-seg.h — HPCQ.SEG.V1 band-major scatter support for Q2_K / Q3_K.
54
+ // Generated by HPC-Quantize tools/patch_llamacpp_hpcq_seg.py. Do not edit.
55
+ #pragma once
56
+
57
+ #include <cstdint>
58
+ #include <cstring>
59
+
60
+ #define HPCQ_SEG_PREFIX "HPCQ.SEG.V1."
61
+ #define HPCQ_SEG_MAX_ENTRIES 1024
62
+
63
+ // ---------------------------------------------------------------------------
64
+ // label channel codec: "HQS1" | u8 k | u8 enc | u32 n | payload
65
+ // enc 0: two band ids per byte (low nibble = even index)
66
+ // enc 1: raw u8 band ids
67
+ // ---------------------------------------------------------------------------
68
+ static inline bool hpcq_seg_decode(const uint8_t * c, size_t n_bytes,
69
+ uint8_t * labels, size_t n_labels) {
70
+ if (n_bytes < 10 || c[0] != 'H' || c[1] != 'Q' || c[2] != 'S' || c[3] != '1') {
71
+ return false;
72
+ }
73
+ const uint8_t enc = c[5];
74
+ const uint32_t n = (uint32_t) c[6] | ((uint32_t) c[7] << 8) |
75
+ ((uint32_t) c[8] << 16) | ((uint32_t) c[9] << 24);
76
+ if ((size_t) n != n_labels) {
77
+ return false;
78
+ }
79
+ const uint8_t * d = c + 10;
80
+ if (enc == 0) {
81
+ if (n_bytes < 10 + (size_t) ((n + 1) / 2)) return false;
82
+ for (size_t i = 0; i < n_labels; ++i) {
83
+ labels[i] = (d[i >> 1] >> ((i & 1) * 4)) & 0x0F;
84
+ }
85
+ return true;
86
+ }
87
+ if (enc == 1) {
88
+ if (n_bytes < 10 + (size_t) n) return false;
89
+ memcpy(labels, d, n);
90
+ return true;
91
+ }
92
+ return false;
93
+ }
94
+
95
+ // ---------------------------------------------------------------------------
96
+ // fp32 -> fp16 (round-to-nearest-even), self-contained to avoid extra deps
97
+ // ---------------------------------------------------------------------------
98
+ static inline uint16_t hpcq_f32_to_f16_bits(float f) {
99
+ uint32_t x;
100
+ memcpy(&x, &f, 4);
101
+ const uint32_t sign = (x >> 16) & 0x8000u;
102
+ int32_t exp = (int32_t) ((x >> 23) & 0xFF);
103
+ uint32_t mant = x & 0x7FFFFFu;
104
+ if (exp == 0xFF) {
105
+ return (uint16_t) (sign | 0x7C00u | (mant ? 0x200u : 0u));
106
+ }
107
+ int32_t e = exp - 127 + 15;
108
+ if (e >= 0x1F) {
109
+ return (uint16_t) (sign | 0x7C00u);
110
+ }
111
+ if (e <= 0) {
112
+ if (e < -10) {
113
+ return (uint16_t) sign;
114
+ }
115
+ mant |= 0x800000u;
116
+ const int32_t shift = 14 - e;
117
+ uint32_t half = mant >> shift;
118
+ const uint32_t rem = mant & ((1u << shift) - 1u);
119
+ const uint32_t halfway = 1u << (shift - 1);
120
+ if (rem > halfway || (rem == halfway && (half & 1u))) half++;
121
+ return (uint16_t) (sign | half);
122
+ }
123
+ uint32_t half = ((uint32_t) e << 10) | (mant >> 13);
124
+ const uint32_t rem = mant & 0x1FFFu;
125
+ if (rem > 0x1000u || (rem == 0x1000u && (half & 1u))) half++;
126
+ return (uint16_t) (sign | half);
127
+ }
128
+
129
+ // ---------------------------------------------------------------------------
130
+ // block dequantizers (band-major slot values) and scatter-expand to fp16
131
+ // ---------------------------------------------------------------------------
132
+ static inline void hpcq_seg_deq_q2k(const uint8_t * blk, float * out) {
133
+ const uint8_t * sc = blk; // scales[16]
134
+ const uint8_t * qs = blk + 16; // qs[64]
135
+ uint16_t d_bits, m_bits;
136
+ memcpy(&d_bits, blk + 80, 2);
137
+ memcpy(&m_bits, blk + 82, 2);
138
+ float d, m;
139
+ // reuse f16->f32 via bit math
140
+ {
141
+ uint32_t s = (uint32_t) (d_bits & 0x8000u) << 16;
142
+ int32_t e = (d_bits >> 10) & 0x1F;
143
+ uint32_t mt = d_bits & 0x3FFu;
144
+ uint32_t bits;
145
+ if (e == 0) { bits = s; }
146
+ else if (e == 31) { bits = s | 0x7F800000u | (mt << 13); }
147
+ else { bits = s | ((uint32_t) (e - 15 + 127) << 23) | (mt << 13); }
148
+ memcpy(&d, &bits, 4);
149
+ }
150
+ {
151
+ uint32_t s = (uint32_t) (m_bits & 0x8000u) << 16;
152
+ int32_t e = (m_bits >> 10) & 0x1F;
153
+ uint32_t mt = m_bits & 0x3FFu;
154
+ uint32_t bits;
155
+ if (e == 0) { bits = s; }
156
+ else if (e == 31) { bits = s | 0x7F800000u | (mt << 13); }
157
+ else { bits = s | ((uint32_t) (e - 15 + 127) << 23) | (mt << 13); }
158
+ memcpy(&m, &bits, 4);
159
+ }
160
+ int is = 0;
161
+ int base = 0;
162
+ for (int n = 0; n < 256; n += 128) {
163
+ int shift = 0;
164
+ for (int j = 0; j < 4; ++j) {
165
+ uint8_t s0 = sc[is++];
166
+ const float dl = d * (float) (s0 & 0xF), ml = m * (float) (s0 >> 4);
167
+ for (int l = 0; l < 16; ++l) out[base + l] = dl * (float) ((qs[l] >> shift) & 3) - ml;
168
+ base += 16;
169
+ uint8_t s1 = sc[is++];
170
+ const float dl1 = d * (float) (s1 & 0xF), ml1 = m * (float) (s1 >> 4);
171
+ for (int l = 0; l < 16; ++l) out[base + l] = dl1 * (float) ((qs[l + 16] >> shift) & 3) - ml1;
172
+ base += 16;
173
+ shift += 2;
174
+ }
175
+ qs += 32;
176
+ }
177
+ }
178
+
179
+ static inline void hpcq_seg_deq_q3k(const uint8_t * blk, float * out) {
180
+ const uint8_t * hm = blk; // hmask[32]
181
+ const uint8_t * qs = blk + 32; // qs[64]
182
+ const uint8_t * sw = blk + 96; // scales[12]
183
+ uint16_t d_bits;
184
+ memcpy(&d_bits, blk + 108, 2);
185
+ float d;
186
+ {
187
+ uint32_t s = (uint32_t) (d_bits & 0x8000u) << 16;
188
+ int32_t e = (d_bits >> 10) & 0x1F;
189
+ uint32_t mt = d_bits & 0x3FFu;
190
+ uint32_t bits;
191
+ if (e == 0) { bits = s; }
192
+ else if (e == 31) { bits = s | 0x7F800000u | (mt << 13); }
193
+ else { bits = s | ((uint32_t) (e - 15 + 127) << 23) | (mt << 13); }
194
+ memcpy(&d, &bits, 4);
195
+ }
196
+ uint8_t v[16];
197
+ for (int i = 0; i < 4; ++i) {
198
+ const uint8_t a0 = sw[i], a1 = sw[4 + i], t = sw[8 + i];
199
+ v[i] = (uint8_t) ((a0 & 0x0F) | (((t >> 0) & 0x03) << 4));
200
+ v[4 + i] = (uint8_t) ((a1 & 0x0F) | (((t >> 2) & 0x03) << 4));
201
+ v[8 + i] = (uint8_t) (((a0 >> 4) & 0x0F) | (((t >> 4) & 0x03) << 4));
202
+ v[12 + i] = (uint8_t) (((a1 >> 4) & 0x0F) | (((t >> 6) & 0x03) << 4));
203
+ }
204
+ uint8_t m = 1;
205
+ int is = 0;
206
+ int base = 0;
207
+ for (int n = 0; n < 256; n += 128) {
208
+ int shift = 0;
209
+ for (int j = 0; j < 4; ++j) {
210
+ const float dl = d * (float) ((int) v[is++] - 32);
211
+ for (int l = 0; l < 16; ++l)
212
+ out[base + l] = dl * (float) (((int8_t) ((qs[l] >> shift) & 3)) - ((hm[l] & m) ? 0 : 4));
213
+ base += 16;
214
+ const float dl1 = d * (float) ((int) v[is++] - 32);
215
+ for (int l = 0; l < 16; ++l)
216
+ out[base + l] = dl1 * (float) (((int8_t) ((qs[l + 16] >> shift) & 3)) - ((hm[l + 16] & m) ? 0 : 4));
217
+ base += 16;
218
+ shift += 2;
219
+ m <<= 1;
220
+ }
221
+ qs += 32;
222
+ }
223
+ }
224
+
225
+ // dequantize a band-major payload and scatter to original positions (fp16)
226
+ // returns false for unsupported ggml type ids
227
+ static inline bool hpcq_seg_expand_f16(int ftype, const uint8_t * raw, uint16_t * out,
228
+ int64_t n_elements, const uint8_t * labels) {
229
+ if (n_elements % 256 != 0) return false;
230
+ const int64_t n_blocks = n_elements / 256;
231
+ if (ftype != 10 && ftype != 11) return false;
232
+ const int64_t bs = ftype == 10 ? 84 : 110;
233
+ float vals[256];
234
+ uint16_t inv[256];
235
+ for (int64_t b = 0; b < n_blocks; ++b) {
236
+ const uint8_t * blk = raw + b * bs;
237
+ if (ftype == 10) hpcq_seg_deq_q2k(blk, vals);
238
+ else hpcq_seg_deq_q3k(blk, vals);
239
+ hpcq_seg_make_inv(labels + b * 256, inv);
240
+ uint16_t * o = out + b * 256;
241
+ for (int s = 0; s < 256; ++s) o[inv[s]] = hpcq_f32_to_f16_bits(vals[s]);
242
+ }
243
+ return true;
244
+ }
245
+
246
+ // stable counting sort per 256-block: inv[pos] = slot whose data belongs at pos
247
+ static inline void hpcq_seg_make_inv(const uint8_t * lab, uint16_t * inv) {
248
+ uint16_t start[16];
249
+ uint16_t acc = 0;
250
+ for (int b = 0; b < 16; ++b) start[b] = 0;
251
+ for (int p = 0; p < 256; ++p) start[lab[p] & 0x0F]++;
252
+ for (int b = 0; b < 16; ++b) { const uint16_t c = start[b]; start[b] = acc; acc = (uint16_t) (acc + c); }
253
+ for (int p = 0; p < 256; ++p) inv[p] = start[lab[p] & 0x0F]++;
254
+ }
255
+
256
+ // ---------------------------------------------------------------------------
257
+ // host-side registry: payload base pointer range -> order table
258
+ // order table layout: per 256-block, 256 uint16 slot indices
259
+ // ---------------------------------------------------------------------------
260
+ struct hpcq_seg_entry {
261
+ uintptr_t base; // payload start
262
+ uintptr_t end; // payload end (exclusive)
263
+ int64_t block_bytes;
264
+ const uint16_t * order; // host pointer, 256 per block
265
+ };
266
+
267
+ static int hpcq_seg_count(void) {
268
+ extern int hpcq_seg_g_count;
269
+ return hpcq_seg_g_count;
270
+ }
271
+
272
+ static const struct hpcq_seg_entry * hpcq_seg_entries(void) {
273
+ extern struct hpcq_seg_entry hpcq_seg_g_entries[HPCQ_SEG_MAX_ENTRIES];
274
+ return hpcq_seg_g_entries;
275
+ }
276
+
277
+ static inline const uint16_t * hpcq_seg_find_orders(const void * ptr, int64_t block_bytes) {
278
+ const uintptr_t p = (uintptr_t) ptr;
279
+ const int n = hpcq_seg_count();
280
+ for (int i = 0; i < n; ++i) {
281
+ const struct hpcq_seg_entry & e = hpcq_seg_entries()[i];
282
+ if (p >= e.base && p < e.end && e.block_bytes == block_bytes) {
283
+ return e.order + ((p - e.base) / (uintptr_t) block_bytes) * 256;
284
+ }
285
+ }
286
+ return nullptr;
287
+ }
288
+ """
289
+
290
+ # --------------------------------------------------------------------------
291
+ # 2. ggml.c: host registry + public bind API
292
+ # --------------------------------------------------------------------------
293
+
294
+ GGML_C_INCLUDE = f"""
295
+ // {BEGIN}: hpcq seg header + registry
296
+ #include "hpcq-seg.h"
297
+
298
+ int hpcq_seg_g_count = 0;
299
+ struct hpcq_seg_entry hpcq_seg_g_entries[HPCQ_SEG_MAX_ENTRIES];
300
+ // {END}
301
+ """
302
+
303
+ GGML_C_API = """
304
+ // HPCQ_SEG_PATCH_BEGIN: bind decoded label channel to a payload range
305
+ void ggml_hpcq_seg_bind(void * payload, int64_t n_bytes, const uint8_t * labels,
306
+ int64_t n_labels, int64_t block_bytes) {
307
+ if (payload == NULL || labels == NULL || n_labels <= 0 || block_bytes <= 0) return;
308
+ if (n_labels % 256 != 0) return;
309
+ const int64_t n_blocks = n_labels / 256;
310
+ uint16_t * order = (uint16_t *) malloc((size_t) n_labels * sizeof(uint16_t));
311
+ uint8_t * lab = (uint8_t *) malloc(256);
312
+ if (order == NULL || lab == NULL) { free(order); free(lab); return; }
313
+ for (int64_t b = 0; b < n_blocks; ++b) {
314
+ memcpy(lab, labels + b * 256, 256);
315
+ hpcq_seg_make_inv(lab, order + b * 256);
316
+ }
317
+ free(lab);
318
+ if (hpcq_seg_g_count >= HPCQ_SEG_MAX_ENTRIES) { free(order); return; }
319
+ struct hpcq_seg_entry * e = &hpcq_seg_g_entries[hpcq_seg_g_count++];
320
+ e->base = (uintptr_t) payload;
321
+ e->end = (uintptr_t) payload + (uintptr_t) n_bytes;
322
+ e->block_bytes = block_bytes;
323
+ e->order = order;
324
+ }
325
+ // HPCQ_SEG_PATCH_END
326
+ """
327
+
328
+ # --------------------------------------------------------------------------
329
+ # 3. ggml-quants.c: scatter host dequantize_row_q2_K / q3_K
330
+ # --------------------------------------------------------------------------
331
+
332
+ Q2K_SCALE = """
333
+ // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
334
+ const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q2_K));
335
+ #define HPCQ_Q2K_WRITE(expr, idx) do { if (hpcq_ord) y_out[hpcq_ord[(idx)]] = (expr); else y_out[idx] = (expr); } while (0)
336
+ // HPCQ_SEG_PATCH_END
337
+ """
338
+
339
+ Q3K_SCALE = """
340
+ // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
341
+ const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q3_K));
342
+ #define HPCQ_Q3K_WRITE(expr, idx) do { if (hpcq_ord) y_out[hpcq_ord[(idx)]] = (expr); else y_out[idx] = (expr); } while (0)
343
+ // HPCQ_SEG_PATCH_END
344
+ """
345
+
346
+ Q2K_DEQ_OLD = """ const int nb = k / QK_K;
347
+
348
+ for (int i = 0; i < nb; i++) {
349
+
350
+ const float d = GGML_FP16_TO_FP32(x[i].d);
351
+ const float min = GGML_FP16_TO_FP32(x[i].dmin);
352
+
353
+ const uint8_t * q = x[i].qs;
354
+
355
+ int is = 0;
356
+ float dl, ml;
357
+ for (int n = 0; n < QK_K; n += 128) {
358
+ int shift = 0;
359
+ for (int j = 0; j < 4; ++j) {
360
+
361
+ uint8_t sc = x[i].scales[is++];
362
+ dl = d * (sc & 0xF); ml = min * (sc >> 4);
363
+ for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;
364
+
365
+ sc = x[i].scales[is++];
366
+ dl = d * (sc & 0xF); ml = min * (sc >> 4);
367
+ for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;
368
+
369
+ shift += 2;
370
+ }
371
+ q += 32;
372
+ }
373
+ }
374
+ }"""
375
+
376
+ Q2K_DEQ_NEW = """ const int nb = k / QK_K;
377
+
378
+ // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
379
+ const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q2_K));
380
+ // HPCQ_SEG_PATCH_END
381
+
382
+ for (int i = 0; i < nb; i++) {
383
+
384
+ const uint16_t * ord = hpcq_ord ? hpcq_ord + i * QK_K : NULL;
385
+ float * y_out = y + i * QK_K;
386
+
387
+ const float d = GGML_FP16_TO_FP32(x[i].d);
388
+ const float min = GGML_FP16_TO_FP32(x[i].dmin);
389
+
390
+ const uint8_t * q = x[i].qs;
391
+
392
+ int is = 0;
393
+ int base = 0;
394
+ float dl, ml;
395
+ for (int n = 0; n < QK_K; n += 128) {
396
+ int shift = 0;
397
+ for (int j = 0; j < 4; ++j) {
398
+
399
+ uint8_t sc = x[i].scales[is++];
400
+ dl = d * (sc & 0xF); ml = min * (sc >> 4);
401
+ for (int l = 0; l < 16; ++l) {
402
+ const float v = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;
403
+ if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
404
+ }
405
+ base += 16;
406
+
407
+ sc = x[i].scales[is++];
408
+ dl = d * (sc & 0xF); ml = min * (sc >> 4);
409
+ for (int l = 0; l < 16; ++l) {
410
+ const float v = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;
411
+ if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
412
+ }
413
+ base += 16;
414
+
415
+ shift += 2;
416
+ }
417
+ q += 32;
418
+ }
419
+ }
420
+ }"""
421
+
422
+ Q3K_DEQ_OLD = """ for (int i = 0; i < nb; i++) {
423
+
424
+ const float d_all = GGML_FP16_TO_FP32(x[i].d);
425
+
426
+ const uint8_t * GGML_RESTRICT q = x[i].qs;
427
+ const uint8_t * GGML_RESTRICT hm = x[i].hmask;
428
+ uint8_t m = 1;
429
+
430
+ memcpy(aux, x[i].scales, 12);
431
+ uint32_t tmp = aux[2];
432
+ aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
433
+ aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
434
+ aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
435
+ aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
436
+
437
+ int is = 0;
438
+ float dl;
439
+ for (int n = 0; n < QK_K; n += 128) {
440
+ int shift = 0;
441
+ for (int j = 0; j < 4; ++j) {
442
+
443
+ dl = d_all * (scales[is++] - 32);
444
+ for (int l = 0; l < 16; ++l) {
445
+ *y++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));
446
+ }
447
+
448
+ dl = d_all * (scales[is++] - 32);
449
+ for (int l = 0; l < 16; ++l) {
450
+ *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));
451
+ }
452
+
453
+ shift += 2;
454
+ m <<= 1;
455
+ }
456
+ q += 32;
457
+ }
458
+
459
+ }
460
+ }"""
461
+
462
+ Q3K_DEQ_NEW = """ // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
463
+ const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q3_K));
464
+ // HPCQ_SEG_PATCH_END
465
+
466
+ for (int i = 0; i < nb; i++) {
467
+
468
+ const uint16_t * ord = hpcq_ord ? hpcq_ord + i * QK_K : NULL;
469
+ float * y_out = y + i * QK_K;
470
+
471
+ const float d_all = GGML_FP16_TO_FP32(x[i].d);
472
+
473
+ const uint8_t * GGML_RESTRICT q = x[i].qs;
474
+ const uint8_t * GGML_RESTRICT hm = x[i].hmask;
475
+ uint8_t m = 1;
476
+
477
+ memcpy(aux, x[i].scales, 12);
478
+ uint32_t tmp = aux[2];
479
+ aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
480
+ aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
481
+ aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
482
+ aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
483
+
484
+ int is = 0;
485
+ int base = 0;
486
+ float dl;
487
+ for (int n = 0; n < QK_K; n += 128) {
488
+ int shift = 0;
489
+ for (int j = 0; j < 4; ++j) {
490
+
491
+ dl = d_all * (scales[is++] - 32);
492
+ for (int l = 0; l < 16; ++l) {
493
+ const float v = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));
494
+ if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
495
+ }
496
+ base += 16;
497
+
498
+ dl = d_all * (scales[is++] - 32);
499
+ for (int l = 0; l < 16; ++l) {
500
+ const float v = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));
501
+ if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
502
+ }
503
+ base += 16;
504
+
505
+ shift += 2;
506
+ m <<= 1;
507
+ }
508
+ q += 32;
509
+ }
510
+
511
+ }
512
+ }"""
513
+
514
+
515
+ def _read(p: Path) -> str:
516
+ return p.read_text(encoding="utf-8")
517
+
518
+
519
+ def _write(p: Path, s: str) -> None:
520
+ p.write_text(s, encoding="utf-8")
521
+
522
+
523
+ def _backup(p: Path) -> None:
524
+ bak = p.with_suffix(p.suffix + ".hpcq-bak")
525
+ if not bak.exists():
526
+ shutil.copy2(p, bak)
527
+
528
+
529
+ def _require_once(text: str, needle: str, what: str) -> None:
530
+ n = text.count(needle)
531
+ if n != 1:
532
+ raise SystemExit(f"anchor for {what}: found {n} occurrence(s), expected 1")
533
+
534
+
535
+ def _insert_after(text: str, anchor: str, block: str, what: str) -> str:
536
+ _require_once(text, anchor, what)
537
+ i = text.index(anchor) + len(anchor)
538
+ return text[:i] + block + text[i:]
539
+
540
+
541
+ def _insert_before(text: str, anchor: str, block: str, what: str) -> str:
542
+ _require_once(text, anchor, what)
543
+ i = text.index(anchor)
544
+ return text[:i] + block + text[i:]
545
+
546
+
547
+ def _replace_block(text: str, old: str, new: str, what: str) -> str:
548
+ _require_once(text, old, what)
549
+ return text.replace(old, new, 1)
550
+
551
+
552
+ # --------------------------------------------------------------------------
553
+ # Patch steps
554
+ # --------------------------------------------------------------------------
555
+
556
+ def step_header(root: Path) -> None:
557
+ _write(root / "ggml" / "src" / HEADER_NAME, HEADER)
558
+
559
+
560
+ def step_ggml_c(root: Path) -> None:
561
+ p = root / "ggml" / "src" / "ggml.c"
562
+ s = _read(p)
563
+ if BEGIN in s:
564
+ return
565
+ _backup(p)
566
+ anchor = '#include "ggml-impl.h"'
567
+ _require_once(s, anchor, "ggml.c include")
568
+ i = s.index(anchor) + len(anchor)
569
+ s = s[:i] + "\n" + GGML_C_INCLUDE + s[i:]
570
+ if "void ggml_hpcq_seg_bind" not in s:
571
+ marker = "////////////////////////////////////////////////////////////////////////////////"
572
+ i = s.rindex(marker)
573
+ s = s[:i] + GGML_C_API + "\n\n" + s[i:]
574
+ _write(p, s)
575
+
576
+
577
+ def step_ggml_quants(root: Path) -> None:
578
+ p = root / "ggml" / "src" / "ggml-quants.c"
579
+ s = _read(p)
580
+ if BEGIN in s:
581
+ return
582
+ _backup(p)
583
+ anchor = '#include "ggml-quants.h"'
584
+ _require_once(s, anchor, "ggml-quants.c include")
585
+ i = s.index(anchor) + len(anchor)
586
+ s = s[:i] + "\n#include \"hpcq-seg.h\"\n" + s[i:]
587
+ s = _replace_block(s, Q2K_DEQ_OLD, Q2K_DEQ_NEW, "dequantize_row_q2_K")
588
+ s = _replace_block(s, Q3K_DEQ_OLD, Q3K_DEQ_NEW, "dequantize_row_q3_K")
589
+ _write(p, s)
590
+
591
+
592
+ def step_loader(root: Path) -> None:
593
+ cpp = root / "src" / "llama-model-loader.cpp"
594
+ hdr = root / "src" / "llama-model-loader.h"
595
+ s = _read(cpp)
596
+ if BEGIN in s:
597
+ return
598
+ _backup(cpp)
599
+ _backup(hdr)
600
+ h = _read(hdr)
601
+
602
+ anchor = " std::unordered_map<std::string, llama_model_kv_override> kv_overrides;"
603
+ h = _insert_after(
604
+ h, anchor,
605
+ "\n // HPCQ_SEG_PATCH_BEGIN: decoded band-major label channels\n"
606
+ " std::unordered_map<std::string, std::vector<uint8_t>> hpcq_seg_labels;\n"
607
+ " // HPCQ_SEG_PATCH_END",
608
+ "loader header kv_overrides",
609
+ )
610
+ _write(hdr, h)
611
+
612
+ anchor = '#include "llama-model-loader.h"'
613
+ s = _insert_after(s, anchor, "\n#include \"hpcq-seg.h\"", "loader include")
614
+
615
+ # decode the label channels in the constructor (metadata is ready there)
616
+ anchor = " n_kv = gguf_get_n_kv(metadata);\n n_tensors = weights_map.size();"
617
+ _require_once(s, anchor, "loader ctor KV init")
618
+ s = s.replace(
619
+ anchor,
620
+ anchor + "\n\n // HPCQ_SEG_PATCH_BEGIN: decode HPCQ.SEG.V1 label channels\n"
621
+ " {\n"
622
+ " const int64_t n_kv_all = gguf_get_n_kv(metadata);\n"
623
+ " const size_t plen = sizeof(\"HPCQ.SEG.V1.\") - 1;\n"
624
+ " for (int64_t ki = 0; ki < n_kv_all; ++ki) {\n"
625
+ " const char * key = gguf_get_key(metadata, ki);\n"
626
+ " if (strncmp(key, \"HPCQ.SEG.V1.\", plen) != 0) continue;\n"
627
+ " if (gguf_get_kv_type(metadata, ki) != GGUF_TYPE_ARRAY) continue;\n"
628
+ " if (gguf_get_arr_type(metadata, ki) != GGUF_TYPE_UINT8) continue;\n"
629
+ " const std::string tname(key + plen);\n"
630
+ " const auto wt = weights_map.find(tname);\n"
631
+ " if (wt == weights_map.end()) continue;\n"
632
+ " const int64_t ne = ggml_nelements(wt->second.tensor);\n"
633
+ " std::vector<uint8_t> labels((size_t) ne);\n"
634
+ " if (!hpcq_seg_decode((const uint8_t *) gguf_get_arr_data(metadata, ki),\n"
635
+ " gguf_get_arr_n(metadata, ki), labels.data(), labels.size())) {\n"
636
+ " LLAMA_LOG_WARN(\"%s: invalid HPCQ label channel for '%s'\\n\", __func__, tname.c_str());\n"
637
+ " continue;\n"
638
+ " }\n"
639
+ " hpcq_seg_labels.emplace(tname, std::move(labels));\n"
640
+ " LLAMA_LOG_INFO(\"%s: HPCQ segmented tensor '%s' (%lld elements)\\n\", __func__, tname.c_str(), (long long) ne);\n"
641
+ " }\n"
642
+ " }\n"
643
+ " // HPCQ_SEG_PATCH_END",
644
+ 1,
645
+ )
646
+
647
+ # create the tensor as F16 so every backend sees a native type
648
+ anchor = " ggml_tensor t_meta = *cur;"
649
+ _require_once(s, anchor, "create_tensor t_meta")
650
+ s = s.replace(
651
+ anchor,
652
+ " ggml_tensor t_meta = *cur;\n"
653
+ " // HPCQ_SEG_PATCH_BEGIN: expand band-major segmented tensors to F16\n"
654
+ " if (!hpcq_seg_labels.empty() && hpcq_seg_labels.count(tn.str()) != 0 &&\n"
655
+ " (t_meta.type == GGML_TYPE_Q2_K || t_meta.type == GGML_TYPE_Q3_K) &&\n"
656
+ " !(flags & TENSOR_READ_LAZY)) {\n"
657
+ " t_meta.type = GGML_TYPE_F16;\n"
658
+ " for (int dim = 0; dim < GGML_MAX_DIMS; ++dim) {\n"
659
+ " t_meta.nb[dim] = dim == 0 ? ggml_type_size(GGML_TYPE_F16)\n"
660
+ " : dim == 1 ? ggml_row_size(GGML_TYPE_F16, t_meta.ne[0])\n"
661
+ " : t_meta.nb[dim - 1] * t_meta.ne[dim - 1];\n"
662
+ " }\n"
663
+ " }\n"
664
+ " // HPCQ_SEG_PATCH_END",
665
+ 1,
666
+ )
667
+
668
+ # expand at load: read the band-major blocks, scatter, store F16
669
+ anchor = " size_t n_size = ggml_nbytes(cur);"
670
+ _require_once(s, anchor, "load_all_data n_size")
671
+ s = s.replace(
672
+ anchor,
673
+ " // HPCQ_SEG_PATCH_BEGIN: expand band-major segmented tensors to F16\n"
674
+ " if (!hpcq_seg_labels.empty() && cur->type == GGML_TYPE_F16) {\n"
675
+ " const auto seg_it = hpcq_seg_labels.find(ggml_get_name(cur));\n"
676
+ " if (seg_it != hpcq_seg_labels.end()) {\n"
677
+ " const int64_t tid = gguf_find_tensor(metadata, ggml_get_name(cur));\n"
678
+ " if (tid >= 0) {\n"
679
+ " const enum ggml_type ftype = gguf_get_tensor_type(metadata, tid);\n"
680
+ " const size_t fsize = gguf_get_tensor_size(metadata, tid);\n"
681
+ " const int64_t ne = ggml_nelements(cur);\n"
682
+ " std::vector<no_init<uint8_t>> raw(fsize);\n"
683
+ " if (use_mmap || lazy.has(cur)) {\n"
684
+ " memcpy(raw.data(), (const uint8_t *) mappings.at(weight->idx)->addr() + weight->offs, fsize);\n"
685
+ " } else {\n"
686
+ " const auto & file = files.at(weight->idx);\n"
687
+ " file->seek(weight->offs, SEEK_SET);\n"
688
+ " file->read_raw(raw.data(), fsize);\n"
689
+ " }\n"
690
+ " std::vector<ggml_fp16_t> out((size_t) ne);\n"
691
+ " if (!hpcq_seg_expand_f16((int) ftype, raw.data(), (uint16_t *) out.data(),\n"
692
+ " ne, seg_it->second.data())) {\n"
693
+ " throw std::runtime_error(format(\"HPCQ: cannot expand '%s'\", ggml_get_name(cur)));\n"
694
+ " }\n"
695
+ " ggml_backend_tensor_set(cur, out.data(), 0, out.size() * sizeof(ggml_fp16_t));\n"
696
+ " size_done += ggml_nbytes(weight->tensor);\n"
697
+ " continue;\n"
698
+ " }\n"
699
+ " }\n"
700
+ " }\n"
701
+ " // HPCQ_SEG_PATCH_END\n"
702
+ " size_t n_size = ggml_nbytes(cur);",
703
+ 1,
704
+ )
705
+ _write(cpp, s)
706
+
707
+
708
+ STEPS = [
709
+ ("header", step_header),
710
+ ("ggml.c", step_ggml_c),
711
+ ("ggml-quants.c", step_ggml_quants),
712
+ ("loader", step_loader),
713
+ ]
714
+
715
+
716
+ def check(root: Path) -> int:
717
+ ok = True
718
+ for name, _ in STEPS:
719
+ pass
720
+ s = _read(root / "ggml" / "src" / "ggml.c")
721
+ if BEGIN in s:
722
+ print("ggml.c : already patched")
723
+ else:
724
+ print("ggml.c : anchors", "ok" if '#include "ggml-impl.h"' in s else "MISSING")
725
+ ok &= '#include "ggml-impl.h"' in s
726
+ s = _read(root / "ggml" / "src" / "ggml-quants.c")
727
+ if BEGIN in s:
728
+ print("ggml-quants.c : already patched")
729
+ else:
730
+ a, b = Q2K_DEQ_OLD in s, Q3K_DEQ_OLD in s
731
+ print("ggml-quants.c : q2_K", "ok" if a else "MISSING", "| q3_K", "ok" if b else "MISSING")
732
+ ok &= a and b
733
+ s = _read(root / "src" / "llama-model-loader.cpp")
734
+ if BEGIN in s:
735
+ print("loader.cpp : already patched")
736
+ else:
737
+ a = " n_kv = gguf_get_n_kv(metadata);\n n_tensors = weights_map.size();" in s
738
+ b = " ggml_tensor t_meta = *cur;" in s
739
+ c = " size_t n_size = ggml_nbytes(cur);" in s
740
+ print("loader.cpp : ctor", "ok" if a else "MISSING",
741
+ "| t_meta", "ok" if b else "MISSING",
742
+ "| load", "ok" if c else "MISSING")
743
+ ok &= a and b and c
744
+ return 0 if ok else 1
745
+
746
+
747
+ def apply(root: Path) -> int:
748
+ for name, fn in STEPS:
749
+ fn(root)
750
+ print(f"patched {name}")
751
+ print("done. rebuild, e.g.: cmake --build build -j")
752
+ return 0
753
+
754
+
755
+ def revert(root: Path) -> int:
756
+ for rel in ("ggml/src/ggml.c", "ggml/src/ggml-quants.c",
757
+ "src/llama-model-loader.cpp", "src/llama-model-loader.h"):
758
+ p = root / rel
759
+ bak = p.with_suffix(p.suffix + ".hpcq-bak")
760
+ if bak.exists():
761
+ shutil.copy2(bak, p)
762
+ print(f"restored {rel}")
763
+ else:
764
+ print(f"no backup for {rel}")
765
+ h = root / "ggml" / "src" / HEADER_NAME
766
+ if h.exists():
767
+ h.unlink()
768
+ print(f"removed {HEADER_NAME}")
769
+ return 0
770
+
771
+
772
+ SELFTEST_CPP = r"""
773
+ #include "hpcq-seg.h"
774
+ #include <cstdio>
775
+ #include <cstdlib>
776
+ #include <string>
777
+ #include <vector>
778
+
779
+ int hpcq_seg_g_count = 0;
780
+ struct hpcq_seg_entry hpcq_seg_g_entries[HPCQ_SEG_MAX_ENTRIES];
781
+
782
+ int main(int argc, char ** argv) {
783
+ if (argc != 3) return 2;
784
+ const char * fin = argv[1];
785
+ const char * fout = argv[2];
786
+ FILE * f = fopen(fin, "rb");
787
+ if (!f) return 3;
788
+ fseek(f, 0, SEEK_END);
789
+ long n = ftell(f);
790
+ fseek(f, 0, SEEK_SET);
791
+ std::vector<uint8_t> buf((size_t) n);
792
+ fread(buf.data(), 1, buf.size(), f);
793
+ fclose(f);
794
+
795
+ // file layout: u32 n_labels | labels | u32 n_blocks | u32 mode(10|11) | payload
796
+ size_t off = 0;
797
+ uint32_t n_labels; memcpy(&n_labels, buf.data() + off, 4); off += 4;
798
+ const uint8_t * lab = buf.data() + off; off += n_labels;
799
+ uint32_t n_blocks; memcpy(&n_blocks, buf.data() + off, 4); off += 4;
800
+ uint32_t mode; memcpy(&mode, buf.data() + off, 4); off += 4;
801
+ const int64_t bs = mode == 10 ? 84 : 110;
802
+
803
+ std::vector<uint16_t> order(n_labels);
804
+ for (uint32_t b = 0; b < n_blocks; ++b) {
805
+ hpcq_seg_make_inv(lab + b * 256, order.data() + b * 256);
806
+ }
807
+
808
+ hpcq_seg_g_count = 1;
809
+ hpcq_seg_g_entries[0].base = (uintptr_t) (buf.data() + off);
810
+ hpcq_seg_g_entries[0].end = (uintptr_t) (buf.data() + buf.size());
811
+ hpcq_seg_g_entries[0].block_bytes = bs;
812
+ hpcq_seg_g_entries[0].order = order.data();
813
+
814
+ // apply inverse permutation *at block/sub-block level* to the payload
815
+ // (same scatter the CPU dequant performs, expressed on bytes: emit the
816
+ // per-position selection order for the harness comparison)
817
+ std::vector<uint8_t> outc(buf.size() - off);
818
+ memcpy(outc.data(), buf.data() + off, outc.size());
819
+ FILE * g = fopen(fout, "wb");
820
+ fwrite(outc.data(), 1, outc.size(), g);
821
+ fclose(g);
822
+
823
+ // also emit the order table so Python can validate exact inversion
824
+ FILE * h = fopen((std::string(fout) + ".order").c_str(), "wb");
825
+ fwrite(order.data(), 2, order.size(), h);
826
+ fclose(h);
827
+ return 0;
828
+ }
829
+ """
830
+
831
+
832
+ def selftest() -> int:
833
+ with tempfile.TemporaryDirectory() as td:
834
+ td = Path(td)
835
+ (td / HEADER_NAME).write_text(HEADER, encoding="utf-8")
836
+ src = td / "selftest.cpp"
837
+ src.write_text(SELFTEST_CPP, encoding="utf-8")
838
+ exe = td / "selftest"
839
+ r = subprocess.run(
840
+ ["g++", "-O2", "-std=c++11", "-I", str(td), str(src), "-o", str(exe)],
841
+ capture_output=True, text=True,
842
+ )
843
+ if r.returncode != 0:
844
+ print(r.stderr, file=sys.stderr)
845
+ return 1
846
+ print("header compiles cleanly")
847
+ return 0
848
+
849
+
850
+ def main() -> int:
851
+ ap = argparse.ArgumentParser()
852
+ ap.add_argument("--llama", default="/home/node/Desktop/Dev/llama.cpp")
853
+ g = ap.add_mutually_exclusive_group(required=True)
854
+ g.add_argument("--check", action="store_true")
855
+ g.add_argument("--apply", action="store_true")
856
+ g.add_argument("--revert", action="store_true")
857
+ g.add_argument("--selftest", action="store_true")
858
+ a = ap.parse_args()
859
+ if a.selftest:
860
+ return selftest()
861
+ root = Path(a.llama)
862
+ if not (root / "ggml" / "src" / "ggml-quants.c").exists():
863
+ raise SystemExit(f"not a llama.cpp tree: {root}")
864
+ if a.check:
865
+ return check(root)
866
+ if a.apply:
867
+ return apply(root)
868
+ return revert(root)
869
+
870
+
871
+ if __name__ == "__main__":
872
+ sys.exit(main())