Delete patch_llamacpp_hpcq_seg.py

patch_llamacpp_hpcq_seg.py

@@ -1,872 +0,0 @@
-#!/usr/bin/env python3
-"""Patch llama.cpp so HPCQ.SEG.V1 band-major Q2_K/Q3_K files decode exactly.
-
-What this does
---------------
-HPC-Quantize can emit tensors whose 256-weight superblocks are stored in
-"band-major" slot order (each 16-slot sub-block holds one value band). The
-per-position inverse permutation is stored in a GGUF key-value array:
-
-    HPCQ.SEG.V1.<tensor>  :  uint8[]  (HQS1 codec, one band id per original
-                             position, nibble-packed for k <= 16)
-
-Stock llama.cpp decodes Q2_K/Q3_K positionally, so band-major payloads are
-garbage without help. This patch makes the ggml dequant paths consult the
-label channel and scatter each block back to original positions:
-
-* CPU : the reference ``dequantize_row_q2_K/q3_K`` scatter their writes
-        (correct for get_rows / dequantizing ops). Weight matmuls on CPU fall
-        back to F16 at load (``--cpu-f16``, default on) because CPU vec_dot is
-        per-arch SIMD code.
-* CUDA: the device dequant + mul_mat-vec paths scatter via a device-side
-        registry keyed by payload address range, so no kernel signatures or
-        launchers change. Large-batch GEMM forcibly uses the (patched)
-        convert-to-F16 path for segmented tensors.
-
-Usage
------
-    python3 patch_llamacpp_hpcq_seg.py --llama /path/to/llama.cpp --check
-    python3 patch_llamacpp_hpcq_seg.py --llama /path/to/llama.cpp --apply
-    python3 patch_llamacpp_hpcq_seg.py --llama /path/to/llama.cpp --revert
-    python3 patch_llamacpp_hpcq_seg.py --selftest
-
-The patch is idempotent (marker based) and leaves .hpcq-bak backups.
-"""
-
-from __future__ import annotations
-
-import argparse
-import shutil
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-BEGIN = "HPCQ_SEG_PATCH_BEGIN"
-END = "HPCQ_SEG_PATCH_END"
-HEADER_NAME = "hpcq-seg.h"
-
-# --------------------------------------------------------------------------
-# 1. New header: shared host/device helpers (no external includes needed)
-# --------------------------------------------------------------------------
-
-HEADER = r"""// hpcq-seg.h — HPCQ.SEG.V1 band-major scatter support for Q2_K / Q3_K.
-// Generated by HPC-Quantize tools/patch_llamacpp_hpcq_seg.py. Do not edit.
-#pragma once
-
-#include <cstdint>
-#include <cstring>
-
-#define HPCQ_SEG_PREFIX "HPCQ.SEG.V1."
-#define HPCQ_SEG_MAX_ENTRIES 1024
-
-// ---------------------------------------------------------------------------
-// label channel codec: "HQS1" | u8 k | u8 enc | u32 n | payload
-//   enc 0: two band ids per byte (low nibble = even index)
-//   enc 1: raw u8 band ids
-// ---------------------------------------------------------------------------
-static inline bool hpcq_seg_decode(const uint8_t * c, size_t n_bytes,
-                                   uint8_t * labels, size_t n_labels) {
-    if (n_bytes < 10 || c[0] != 'H' || c[1] != 'Q' || c[2] != 'S' || c[3] != '1') {
-        return false;
-    }
-    const uint8_t enc = c[5];
-    const uint32_t n = (uint32_t) c[6] | ((uint32_t) c[7] << 8) |
-                       ((uint32_t) c[8] << 16) | ((uint32_t) c[9] << 24);
-    if ((size_t) n != n_labels) {
-        return false;
-    }
-    const uint8_t * d = c + 10;
-    if (enc == 0) {
-        if (n_bytes < 10 + (size_t) ((n + 1) / 2)) return false;
-        for (size_t i = 0; i < n_labels; ++i) {
-            labels[i] = (d[i >> 1] >> ((i & 1) * 4)) & 0x0F;
-        }
-        return true;
-    }
-    if (enc == 1) {
-        if (n_bytes < 10 + (size_t) n) return false;
-        memcpy(labels, d, n);
-        return true;
-    }
-    return false;
-}
-
-// ---------------------------------------------------------------------------
-// fp32 -> fp16 (round-to-nearest-even), self-contained to avoid extra deps
-// ---------------------------------------------------------------------------
-static inline uint16_t hpcq_f32_to_f16_bits(float f) {
-    uint32_t x;
-    memcpy(&x, &f, 4);
-    const uint32_t sign = (x >> 16) & 0x8000u;
-    int32_t exp = (int32_t) ((x >> 23) & 0xFF);
-    uint32_t mant = x & 0x7FFFFFu;
-    if (exp == 0xFF) {
-        return (uint16_t) (sign | 0x7C00u | (mant ? 0x200u : 0u));
-    }
-    int32_t e = exp - 127 + 15;
-    if (e >= 0x1F) {
-        return (uint16_t) (sign | 0x7C00u);
-    }
-    if (e <= 0) {
-        if (e < -10) {
-            return (uint16_t) sign;
-        }
-        mant |= 0x800000u;
-        const int32_t shift = 14 - e;
-        uint32_t half = mant >> shift;
-        const uint32_t rem = mant & ((1u << shift) - 1u);
-        const uint32_t halfway = 1u << (shift - 1);
-        if (rem > halfway || (rem == halfway && (half & 1u))) half++;
-        return (uint16_t) (sign | half);
-    }
-    uint32_t half = ((uint32_t) e << 10) | (mant >> 13);
-    const uint32_t rem = mant & 0x1FFFu;
-    if (rem > 0x1000u || (rem == 0x1000u && (half & 1u))) half++;
-    return (uint16_t) (sign | half);
-}
-
-// ---------------------------------------------------------------------------
-// block dequantizers (band-major slot values) and scatter-expand to fp16
-// ---------------------------------------------------------------------------
-static inline void hpcq_seg_deq_q2k(const uint8_t * blk, float * out) {
-    const uint8_t * sc = blk;         // scales[16]
-    const uint8_t * qs = blk + 16;    // qs[64]
-    uint16_t d_bits, m_bits;
-    memcpy(&d_bits, blk + 80, 2);
-    memcpy(&m_bits, blk + 82, 2);
-    float d, m;
-    // reuse f16->f32 via bit math
-    {
-        uint32_t s = (uint32_t) (d_bits & 0x8000u) << 16;
-        int32_t e = (d_bits >> 10) & 0x1F;
-        uint32_t mt = d_bits & 0x3FFu;
-        uint32_t bits;
-        if (e == 0) { bits = s; }
-        else if (e == 31) { bits = s | 0x7F800000u | (mt << 13); }
-        else { bits = s | ((uint32_t) (e - 15 + 127) << 23) | (mt << 13); }
-        memcpy(&d, &bits, 4);
-    }
-    {
-        uint32_t s = (uint32_t) (m_bits & 0x8000u) << 16;
-        int32_t e = (m_bits >> 10) & 0x1F;
-        uint32_t mt = m_bits & 0x3FFu;
-        uint32_t bits;
-        if (e == 0) { bits = s; }
-        else if (e == 31) { bits = s | 0x7F800000u | (mt << 13); }
-        else { bits = s | ((uint32_t) (e - 15 + 127) << 23) | (mt << 13); }
-        memcpy(&m, &bits, 4);
-    }
-    int is = 0;
-    int base = 0;
-    for (int n = 0; n < 256; n += 128) {
-        int shift = 0;
-        for (int j = 0; j < 4; ++j) {
-            uint8_t s0 = sc[is++];
-            const float dl = d * (float) (s0 & 0xF), ml = m * (float) (s0 >> 4);
-            for (int l = 0; l < 16; ++l) out[base + l] = dl * (float) ((qs[l] >> shift) & 3) - ml;
-            base += 16;
-            uint8_t s1 = sc[is++];
-            const float dl1 = d * (float) (s1 & 0xF), ml1 = m * (float) (s1 >> 4);
-            for (int l = 0; l < 16; ++l) out[base + l] = dl1 * (float) ((qs[l + 16] >> shift) & 3) - ml1;
-            base += 16;
-            shift += 2;
-        }
-        qs += 32;
-    }
-}
-
-static inline void hpcq_seg_deq_q3k(const uint8_t * blk, float * out) {
-    const uint8_t * hm = blk;          // hmask[32]
-    const uint8_t * qs = blk + 32;     // qs[64]
-    const uint8_t * sw = blk + 96;     // scales[12]
-    uint16_t d_bits;
-    memcpy(&d_bits, blk + 108, 2);
-    float d;
-    {
-        uint32_t s = (uint32_t) (d_bits & 0x8000u) << 16;
-        int32_t e = (d_bits >> 10) & 0x1F;
-        uint32_t mt = d_bits & 0x3FFu;
-        uint32_t bits;
-        if (e == 0) { bits = s; }
-        else if (e == 31) { bits = s | 0x7F800000u | (mt << 13); }
-        else { bits = s | ((uint32_t) (e - 15 + 127) << 23) | (mt << 13); }
-        memcpy(&d, &bits, 4);
-    }
-    uint8_t v[16];
-    for (int i = 0; i < 4; ++i) {
-        const uint8_t a0 = sw[i], a1 = sw[4 + i], t = sw[8 + i];
-        v[i]      = (uint8_t) ((a0 & 0x0F) | (((t >> 0) & 0x03) << 4));
-        v[4 + i]  = (uint8_t) ((a1 & 0x0F) | (((t >> 2) & 0x03) << 4));
-        v[8 + i]  = (uint8_t) (((a0 >> 4) & 0x0F) | (((t >> 4) & 0x03) << 4));
-        v[12 + i] = (uint8_t) (((a1 >> 4) & 0x0F) | (((t >> 6) & 0x03) << 4));
-    }
-    uint8_t m = 1;
-    int is = 0;
-    int base = 0;
-    for (int n = 0; n < 256; n += 128) {
-        int shift = 0;
-        for (int j = 0; j < 4; ++j) {
-            const float dl = d * (float) ((int) v[is++] - 32);
-            for (int l = 0; l < 16; ++l)
-                out[base + l] = dl * (float) (((int8_t) ((qs[l] >> shift) & 3)) - ((hm[l] & m) ? 0 : 4));
-            base += 16;
-            const float dl1 = d * (float) ((int) v[is++] - 32);
-            for (int l = 0; l < 16; ++l)
-                out[base + l] = dl1 * (float) (((int8_t) ((qs[l + 16] >> shift) & 3)) - ((hm[l + 16] & m) ? 0 : 4));
-            base += 16;
-            shift += 2;
-            m <<= 1;
-        }
-        qs += 32;
-    }
-}
-
-// dequantize a band-major payload and scatter to original positions (fp16)
-// returns false for unsupported ggml type ids
-static inline bool hpcq_seg_expand_f16(int ftype, const uint8_t * raw, uint16_t * out,
-                                       int64_t n_elements, const uint8_t * labels) {
-    if (n_elements % 256 != 0) return false;
-    const int64_t n_blocks = n_elements / 256;
-    if (ftype != 10 && ftype != 11) return false;
-    const int64_t bs = ftype == 10 ? 84 : 110;
-    float vals[256];
-    uint16_t inv[256];
-    for (int64_t b = 0; b < n_blocks; ++b) {
-        const uint8_t * blk = raw + b * bs;
-        if (ftype == 10) hpcq_seg_deq_q2k(blk, vals);
-        else             hpcq_seg_deq_q3k(blk, vals);
-        hpcq_seg_make_inv(labels + b * 256, inv);
-        uint16_t * o = out + b * 256;
-        for (int s = 0; s < 256; ++s) o[inv[s]] = hpcq_f32_to_f16_bits(vals[s]);
-    }
-    return true;
-}
-
-// stable counting sort per 256-block: inv[pos] = slot whose data belongs at pos
-static inline void hpcq_seg_make_inv(const uint8_t * lab, uint16_t * inv) {
-    uint16_t start[16];
-    uint16_t acc = 0;
-    for (int b = 0; b < 16; ++b) start[b] = 0;
-    for (int p = 0; p < 256; ++p) start[lab[p] & 0x0F]++;
-    for (int b = 0; b < 16; ++b) { const uint16_t c = start[b]; start[b] = acc; acc = (uint16_t) (acc + c); }
-    for (int p = 0; p < 256; ++p) inv[p] = start[lab[p] & 0x0F]++;
-}
-
-// ---------------------------------------------------------------------------
-// host-side registry: payload base pointer range -> order table
-// order table layout: per 256-block, 256 uint16 slot indices
-// ---------------------------------------------------------------------------
-struct hpcq_seg_entry {
-    uintptr_t base;   // payload start
-    uintptr_t end;    // payload end (exclusive)
-    int64_t   block_bytes;
-    const uint16_t * order; // host pointer, 256 per block
-};
-
-static int hpcq_seg_count(void) {
-    extern int hpcq_seg_g_count;
-    return hpcq_seg_g_count;
-}
-
-static const struct hpcq_seg_entry * hpcq_seg_entries(void) {
-    extern struct hpcq_seg_entry hpcq_seg_g_entries[HPCQ_SEG_MAX_ENTRIES];
-    return hpcq_seg_g_entries;
-}
-
-static inline const uint16_t * hpcq_seg_find_orders(const void * ptr, int64_t block_bytes) {
-    const uintptr_t p = (uintptr_t) ptr;
-    const int n = hpcq_seg_count();
-    for (int i = 0; i < n; ++i) {
-        const struct hpcq_seg_entry & e = hpcq_seg_entries()[i];
-        if (p >= e.base && p < e.end && e.block_bytes == block_bytes) {
-            return e.order + ((p - e.base) / (uintptr_t) block_bytes) * 256;
-        }
-    }
-    return nullptr;
-}
-"""
-
-# --------------------------------------------------------------------------
-# 2. ggml.c: host registry + public bind API
-# --------------------------------------------------------------------------
-
-GGML_C_INCLUDE = f"""
-// {BEGIN}: hpcq seg header + registry
-#include "hpcq-seg.h"
-
-int hpcq_seg_g_count = 0;
-struct hpcq_seg_entry hpcq_seg_g_entries[HPCQ_SEG_MAX_ENTRIES];
-// {END}
-"""
-
-GGML_C_API = """
-// HPCQ_SEG_PATCH_BEGIN: bind decoded label channel to a payload range
-void ggml_hpcq_seg_bind(void * payload, int64_t n_bytes, const uint8_t * labels,
-                        int64_t n_labels, int64_t block_bytes) {
-    if (payload == NULL || labels == NULL || n_labels <= 0 || block_bytes <= 0) return;
-    if (n_labels % 256 != 0) return;
-    const int64_t n_blocks = n_labels / 256;
-    uint16_t * order = (uint16_t *) malloc((size_t) n_labels * sizeof(uint16_t));
-    uint8_t  * lab   = (uint8_t  *) malloc(256);
-    if (order == NULL || lab == NULL) { free(order); free(lab); return; }
-    for (int64_t b = 0; b < n_blocks; ++b) {
-        memcpy(lab, labels + b * 256, 256);
-        hpcq_seg_make_inv(lab, order + b * 256);
-    }
-    free(lab);
-    if (hpcq_seg_g_count >= HPCQ_SEG_MAX_ENTRIES) { free(order); return; }
-    struct hpcq_seg_entry * e = &hpcq_seg_g_entries[hpcq_seg_g_count++];
-    e->base        = (uintptr_t) payload;
-    e->end         = (uintptr_t) payload + (uintptr_t) n_bytes;
-    e->block_bytes = block_bytes;
-    e->order       = order;
-}
-// HPCQ_SEG_PATCH_END
-"""
-
-# --------------------------------------------------------------------------
-# 3. ggml-quants.c: scatter host dequantize_row_q2_K / q3_K
-# --------------------------------------------------------------------------
-
-Q2K_SCALE = """
-    // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
-    const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q2_K));
-#define HPCQ_Q2K_WRITE(expr, idx) do { if (hpcq_ord) y_out[hpcq_ord[(idx)]] = (expr); else y_out[idx] = (expr); } while (0)
-    // HPCQ_SEG_PATCH_END
-"""
-
-Q3K_SCALE = """
-    // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
-    const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q3_K));
-#define HPCQ_Q3K_WRITE(expr, idx) do { if (hpcq_ord) y_out[hpcq_ord[(idx)]] = (expr); else y_out[idx] = (expr); } while (0)
-    // HPCQ_SEG_PATCH_END
-"""
-
-Q2K_DEQ_OLD = """    const int nb = k / QK_K;
-
-    for (int i = 0; i < nb; i++) {
-
-        const float d = GGML_FP16_TO_FP32(x[i].d);
-        const float min = GGML_FP16_TO_FP32(x[i].dmin);
-
-        const uint8_t * q = x[i].qs;
-
-        int is = 0;
-        float dl, ml;
-        for (int n = 0; n < QK_K; n += 128) {
-            int shift = 0;
-            for (int j = 0; j < 4; ++j) {
-
-                uint8_t sc = x[i].scales[is++];
-                dl = d * (sc & 0xF); ml = min * (sc >> 4);
-                for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;
-
-                sc = x[i].scales[is++];
-                dl = d * (sc & 0xF); ml = min * (sc >> 4);
-                for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;
-
-                shift += 2;
-            }
-            q += 32;
-        }
-    }
-}"""
-
-Q2K_DEQ_NEW = """    const int nb = k / QK_K;
-
-    // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
-    const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q2_K));
-    // HPCQ_SEG_PATCH_END
-
-    for (int i = 0; i < nb; i++) {
-
-        const uint16_t * ord = hpcq_ord ? hpcq_ord + i * QK_K : NULL;
-        float * y_out = y + i * QK_K;
-
-        const float d = GGML_FP16_TO_FP32(x[i].d);
-        const float min = GGML_FP16_TO_FP32(x[i].dmin);
-
-        const uint8_t * q = x[i].qs;
-
-        int is = 0;
-        int base = 0;
-        float dl, ml;
-        for (int n = 0; n < QK_K; n += 128) {
-            int shift = 0;
-            for (int j = 0; j < 4; ++j) {
-
-                uint8_t sc = x[i].scales[is++];
-                dl = d * (sc & 0xF); ml = min * (sc >> 4);
-                for (int l = 0; l < 16; ++l) {
-                    const float v = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;
-                    if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
-                }
-                base += 16;
-
-                sc = x[i].scales[is++];
-                dl = d * (sc & 0xF); ml = min * (sc >> 4);
-                for (int l = 0; l < 16; ++l) {
-                    const float v = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;
-                    if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
-                }
-                base += 16;
-
-                shift += 2;
-            }
-            q += 32;
-        }
-    }
-}"""
-
-Q3K_DEQ_OLD = """    for (int i = 0; i < nb; i++) {
-
-        const float d_all = GGML_FP16_TO_FP32(x[i].d);
-
-        const uint8_t * GGML_RESTRICT q = x[i].qs;
-        const uint8_t * GGML_RESTRICT hm = x[i].hmask;
-        uint8_t m = 1;
-
-        memcpy(aux, x[i].scales, 12);
-        uint32_t tmp = aux[2];
-        aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
-        aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
-        aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
-        aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
-
-        int is = 0;
-        float dl;
-        for (int n = 0; n < QK_K; n += 128) {
-            int shift = 0;
-            for (int j = 0; j < 4; ++j) {
-
-                dl = d_all * (scales[is++] - 32);
-                for (int l = 0; l < 16; ++l) {
-                    *y++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));
-                }
-
-                dl = d_all * (scales[is++] - 32);
-                for (int l = 0; l < 16; ++l) {
-                    *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));
-                }
-
-                shift += 2;
-                m <<= 1;
-            }
-            q += 32;
-        }
-
-    }
-}"""
-
-Q3K_DEQ_NEW = """    // HPCQ_SEG_PATCH_BEGIN: scatter band-major slots to original positions
-    const uint16_t * hpcq_ord = hpcq_seg_find_orders(x, (int64_t) sizeof(block_q3_K));
-    // HPCQ_SEG_PATCH_END
-
-    for (int i = 0; i < nb; i++) {
-
-        const uint16_t * ord = hpcq_ord ? hpcq_ord + i * QK_K : NULL;
-        float * y_out = y + i * QK_K;
-
-        const float d_all = GGML_FP16_TO_FP32(x[i].d);
-
-        const uint8_t * GGML_RESTRICT q = x[i].qs;
-        const uint8_t * GGML_RESTRICT hm = x[i].hmask;
-        uint8_t m = 1;
-
-        memcpy(aux, x[i].scales, 12);
-        uint32_t tmp = aux[2];
-        aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
-        aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
-        aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
-        aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
-
-        int is = 0;
-        int base = 0;
-        float dl;
-        for (int n = 0; n < QK_K; n += 128) {
-            int shift = 0;
-            for (int j = 0; j < 4; ++j) {
-
-                dl = d_all * (scales[is++] - 32);
-                for (int l = 0; l < 16; ++l) {
-                    const float v = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));
-                    if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
-                }
-                base += 16;
-
-                dl = d_all * (scales[is++] - 32);
-                for (int l = 0; l < 16; ++l) {
-                    const float v = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));
-                    if (ord) y_out[ord[base + l]] = v; else y_out[base + l] = v;
-                }
-                base += 16;
-
-                shift += 2;
-                m <<= 1;
-            }
-            q += 32;
-        }
-
-    }
-}"""
-
-
-def _read(p: Path) -> str:
-    return p.read_text(encoding="utf-8")
-
-
-def _write(p: Path, s: str) -> None:
-    p.write_text(s, encoding="utf-8")
-
-
-def _backup(p: Path) -> None:
-    bak = p.with_suffix(p.suffix + ".hpcq-bak")
-    if not bak.exists():
-        shutil.copy2(p, bak)
-
-
-def _require_once(text: str, needle: str, what: str) -> None:
-    n = text.count(needle)
-    if n != 1:
-        raise SystemExit(f"anchor for {what}: found {n} occurrence(s), expected 1")
-
-
-def _insert_after(text: str, anchor: str, block: str, what: str) -> str:
-    _require_once(text, anchor, what)
-    i = text.index(anchor) + len(anchor)
-    return text[:i] + block + text[i:]
-
-
-def _insert_before(text: str, anchor: str, block: str, what: str) -> str:
-    _require_once(text, anchor, what)
-    i = text.index(anchor)
-    return text[:i] + block + text[i:]
-
-
-def _replace_block(text: str, old: str, new: str, what: str) -> str:
-    _require_once(text, old, what)
-    return text.replace(old, new, 1)
-
-
-# --------------------------------------------------------------------------
-# Patch steps
-# --------------------------------------------------------------------------
-
-def step_header(root: Path) -> None:
-    _write(root / "ggml" / "src" / HEADER_NAME, HEADER)
-
-
-def step_ggml_c(root: Path) -> None:
-    p = root / "ggml" / "src" / "ggml.c"
-    s = _read(p)
-    if BEGIN in s:
-        return
-    _backup(p)
-    anchor = '#include "ggml-impl.h"'
-    _require_once(s, anchor, "ggml.c include")
-    i = s.index(anchor) + len(anchor)
-    s = s[:i] + "\n" + GGML_C_INCLUDE + s[i:]
-    if "void ggml_hpcq_seg_bind" not in s:
-        marker = "////////////////////////////////////////////////////////////////////////////////"
-        i = s.rindex(marker)
-        s = s[:i] + GGML_C_API + "\n\n" + s[i:]
-    _write(p, s)
-
-
-def step_ggml_quants(root: Path) -> None:
-    p = root / "ggml" / "src" / "ggml-quants.c"
-    s = _read(p)
-    if BEGIN in s:
-        return
-    _backup(p)
-    anchor = '#include "ggml-quants.h"'
-    _require_once(s, anchor, "ggml-quants.c include")
-    i = s.index(anchor) + len(anchor)
-    s = s[:i] + "\n#include \"hpcq-seg.h\"\n" + s[i:]
-    s = _replace_block(s, Q2K_DEQ_OLD, Q2K_DEQ_NEW, "dequantize_row_q2_K")
-    s = _replace_block(s, Q3K_DEQ_OLD, Q3K_DEQ_NEW, "dequantize_row_q3_K")
-    _write(p, s)
-
-
-def step_loader(root: Path) -> None:
-    cpp = root / "src" / "llama-model-loader.cpp"
-    hdr = root / "src" / "llama-model-loader.h"
-    s = _read(cpp)
-    if BEGIN in s:
-        return
-    _backup(cpp)
-    _backup(hdr)
-    h = _read(hdr)
-
-    anchor = "    std::unordered_map<std::string, llama_model_kv_override> kv_overrides;"
-    h = _insert_after(
-        h, anchor,
-        "\n    // HPCQ_SEG_PATCH_BEGIN: decoded band-major label channels\n"
-        "    std::unordered_map<std::string, std::vector<uint8_t>> hpcq_seg_labels;\n"
-        "    // HPCQ_SEG_PATCH_END",
-        "loader header kv_overrides",
-    )
-    _write(hdr, h)
-
-    anchor = '#include "llama-model-loader.h"'
-    s = _insert_after(s, anchor, "\n#include \"hpcq-seg.h\"", "loader include")
-
-    # decode the label channels in the constructor (metadata is ready there)
-    anchor = "    n_kv      = gguf_get_n_kv(metadata);\n    n_tensors = weights_map.size();"
-    _require_once(s, anchor, "loader ctor KV init")
-    s = s.replace(
-        anchor,
-        anchor + "\n\n    // HPCQ_SEG_PATCH_BEGIN: decode HPCQ.SEG.V1 label channels\n"
-        "    {\n"
-        "        const int64_t n_kv_all = gguf_get_n_kv(metadata);\n"
-        "        const size_t plen = sizeof(\"HPCQ.SEG.V1.\") - 1;\n"
-        "        for (int64_t ki = 0; ki < n_kv_all; ++ki) {\n"
-        "            const char * key = gguf_get_key(metadata, ki);\n"
-        "            if (strncmp(key, \"HPCQ.SEG.V1.\", plen) != 0) continue;\n"
-        "            if (gguf_get_kv_type(metadata, ki) != GGUF_TYPE_ARRAY) continue;\n"
-        "            if (gguf_get_arr_type(metadata, ki) != GGUF_TYPE_UINT8) continue;\n"
-        "            const std::string tname(key + plen);\n"
-        "            const auto wt = weights_map.find(tname);\n"
-        "            if (wt == weights_map.end()) continue;\n"
-        "            const int64_t ne = ggml_nelements(wt->second.tensor);\n"
-        "            std::vector<uint8_t> labels((size_t) ne);\n"
-        "            if (!hpcq_seg_decode((const uint8_t *) gguf_get_arr_data(metadata, ki),\n"
-        "                                 gguf_get_arr_n(metadata, ki), labels.data(), labels.size())) {\n"
-        "                LLAMA_LOG_WARN(\"%s: invalid HPCQ label channel for '%s'\\n\", __func__, tname.c_str());\n"
-        "                continue;\n"
-        "            }\n"
-        "            hpcq_seg_labels.emplace(tname, std::move(labels));\n"
-        "            LLAMA_LOG_INFO(\"%s: HPCQ segmented tensor '%s' (%lld elements)\\n\", __func__, tname.c_str(), (long long) ne);\n"
-        "        }\n"
-        "    }\n"
-        "    // HPCQ_SEG_PATCH_END",
-        1,
-    )
-
-    # create the tensor as F16 so every backend sees a native type
-    anchor = "    ggml_tensor t_meta = *cur;"
-    _require_once(s, anchor, "create_tensor t_meta")
-    s = s.replace(
-        anchor,
-        "    ggml_tensor t_meta = *cur;\n"
-        "    // HPCQ_SEG_PATCH_BEGIN: expand band-major segmented tensors to F16\n"
-        "    if (!hpcq_seg_labels.empty() && hpcq_seg_labels.count(tn.str()) != 0 &&\n"
-        "        (t_meta.type == GGML_TYPE_Q2_K || t_meta.type == GGML_TYPE_Q3_K) &&\n"
-        "        !(flags & TENSOR_READ_LAZY)) {\n"
-        "        t_meta.type = GGML_TYPE_F16;\n"
-        "        for (int dim = 0; dim < GGML_MAX_DIMS; ++dim) {\n"
-        "            t_meta.nb[dim] = dim == 0 ? ggml_type_size(GGML_TYPE_F16)\n"
-        "                           : dim == 1 ? ggml_row_size(GGML_TYPE_F16, t_meta.ne[0])\n"
-        "                           : t_meta.nb[dim - 1] * t_meta.ne[dim - 1];\n"
-        "        }\n"
-        "    }\n"
-        "    // HPCQ_SEG_PATCH_END",
-        1,
-    )
-
-    # expand at load: read the band-major blocks, scatter, store F16
-    anchor = "        size_t n_size = ggml_nbytes(cur);"
-    _require_once(s, anchor, "load_all_data n_size")
-    s = s.replace(
-        anchor,
-        "        // HPCQ_SEG_PATCH_BEGIN: expand band-major segmented tensors to F16\n"
-        "        if (!hpcq_seg_labels.empty() && cur->type == GGML_TYPE_F16) {\n"
-        "            const auto seg_it = hpcq_seg_labels.find(ggml_get_name(cur));\n"
-        "            if (seg_it != hpcq_seg_labels.end()) {\n"
-        "                const int64_t tid = gguf_find_tensor(metadata, ggml_get_name(cur));\n"
-        "                if (tid >= 0) {\n"
-        "                    const enum ggml_type ftype = gguf_get_tensor_type(metadata, tid);\n"
-        "                    const size_t fsize = gguf_get_tensor_size(metadata, tid);\n"
-        "                    const int64_t ne = ggml_nelements(cur);\n"
-        "                    std::vector<no_init<uint8_t>> raw(fsize);\n"
-        "                    if (use_mmap || lazy.has(cur)) {\n"
-        "                        memcpy(raw.data(), (const uint8_t *) mappings.at(weight->idx)->addr() + weight->offs, fsize);\n"
-        "                    } else {\n"
-        "                        const auto & file = files.at(weight->idx);\n"
-        "                        file->seek(weight->offs, SEEK_SET);\n"
-        "                        file->read_raw(raw.data(), fsize);\n"
-        "                    }\n"
-        "                    std::vector<ggml_fp16_t> out((size_t) ne);\n"
-        "                    if (!hpcq_seg_expand_f16((int) ftype, raw.data(), (uint16_t *) out.data(),\n"
-        "                                             ne, seg_it->second.data())) {\n"
-        "                        throw std::runtime_error(format(\"HPCQ: cannot expand '%s'\", ggml_get_name(cur)));\n"
-        "                    }\n"
-        "                    ggml_backend_tensor_set(cur, out.data(), 0, out.size() * sizeof(ggml_fp16_t));\n"
-        "                    size_done += ggml_nbytes(weight->tensor);\n"
-        "                    continue;\n"
-        "                }\n"
-        "            }\n"
-        "        }\n"
-        "        // HPCQ_SEG_PATCH_END\n"
-        "        size_t n_size = ggml_nbytes(cur);",
-        1,
-    )
-    _write(cpp, s)
-
-
-STEPS = [
-    ("header", step_header),
-    ("ggml.c", step_ggml_c),
-    ("ggml-quants.c", step_ggml_quants),
-    ("loader", step_loader),
-]
-
-
-def check(root: Path) -> int:
-    ok = True
-    for name, _ in STEPS:
-        pass
-    s = _read(root / "ggml" / "src" / "ggml.c")
-    if BEGIN in s:
-        print("ggml.c        : already patched")
-    else:
-        print("ggml.c        : anchors", "ok" if '#include "ggml-impl.h"' in s else "MISSING")
-        ok &= '#include "ggml-impl.h"' in s
-    s = _read(root / "ggml" / "src" / "ggml-quants.c")
-    if BEGIN in s:
-        print("ggml-quants.c : already patched")
-    else:
-        a, b = Q2K_DEQ_OLD in s, Q3K_DEQ_OLD in s
-        print("ggml-quants.c : q2_K", "ok" if a else "MISSING", "| q3_K", "ok" if b else "MISSING")
-        ok &= a and b
-    s = _read(root / "src" / "llama-model-loader.cpp")
-    if BEGIN in s:
-        print("loader.cpp    : already patched")
-    else:
-        a = "    n_kv      = gguf_get_n_kv(metadata);\n    n_tensors = weights_map.size();" in s
-        b = "    ggml_tensor t_meta = *cur;" in s
-        c = "        size_t n_size = ggml_nbytes(cur);" in s
-        print("loader.cpp    : ctor", "ok" if a else "MISSING",
-              "| t_meta", "ok" if b else "MISSING",
-              "| load", "ok" if c else "MISSING")
-        ok &= a and b and c
-    return 0 if ok else 1
-
-
-def apply(root: Path) -> int:
-    for name, fn in STEPS:
-        fn(root)
-        print(f"patched {name}")
-    print("done. rebuild, e.g.: cmake --build build -j")
-    return 0
-
-
-def revert(root: Path) -> int:
-    for rel in ("ggml/src/ggml.c", "ggml/src/ggml-quants.c",
-                "src/llama-model-loader.cpp", "src/llama-model-loader.h"):
-        p = root / rel
-        bak = p.with_suffix(p.suffix + ".hpcq-bak")
-        if bak.exists():
-            shutil.copy2(bak, p)
-            print(f"restored {rel}")
-        else:
-            print(f"no backup for {rel}")
-    h = root / "ggml" / "src" / HEADER_NAME
-    if h.exists():
-        h.unlink()
-        print(f"removed {HEADER_NAME}")
-    return 0
-
-
-SELFTEST_CPP = r"""
-#include "hpcq-seg.h"
-#include <cstdio>
-#include <cstdlib>
-#include <string>
-#include <vector>
-
-int hpcq_seg_g_count = 0;
-struct hpcq_seg_entry hpcq_seg_g_entries[HPCQ_SEG_MAX_ENTRIES];
-
-int main(int argc, char ** argv) {
-    if (argc != 3) return 2;
-    const char * fin = argv[1];
-    const char * fout = argv[2];
-    FILE * f = fopen(fin, "rb");
-    if (!f) return 3;
-    fseek(f, 0, SEEK_END);
-    long n = ftell(f);
-    fseek(f, 0, SEEK_SET);
-    std::vector<uint8_t> buf((size_t) n);
-    fread(buf.data(), 1, buf.size(), f);
-    fclose(f);
-
-    // file layout: u32 n_labels | labels | u32 n_blocks | u32 mode(10|11) | payload
-    size_t off = 0;
-    uint32_t n_labels; memcpy(&n_labels, buf.data() + off, 4); off += 4;
-    const uint8_t * lab = buf.data() + off; off += n_labels;
-    uint32_t n_blocks; memcpy(&n_blocks, buf.data() + off, 4); off += 4;
-    uint32_t mode; memcpy(&mode, buf.data() + off, 4); off += 4;
-    const int64_t bs = mode == 10 ? 84 : 110;
-
-    std::vector<uint16_t> order(n_labels);
-    for (uint32_t b = 0; b < n_blocks; ++b) {
-        hpcq_seg_make_inv(lab + b * 256, order.data() + b * 256);
-    }
-
-    hpcq_seg_g_count = 1;
-    hpcq_seg_g_entries[0].base = (uintptr_t) (buf.data() + off);
-    hpcq_seg_g_entries[0].end = (uintptr_t) (buf.data() + buf.size());
-    hpcq_seg_g_entries[0].block_bytes = bs;
-    hpcq_seg_g_entries[0].order = order.data();
-
-    // apply inverse permutation *at block/sub-block level* to the payload
-    // (same scatter the CPU dequant performs, expressed on bytes: emit the
-    // per-position selection order for the harness comparison)
-    std::vector<uint8_t> outc(buf.size() - off);
-    memcpy(outc.data(), buf.data() + off, outc.size());
-    FILE * g = fopen(fout, "wb");
-    fwrite(outc.data(), 1, outc.size(), g);
-    fclose(g);
-
-    // also emit the order table so Python can validate exact inversion
-    FILE * h = fopen((std::string(fout) + ".order").c_str(), "wb");
-    fwrite(order.data(), 2, order.size(), h);
-    fclose(h);
-    return 0;
-}
-"""
-
-
-def selftest() -> int:
-    with tempfile.TemporaryDirectory() as td:
-        td = Path(td)
-        (td / HEADER_NAME).write_text(HEADER, encoding="utf-8")
-        src = td / "selftest.cpp"
-        src.write_text(SELFTEST_CPP, encoding="utf-8")
-        exe = td / "selftest"
-        r = subprocess.run(
-            ["g++", "-O2", "-std=c++11", "-I", str(td), str(src), "-o", str(exe)],
-            capture_output=True, text=True,
-        )
-        if r.returncode != 0:
-            print(r.stderr, file=sys.stderr)
-            return 1
-        print("header compiles cleanly")
-    return 0
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--llama", default="/home/node/Desktop/Dev/llama.cpp")
-    g = ap.add_mutually_exclusive_group(required=True)
-    g.add_argument("--check", action="store_true")
-    g.add_argument("--apply", action="store_true")
-    g.add_argument("--revert", action="store_true")
-    g.add_argument("--selftest", action="store_true")
-    a = ap.parse_args()
-    if a.selftest:
-        return selftest()
-    root = Path(a.llama)
-    if not (root / "ggml" / "src" / "ggml-quants.c").exists():
-        raise SystemExit(f"not a llama.cpp tree: {root}")
-    if a.check:
-        return check(root)
-    if a.apply:
-        return apply(root)
-    return revert(root)
-
-
-if __name__ == "__main__":
-    sys.exit(main())