--iq2auto

hexstate_quantize.c

@@ -1793,21 +1793,130 @@ void hexstate_set_spectral_params(float dc_lambda, float vw_lambda, float dc_dec
         g_hex_dc_decay = dc_decay;
 }
 
-/* Spectral penalty: (Σe + dc_carry)² + Σ vesica². dc_carry is the
- * rolling residual we want this block to cancel (Σe ≈ −dc_carry).
- * Reconstruction target is always the true weight; we never quantize x−bias. */
+/* ── Fold pyramid ─────────────────────────────────────────────────────────
+ * Vesica is the block folded in half once; DC is the block folded in half
+ * log2(n) times. The levels between are the rest of the same tree:
+ *
+ *   v¹[p] = e[p] + e[p+n/2]        n/2 nodes   (vesica)
+ *   vᵏ[p] = vᵏ⁻¹[p] + vᵏ⁻¹[p+n/2ᵏ] n/2ᵏ nodes
+ *   vᴸ    = Σ e                    1 node      (DC),  L = log2 n
+ *
+ * ‖vᵏ‖² is 2ᵏ× the error energy in the subspace of n/2ᵏ-periodic patterns
+ * (Walsh functions on the low 8−k index bits), so penalising level k drives
+ * the error orthogonal to activations with that period. Level weights are
+ * geometric, λ_k = λ_vw·γᵏ⁻¹/n for k ≤ depth (depth=1 ⇒ the classic single
+ * fold), and the top level keeps λ_dc/n so DC behaviour is unchanged.
+ *
+ * Measured (SmolLM2 ffn_down splice, IQ2_S, λ=(16,1), budget 2e-2): depth 7
+ * γ=2 pulls the level-5..7 row-lane residuals from 0.9 → 0.1–0.4 of white
+ * but costs +2.7% RMSE / +1.7% wRMSE and leaves PPL inside noise. Only DC
+ * is invariant to the (arbitrary) neuron ordering, so the other periodic
+ * lanes have nothing in the activations to cancel against. Default depth is
+ * therefore 1; the tree stays available via hexstate_set_fold_params.
+ *
+ * Pyramid layout: T[0..n/2) level 1, then n/4 level 2, …, T[n-2] = DC.
+ * A carry pyramid C (same layout) holds the decayed cumulative residual per
+ * lane along the row; the penalty is Σ_k λ_k Σ_p (Tᵏ[p] + Cᵏ[p])². */
+#define HEX_FOLD_LEVELS 8               /* log2(QK_K) */
+static int   g_hex_fold_depth = 1;      /* vesica levels 1..depth (7 = every level below DC) */
+static float g_hex_fold_gamma = 2.0f;   /* per-level geometric weight */
+static int   g_hex_carry_cumulative = 1;/* 1: S ← decay·S + Σe (row residual = last miss);
+                                           0: legacy, carry only the previous block's Σe */
+
+void hexstate_set_fold_params(int depth, float gamma, int carry_cumulative)
+{
+    if (depth >= 0 && depth <= HEX_FOLD_LEVELS - 1) g_hex_fold_depth = depth;
+    if (gamma > 0.0f) g_hex_fold_gamma = gamma;
+    if (carry_cumulative >= 0) g_hex_carry_cumulative = carry_cumulative ? 1 : 0;
+}
+
+/* ── Activation lanes ─────────────────────────────────────────────────────
+ * The fold tree is a fixed family of index patterns; DC (the root) is the
+ * only member invariant to the arbitrary neuron order, and measured on the
+ * ffn_down input it carries ~0.1% of E[‖a‖²]. What the output error
+ * eᵀE[aaᵀ]e actually lives in is the top eigen-directions u_k of E[aaᵀ]
+ * (25–80% of the trace in the top 16). So the row lanes are generalised to
+ * arbitrary directions: penalty Σ_k λ·ev_k (u_k·e_row + carry_k)², with the
+ * same cumulative carry along the row. ev_k are in E[a²] units, the same as
+ * the imatrix weights on SSE, so λ=1 is the rank-r correction of the
+ * diagonal (imatrix) objective toward the true output error. Nothing is
+ * needed at inference: only the codeword choice changes. */
+#define HEX_MAX_LANES 64
+static const float *g_lane_U  = NULL;   /* r × cols, row-major */
+static const float *g_lane_ev = NULL;   /* r */
+static int     g_lane_r = 0;
+static int64_t g_lane_cols = 0;
+static float   g_lane_lambda = 1.0f;
+
+void hexstate_set_activation_lanes(const float *U, const float *ev, int r, int64_t cols, float lambda)
+{
+    if (!U || !ev || r <= 0 || cols <= 0) { g_lane_U = NULL; g_lane_ev = NULL; g_lane_r = 0; g_lane_cols = 0; return; }
+    g_lane_U = U; g_lane_ev = ev; g_lane_r = r > HEX_MAX_LANES ? HEX_MAX_LANES : r;
+    g_lane_cols = cols; g_lane_lambda = lambda > 0.0f ? lambda : 1.0f;
+}
+
+typedef struct {
+    const float *U;      /* lane k slice for this block: U + k*stride + i */
+    int64_t      stride;
+    const float *ev;
+    int          r;
+    const float *carry;  /* r decayed cumulative lane residuals */
+    float        lambda;
+} IQ2LaneCtx;
+
+/* Level weights for a block of n = 2^L. wk[k-1] for k = 1..L. */
+static inline void hex_fold_weights(int n, float *wk)
+{
+    int L = 0; while ((1 << L) < n) L++;
+    float g = 1.0f;
+    for (int k = 1; k <= L; k++) {
+        if (k == L)                    wk[k-1] = HEX_DC_LAMBDA / (float)n;
+        else if (k <= g_hex_fold_depth) wk[k-1] = (HEX_VW_LAMBDA / (float)n) * g;
+        else                           wk[k-1] = 0.0f;
+        g *= g_hex_fold_gamma;
+    }
+}
+
+/* Build the pyramid (n-1 floats) from leaves e[n]. */
+static inline void hex_fold_build(const float *e, int n, float *T)
+{
+    const float *src = e; float *dst = T;
+    for (int m = n / 2; m >= 1; m >>= 1) {
+        for (int p = 0; p < m; p++) dst[p] = src[p] + src[p + m];
+        src = dst; dst += m;
+    }
+}
+
+/* Σ_k λ_k Σ_p (Tᵏ[p] + Cᵏ[p])²; C may be NULL (no carry). */
+static inline float hex_fold_energy(const float *T, const float *C, int n)
+{
+    float wk[HEX_FOLD_LEVELS];
+    hex_fold_weights(n, wk);
+    float acc = 0.0f; int off = 0, k = 0;
+    for (int m = n / 2; m >= 1; m >>= 1, k++) {
+        if (wk[k] != 0.0f) {
+            float s = 0.0f;
+            for (int p = 0; p < m; p++) {
+                float v = T[off + p] + (C ? C[off + p] : 0.0f);
+                s += v * v;
+            }
+            acc += wk[k] * s;
+        }
+        off += m;
+    }
+    return acc;
+}
+
+/* Spectral penalty with a scalar DC carry (Q2_K acceptance stages):
+ * Σ_k λ_k‖vᵏ‖² with (vᴸ + dc_carry)² at the top. Reconstruction target is
+ * always the true weight; we never quantize x−bias. */
 static inline float hex_spectral_penalty_ex(const float *e, int n, float dc_carry)
 {
     if (HEX_DC_LAMBDA == 0.0f && HEX_VW_LAMBDA == 0.0f) return 0.0f;
-    float dc = dc_carry, ves = 0.0f;
-    int half = n / 2;
-    for (int i = 0; i < half; i++) {
-        float v = e[i] + e[i + half];
-        dc  += v;
-        ves += v * v;
-    }
-    return (HEX_DC_LAMBDA / (float)n) * dc * dc
-         + (HEX_VW_LAMBDA / (float)n) * ves;
+    float T[QK_K];
+    hex_fold_build(e, n, T);
+    T[n - 2] += dc_carry;
+    return hex_fold_energy(T, NULL, n);
 }
 
 static inline float hex_spectral_penalty(const float *e, int n)
@@ -3851,7 +3960,7 @@ static void quantize_tensor_q2k_hpc(const float *weights, int64_t n_elements,
                     dc_res += x - deq;
                 }
             }
-            rolling_dc = dc_res;
+            rolling_dc = g_hex_carry_cumulative ? block_dc_carry[blk] + dc_res : dc_res;
         }
     }
 
@@ -4946,7 +5055,9 @@ static void quantize_tensor_q2k_hpc(const float *weights, int64_t n_elements,
             float dc_res = 0.0f;
             for (int i = 0; i < QK_K; i++)
                 dc_res += bx[i] - deq[i];
-            rolling_dc = dc_res;
+            /* cumulative: S ← decay·S + Σe, so the metric (Σe + carry)² is
+             * S² and the row residual after each block is just its miss. */
+            rolling_dc = g_hex_carry_cumulative ? carry + dc_res : dc_res;
         }
         total_err = 0.0f;
         for (int64_t blk = 0; blk < n_blocks; blk++)
@@ -5767,38 +5878,111 @@ static float iq2xs_block_at_d(const IQ2Book *bk, const float *x, const float *w,
     return e;
 }
 
+/* Change in fold energy if the 8 leaves of group g move by de[8].
+ * Leaf i lives at node (i & (m-1)) of the level with m nodes; the 8
+ * contiguous leaves of a group hit 8 distinct nodes while m ≥ 8 and fold
+ * onto all m nodes below that. O(8·log n) per evaluation. */
+static inline float iq2_fold_delta(const float *T, const float *C, const float *wk,
+                                   int g, const float de[8])
+{
+    float acc = 0.0f; int off = 0, k = 0;
+    for (int m = QK_K / 2; m >= 1; m >>= 1, k++) {
+        if (wk[k] != 0.0f) {
+            if (m >= 8) {
+                int base = (8 * g) & (m - 1);
+                for (int j = 0; j < 8; j++) {
+                    int q = off + base + j;
+                    float t = T[q] + (C ? C[q] : 0.0f);
+                    acc += wk[k] * de[j] * (2.0f * t + de[j]);
+                }
+            } else {
+                float dn[8] = {0};
+                for (int j = 0; j < 8; j++) dn[j & (m - 1)] += de[j];
+                for (int p = 0; p < m; p++) {
+                    int q = off + p;
+                    float t = T[q] + (C ? C[q] : 0.0f);
+                    acc += wk[k] * dn[p] * (2.0f * t + dn[p]);
+                }
+            }
+        }
+        off += m;
+    }
+    return acc;
+}
+
+static inline void iq2_fold_apply(float *T, int g, const float de[8])
+{
+    int off = 0;
+    for (int m = QK_K / 2; m >= 1; m >>= 1) {
+        if (m >= 8) {
+            int base = (8 * g) & (m - 1);
+            for (int j = 0; j < 8; j++) T[off + base + j] += de[j];
+        } else {
+            for (int j = 0; j < 8; j++) T[off + (j & (m - 1))] += de[j];
+        }
+        off += m;
+    }
+}
+
 /* Greedy re-selection among top-K codewords per group on
- *   SSE + (λ_dc/n)(Σe + carry)² + (λ_vw/n) Σ_p (e_p + e_{p+128})²
- * with a block SSE cap. This is the fold-through-codebook step. */
+ *   SSE + Σ_k λ_k Σ_p (vᵏ[p] + carryᵏ[p])²        (fold pyramid, DC at the top)
+ * with a block SSE cap. This is the fold-through-codebook step. carry is a
+ * pyramid (QK_K-1 floats) of decayed cumulative lane residuals, or NULL. */
+static inline float iq2_lane_delta(const IQ2LaneCtx *lc, const float *Lv, int g, const float de[8])
+{
+    float acc = 0.0f;
+    for (int k = 0; k < lc->r; k++) {
+        const float *u = lc->U + (int64_t)k * lc->stride + 8 * g;
+        float dL = 0.0f;
+        for (int j = 0; j < 8; j++) dL += u[j] * de[j];
+        float t = Lv[k] + (lc->carry ? lc->carry[k] : 0.0f);
+        acc += lc->lambda * lc->ev[k] * dL * (2.0f * t + dL);
+    }
+    return acc;
+}
+
 static void iq2xs_shape_block(const IQ2Book *bk, const float *x, const float *w, float d,
                               const uint8_t ls[IQ2XS_NSUB],
-                              uint32_t code[IQ2XS_NGROUP], float dc_carry)
+                              uint32_t code[IQ2XS_NGROUP], const float *carry,
+                              const IQ2LaneCtx *lc)
 {
-    if (HEX_DC_LAMBDA == 0.0f && HEX_VW_LAMBDA == 0.0f) return;
+    if (HEX_DC_LAMBDA == 0.0f && HEX_VW_LAMBDA == 0.0f && !(lc && lc->r > 0)) return;
+    if (lc && lc->r <= 0) lc = NULL;
 
     IQ2Cand cands[IQ2XS_NGROUP][IQ2XS_TOPK];
     int   ncand[IQ2XS_NGROUP], cur[IQ2XS_NGROUP];
-    float e[QK_K];
-    float sse = 0.0f, dc = dc_carry;
+    float e[QK_K], T[QK_K], wk[HEX_FOLD_LEVELS], Lv[HEX_MAX_LANES];
+    float sse = 0.0f;
 
     for (int g = 0; g < IQ2XS_NGROUP; g++) {
         float db = iq2xs_sub_db(d, ls[g >> 1]);
         if (db <= 0.0f) { ncand[g] = 0; cur[g] = -1;
-            for (int j = 0; j < 8; j++) { e[8*g+j] = x[8*g+j]; sse += w[8*g+j]*x[8*g+j]*x[8*g+j]; dc += e[8*g+j]; }
+            for (int j = 0; j < 8; j++) { e[8*g+j] = x[8*g+j]; sse += w[8*g+j]*x[8*g+j]*x[8*g+j]; }
             continue; }
         ncand[g] = iq2xs_group_candidates(bk, x + 8*g, w + 8*g, db, IQ2XS_TOPK, cands[g]);
         cur[g] = 0;
         for (int c = 0; c < ncand[g]; c++)
             if (cands[g][c].code == code[g]) { cur[g] = c; break; }
         const IQ2Cand *cc = &cands[g][cur[g]];
-        for (int j = 0; j < 8; j++) { e[8*g+j] = x[8*g+j] - cc->deq[j]; dc += e[8*g+j]; }
+        for (int j = 0; j < 8; j++) e[8*g+j] = x[8*g+j] - cc->deq[j];
         sse += cc->sse;
     }
     const float cap = sse * (1.0f + HEX_DC_SSE_BUDGET);
-    float v[QK_K / 2], ves = 0.0f;
-    for (int p = 0; p < QK_K / 2; p++) { v[p] = e[p] + e[p + QK_K/2]; ves += v[p] * v[p]; }
-    const float ldc = HEX_DC_LAMBDA / (float)QK_K, lvw = HEX_VW_LAMBDA / (float)QK_K;
-    float metric = sse + ldc * dc * dc + lvw * ves;
+    hex_fold_weights(QK_K, wk);
+    hex_fold_build(e, QK_K, T);
+    float fold = hex_fold_energy(T, carry, QK_K);
+    float lane = 0.0f;
+    if (lc) {
+        for (int k = 0; k < lc->r; k++) {
+            const float *u = lc->U + (int64_t)k * lc->stride;
+            float s = 0.0f;
+            for (int i = 0; i < QK_K; i++) s += u[i] * e[i];
+            Lv[k] = s;
+            float t = s + (lc->carry ? lc->carry[k] : 0.0f);
+            lane += lc->lambda * lc->ev[k] * t * t;
+        }
+    }
+    float metric = sse + fold + lane;
 
     for (int pass = 0; pass < 96; pass++) {
         int best_g = -1, best_c = 0; float best_m = metric;
@@ -5810,31 +5994,31 @@ static void iq2xs_shape_block(const IQ2Book *bk, const float *x, const float *w,
                 const IQ2Cand *cn = &cands[g][c];
                 float sse2 = sse - co->sse + cn->sse;
                 if (sse2 > cap) continue;
-                float dc2 = dc, ves2 = ves;
-                for (int j = 0; j < 8; j++) {
-                    int i  = 8*g + j;
-                    float de = co->deq[j] - cn->deq[j];     /* e_new − e_old */
-                    dc2 += de;
-                    int p = (i < QK_K/2) ? i : i - QK_K/2;
-                    float vn = v[p] + de;
-                    ves2 += vn * vn - v[p] * v[p];
-                }
-                float m2 = sse2 + ldc * dc2 * dc2 + lvw * ves2;
+                float de[8];
+                for (int j = 0; j < 8; j++) de[j] = co->deq[j] - cn->deq[j];   /* e_new − e_old */
+                float m2 = sse2 + fold + iq2_fold_delta(T, carry, wk, g, de)
+                         + lane + (lc ? iq2_lane_delta(lc, Lv, g, de) : 0.0f);
                 if (m2 < best_m) { best_m = m2; best_g = g; best_c = c; }
             }
         }
         if (best_g < 0) break;
         const IQ2Cand *co = &cands[best_g][cur[best_g]];
         const IQ2Cand *cn = &cands[best_g][best_c];
-        sse += cn->sse - co->sse;
-        for (int j = 0; j < 8; j++) {
-            int i = 8*best_g + j;
-            float de = co->deq[j] - cn->deq[j];
-            e[i] += de; dc += de;
-            int p = (i < QK_K/2) ? i : i - QK_K/2;
-            ves += (v[p] + de) * (v[p] + de) - v[p] * v[p];
-            v[p] += de;
+        float de[8];
+        for (int j = 0; j < 8; j++) de[j] = co->deq[j] - cn->deq[j];
+        fold += iq2_fold_delta(T, carry, wk, best_g, de);
+        iq2_fold_apply(T, best_g, de);
+        if (lc) {
+            lane += iq2_lane_delta(lc, Lv, best_g, de);
+            for (int k = 0; k < lc->r; k++) {
+                const float *u = lc->U + (int64_t)k * lc->stride + 8 * best_g;
+                float dL = 0.0f;
+                for (int j = 0; j < 8; j++) dL += u[j] * de[j];
+                Lv[k] += dL;
+            }
         }
+        for (int j = 0; j < 8; j++) e[8*best_g + j] += de[j];
+        sse += cn->sse - co->sse;
         cur[best_g] = best_c;
         code[best_g] = cn->code;
         metric = best_m;
@@ -5990,7 +6174,7 @@ static void quantize_tensor_iq2_hpc(const IQ2Book *bk,
 
         /* 4. fold/DC shaping among near-equivalent codewords (carry = 0 here;
          *    the sequential pass below applies the true residual carry). */
-        iq2xs_shape_block(bk, x, w, best_d, ls, code, 0.0f);
+        iq2xs_shape_block(bk, x, w, best_d, ls, code, NULL, NULL);
 
         float d_out = best_d;
         if (inflate != 0.0f)
@@ -5998,15 +6182,24 @@ static void quantize_tensor_iq2_hpc(const IQ2Book *bk,
         iq2_pack(bk, output + blk * BB, d_out, ls, code);
     }
 
-    /* 5. sequential TRUE residual carry along each row */
-    if (HEX_DC_LAMBDA > 0.0f && g_hex_dc_decay > 0.0f) {
+    /* 5. sequential TRUE residual carry along each row — one lane per
+     *    pyramid node (255 lanes: 128 vesica pairs … 1 DC). S ← decay·S + vᵏ,
+     *    carry = decay·S, so each lane's row residual is just its last miss. */
+    const int use_lanes = (g_lane_r > 0 && g_lane_U && row_width > 0 && g_lane_cols == row_width);
+    if ((HEX_DC_LAMBDA > 0.0f || HEX_VW_LAMBDA > 0.0f || use_lanes) && g_hex_dc_decay > 0.0f) {
         int64_t bpr = (row_width > 0 && row_width % QK_K == 0) ? row_width / QK_K : 0;
-        float rolling = 0.0f, w[QK_K], deq[QK_K];
+        float S[QK_K], C[QK_K], T[QK_K], e[QK_K], w[QK_K], deq[QK_K];
+        float LS[HEX_MAX_LANES], LC[HEX_MAX_LANES];
         uint8_t ls[IQ2XS_NSUB]; uint32_t code[IQ2XS_NGROUP];
+        IQ2LaneCtx lc = { NULL, g_lane_cols, g_lane_ev, use_lanes ? g_lane_r : 0, LC, g_lane_lambda };
+        memset(S, 0, sizeof(S)); memset(LS, 0, sizeof(LS));
         for (int64_t blk = 0; blk < n_blocks; blk++) {
-            if (bpr > 0 && (blk % bpr) == 0) rolling = 0.0f;
+            if (bpr > 0 && (blk % bpr) == 0) { memset(S, 0, sizeof(S)); memset(LS, 0, sizeof(LS)); }
             const float *x = weights + blk * QK_K;
             uint8_t *b = output + blk * BB;
+            for (int q = 0; q < QK_K - 1; q++) C[q] = g_hex_dc_decay * S[q];
+            for (int k = 0; k < lc.r; k++) LC[k] = g_hex_dc_decay * LS[k];
+            if (use_lanes) lc.U = g_lane_U + (bpr > 0 ? (blk % bpr) * QK_K : 0);
             float d = iq2_unpack(bk, b, ls, code);
             if (d > 0.0f) {
                 float sigma2 = 0.0f;
@@ -6016,13 +6209,20 @@ static void quantize_tensor_iq2_hpc(const IQ2Book *bk,
                     float base = imat_importance ? imat_importance[blk * QK_K + i] : 1.0f;
                     w[i] = (wmode == 1) ? base * sqrtf(sigma2 + x[i] * x[i]) : base;
                 }
-                iq2xs_shape_block(bk, x, w, d, ls, code, g_hex_dc_decay * rolling);
+                iq2xs_shape_block(bk, x, w, d, ls, code, C, use_lanes ? &lc : NULL);
                 iq2_pack(bk, b, d, ls, code);
             }
             iq2_dequant_block(bk, b, deq);
-            float r = 0.0f;
-            for (int i = 0; i < QK_K; i++) r += x[i] - deq[i];
-            rolling = r;
+            for (int i = 0; i < QK_K; i++) e[i] = x[i] - deq[i];
+            hex_fold_build(e, QK_K, T);
+            for (int q = 0; q < QK_K - 1; q++)
+                S[q] = g_hex_carry_cumulative ? C[q] + T[q] : T[q];
+            for (int k = 0; k < lc.r; k++) {
+                const float *u = lc.U + (int64_t)k * lc.stride;
+                float s = 0.0f;
+                for (int i = 0; i < QK_K; i++) s += u[i] * e[i];
+                LS[k] = g_hex_carry_cumulative ? LC[k] + s : s;
+            }
         }
     }
 

hexstate_requantize.py

@@ -155,6 +155,20 @@ def _load_hexstate_lib():
             budget = '0.02'
         if budget is not None and hasattr(lib, 'hexstate_set_sse_budget'):
             lib.hexstate_set_sse_budget(ctypes.c_float(float(budget)))
+        # Fold pyramid: HEX_FOLD_DEPTH levels below DC get vesica weight
+        # (1 = classic single-fold vesica [default], 7 = full tree), geometric
+        # per-level weight HEX_FOLD_GAMMA. HEX_CARRY_CUM=0 restores the legacy
+        # previous-block-only carry (default carries the cumulative residual).
+        fd = os.environ.get('HEX_FOLD_DEPTH')
+        fg = os.environ.get('HEX_FOLD_GAMMA')
+        fc = os.environ.get('HEX_CARRY_CUM')
+        if hasattr(lib, 'hexstate_set_fold_params') and (fd or fg or fc):
+            lib.hexstate_set_fold_params.restype = None
+            lib.hexstate_set_fold_params.argtypes = [ctypes.c_int, ctypes.c_float, ctypes.c_int]
+            lib.hexstate_set_fold_params(
+                ctypes.c_int(int(fd) if fd else -1),
+                ctypes.c_float(float(fg) if fg else 0.0),
+                ctypes.c_int(int(fc) if fc else -1))
         _HEXSTATE_LIB = lib
         return lib
     except Exception as e:
@@ -852,9 +866,14 @@ def _copy_bytes(fin, fout, abs_offset, n_bytes):
     return written
 
 
-def quantize_tensor_iq2_hpc(f32_data, importance=None, row_width=0, kind='iq2xs'):
+def quantize_tensor_iq2_hpc(f32_data, importance=None, row_width=0, kind='iq2xs', lanes=None):
     """IQ2_XS (74 B) or IQ2_S (82 B) per 256 weights via the HPC C library.
-    Returns (bytes, n_blocks, dequant_f32)."""
+    Returns (bytes, n_blocks, dequant_f32).
+
+    lanes: optional (U, ev[, lambda]) — U is (r, row_width) activation
+    directions over the input columns (top eigenvectors of E[a aᵀ]) and ev
+    their eigenvalues in E[a²] units; the sequential pass then also cancels
+    the row error along each u_k with a cumulative carry."""
     suffix = {'iq2xs': 'iq2_xs', 'iq2s': 'iq2_s'}[kind]
     block_bytes = LOWBIT_KINDS[kind][1]
     lib = _load_hexstate_lib()
@@ -872,6 +891,20 @@ def quantize_tensor_iq2_hpc(f32_data, importance=None, row_width=0, kind='iq2xs'
         imat_c = np.ascontiguousarray(importance, dtype=np.float32).reshape(-1)
         if imat_c.size == n:
             imat_ptr = imat_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
+    lane_keep = None
+    if lanes is not None and hasattr(lib, 'hexstate_set_activation_lanes') and row_width > 0:
+        U = np.ascontiguousarray(lanes[0], dtype=np.float32).reshape(-1, int(row_width))
+        ev = np.ascontiguousarray(lanes[1], dtype=np.float32).reshape(-1)
+        lam = float(lanes[2]) if len(lanes) > 2 else 1.0
+        lib.hexstate_set_activation_lanes.restype = None
+        lib.hexstate_set_activation_lanes.argtypes = [
+            ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_float),
+            ctypes.c_int, ctypes.c_int64, ctypes.c_float]
+        lib.hexstate_set_activation_lanes(
+            U.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
+            ev.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
+            ctypes.c_int(int(min(U.shape[0], ev.size))), ctypes.c_int64(int(row_width)), ctypes.c_float(lam))
+        lane_keep = (U, ev)
     getattr(lib, f'hexstate_quantize_tensor_{suffix}_hpc')(
         f32.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
         ctypes.c_int64(n),
@@ -881,6 +914,9 @@ def quantize_tensor_iq2_hpc(f32_data, importance=None, row_width=0, kind='iq2xs'
         ctypes.c_int(0),
         ctypes.c_int64(int(row_width)),
     )
+    if lane_keep is not None:
+        lib.hexstate_set_activation_lanes(None, None, ctypes.c_int(0), ctypes.c_int64(0), ctypes.c_float(0.0))
+        del lane_keep
     deq = np.zeros(n, dtype=np.float32)
     getattr(lib, f'hexstate_dequant_{suffix}')(
         out.ctypes.data_as(ctypes.c_void_p), ctypes.c_int64(n_blocks),
@@ -1161,6 +1197,9 @@ def main():
         print("  HEX_VW_LAMBDA    vesica weight (default 1)")
         print("  HEX_DC_DECAY     rolling residual carry 0..1 (default 0.85)")
         print("  HEX_SSE_BUDGET   relative SSE the shaper may spend (Q2_K 5e-4, IQ2 2e-2)")
+        print("  HEX_FOLD_DEPTH   fold-pyramid levels below DC to shape, 1=vesica only (default 1, 7=full tree)")
+        print("  HEX_FOLD_GAMMA   per-level geometric weight of the pyramid (default 2)")
+        print("  HEX_CARRY_CUM    1: carry cumulative row residual (default), 0: previous block only")
         sys.exit(1)
 
     global LOWBIT_FORMAT