Super small quants

hexstate_requantize.py

@@ -643,6 +643,26 @@ def is_attention_tensor(name):
     return False
 
 
+def is_q4_tensor(name):
+    """Tensors demoted to Q4_0 (HPC sieve) while Q2_K tensors stay Q2_K.
+
+    The full-precision leftovers: LM head / embedding tables plus
+    SSM-side weights (ssm_alpha mirrors ssm_beta, which already ships
+    Q2_K; ssm_conv1d kernels are small dense 2D projections).
+    Attention/FFN are NOT matched here — they keep their Q2_K plan.
+    NOTE: 'output.weight' is exact-matched so 'attn_output.weight'
+    never matches.
+    """
+    if name == 'output.weight' or name.endswith('.output.weight'):
+        return True
+    if name == 'token_embd.weight' or '.token_embd.weight' in name:
+        return True
+    for pat in ('ssm_alpha.weight', 'ssm_conv1d.weight', 'conv1d.weight'):
+        if pat in name:
+            return True
+    return False
+
+
 def q2k_row_compatible(n_dims, dims):
     return n_dims >= 2 and bool(dims) and int(dims[0]) % QK_K == 0
 
@@ -683,12 +703,11 @@ def should_quantize(name, n_dims, dims, tied_embeddings=False):
       - ffn_gate_inp — MoE routing gate
       - layer_output_scale — per-layer scaling factor (scalar)
       - altup, laurel — small Gemma-specific tensors
-      - token_embd.weight / output.weight — always excluded here.
-        When embeddings are TIED, main() routes token_embd.weight to
-        Q8_0 (HPC sieve pipeline) instead: the same tensor serves as both
-        embedding lookup AND LM head, and Q2_K/Q4_0 there destroys logit
-        precision → looping / repetitive generation. --keep-embd keeps
-        it at source precision instead.
+    - token_embd.weight / output.weight — demoted to Q4_0 in main()
+      (is_q4_tensor). When embeddings are TIED, main() routes
+      token_embd.weight to Q8_0 (HPC sieve pipeline) instead: the same
+      tensor serves as both embedding lookup AND LM head. --keep-embd
+      keeps it at source precision instead.
     """
     n_elements = 1
     for d in dims:
@@ -705,20 +724,25 @@ def should_quantize(name, n_dims, dims, tied_embeddings=False):
         return False
     if 'layer_output_scale' in name:
         return False
-    # Embedding table — this is a lookup, not a matmul; Q2_K destroys
-    # token distinctions. Keep at source precision (F16/BF16).
-    if 'token_embd' in name:
-        return False
-    # LM head output projection — logit precision is critical for generation.
-    # (When tied with embeddings, this is the same tensor and also skipped above.)
-    if name == 'output.weight':
-        return False
-    # DeltaNet state-space parameters — keep at full precision
-    if 'ssm_a' in name or 'A_log' in name:
+    # Embedding table / LM head — demoted to Q4_0 in main()
+    # (is_q4_tensor), NOT kept at source precision. (When embeddings are
+    # TIED, main() routes token_embd.weight to Q8_0 instead — that branch
+    # runs before should_quantize is consulted.)
+    # DeltaNet state-space parameters — keep truly scalar/vector ones
+    # at full precision. NOTE: 'ssm_a' must match the standalone 1D tensor
+    # only ('blk.N.ssm_a'), NOT 'ssm_alpha' (contains 'ssm_a' as a
+    # substring). ssm_alpha is a regular 2D projection weight and is
+    # quantizable — it mirrors ssm_beta, which already ships Q2_K.
+    if 'ssm_a' in name.split('.') or 'A_log' in name:
         return False
     if 'ssm_dt' in name or 'dt_bias' in name:
         return False
-    if 'ssm_conv1d' in name or 'conv1d.weight' in name:
+    # ssm_conv1d kernels are quantizable ONLY when their row width is a
+    # multiple of the quant block size: GGUF quant blocks tile along dim0,
+    # so e.g. dims [4, 10240] can never be Q4_0/Q8_0/Q2_K regardless of
+    # total-element divisibility (llama.cpp rejects such files at load).
+    if ('ssm_conv1d' in name or 'conv1d.weight' in name) \
+            and (not dims or dims[0] % 32 != 0):
         return False
     # When embeddings are tied, token_embd.weight doubles as the output
     # projection (LM head). It gets routed to Q4_0 in the quant plan
@@ -881,8 +905,11 @@ def main():
                     else:
                         will_quant = False; total_keep += 1
                         print(f"  Q2ALL→KEEP (row width not block-aligned): {ti['name']}")
-                elif is_attention_tensor(ti['name']) and q4_row_compatible(ti['n_dims'], ti['dims']):
-                    will_quant = 'ATTN_Q4'; total_attn += 1
+                elif is_q4_tensor(ti['name']) and q4_row_compatible(ti['n_dims'], ti['dims']):
+                    # Q4_0 path (HPC sieve). dim0 — not just total n — must be
+                    # block-aligned: GGUF quants tile along rows and llama.cpp
+                    # refuses to load otherwise.
+                    will_quant = 'Q4_HPC'; total_attn += 1
                 elif q2k_row_compatible(ti['n_dims'], ti['dims']):
                     will_quant = True; total_quant += 1
                 elif q4_row_compatible(ti['n_dims'], ti['dims']):
@@ -920,12 +947,12 @@ def main():
                     n_blocks = ti['n_elements'] // 32
                     out_size = n_blocks * 34
                     print(f"  [EMBD→Q8_0·Sieve] {ti['name']} ({ti['n_elements']:,} elements)")
-                elif quant_plan[i] == 'ATTN_Q4':
+                elif quant_plan[i] == 'Q4_HPC':
                     # Attention tensor → Q4_0 HPC (4.5 bpw)
                     out_type = GGML_TYPE_Q4_0
                     n_blocks = (ti['n_elements'] + 31) // 32
                     out_size = n_blocks * 18
-                    print(f"  [ATTN→Q4_0·HPC] {ti['name']} ({ti['n_elements']} elements)")
+                    print(f"  [Q4_0·HPC] {ti['name']} ({ti['n_elements']} elements)")
                 elif quant_plan[i] == 'Q4_0':
                     out_type = GGML_TYPE_Q4_0
                     n_blocks = ti['n_elements'] // 32
@@ -1151,7 +1178,7 @@ def main():
                     quant_count += 1
                     total_quant_bytes += n_blocks_q8 * 34
 
-                elif quant_plan[i] in ('Q4_0', 'ATTN_Q4'):
+                elif quant_plan[i] in ('Q4_0', 'Q4_HPC'):
                     # ── Q4_0 quantization (fallback or attention HPC) ──
                     if ti['type'] == GGML_TYPE_BF16:
                         f32 = bf16_to_f32(raw_data, ti['n_elements'])
@@ -1171,7 +1198,7 @@ def main():
                     n_blocks_q4 = n_el // 32
 
                     # Use HPC for attention tensors if available
-                    if quant_plan[i] == 'ATTN_Q4' and use_hpc and hasattr(_HEXSTATE_LIB, 'hexstate_quantize_tensor_q4_0_hpc'):
+                    if quant_plan[i] == 'Q4_HPC' and use_hpc and hasattr(_HEXSTATE_LIB, 'hexstate_quantize_tensor_q4_0_hpc'):
                         output_buf = np.zeros(n_blocks_q4 * 18, dtype=np.uint8)
                         error = ctypes.c_float(0.0)
                         f32_c = np.ascontiguousarray(f32, dtype=np.float32)