Upload 3 files

gguf.py

@@ -436,6 +436,42 @@ def _is_embd_or_head(name: str) -> bool:
     return "token_embd" in name or name == "output.weight" or name.endswith(".output.weight")
 
 
+def _is_attention_projection(name: str) -> bool:
+    """True for attention and recurrent/SSM projections (Q/K/V/O, QKV, gates, SSM state)."""
+    if "norm" in name or ".bias" in name:
+        return False
+    return any(
+        tok in name
+        for tok in (
+            ".attn_",
+            ".ssm_",
+            "self_attn.",
+            "linear_attn.",
+            "attention.w",
+            "attn.q",
+            "attn.k",
+            "attn.v",
+            "attn.o",
+            "attn.wq",
+            "attn.wk",
+            "attn.wv",
+            "attn.wo",
+        )
+    )
+
+
+def _is_tiny_ssm_gate(name: str, dims: tuple[int, ...]) -> bool:
+    """True for low-rank recurrent step-size/decay gates (e.g. ssm_alpha, ssm_beta, ssm_f_a/b)."""
+    if any(
+        k in name
+        for k in ("ssm_alpha", "ssm_beta", "ssm_f_a", "ssm_f_b", "ssm_g_a", "ssm_g_b")
+    ):
+        return True
+    if ".ssm_" in name and len(dims) >= 2 and min(int(dims[0]), int(dims[1])) <= 128:
+        return True
+    return False
+
+
 def is_q2k_eligible(name: str, dims: tuple[int, ...]) -> bool:
     if _is_skipped_metadata_tensor(name, dims):
         return False
@@ -500,11 +536,13 @@ def target_quant_type(
     quant_format: str = "mixed",
     source_type: int | None = None,
 ) -> str | None:
-    """Determine target quantization format ('Q2_K', 'Q4_0', or None for Copy).
+    """Determine target quantization format ('Q1_0', 'Q2_K', 'Q4_0', or None for Copy).
 
     Modes:
     * 'mixed' (default): all Q2_K-eligible tensors -> Q2_K; non-Q2_K 32-aligned
       tensors (and embeddings/head if quantize_embd_q4) -> Q4_0.
+    * 'mixed_q1': FFN/expert 128-aligned tensors -> Q1_0; attention/SSM projections,
+      embeddings/head, and non-128-aligned 32-aligned tensors -> Q4_0.
     * 'q4_0': all 32-aligned eligible weight tensors -> Q4_0.
     * 'q2_k': only Q2_K-eligible tensors -> Q2_K; everything else copied.
     * 'q1_0': all 128-aligned eligible weight tensors -> Q1_0.
@@ -514,15 +552,19 @@ def target_quant_type(
     """
     tgt: str | None = None
     if quant_format == "q1_0":
-        if is_q1_0_eligible(name, dims, include_embd=quantize_embd_q4):
+        if promote_attn_q4 and _is_attention_projection(name) and is_q4_0_eligible(
+            name, dims, include_embd=quantize_embd_q4, promote_attn=True, allow_q2k_tensors=True
+        ):
+            tgt = "Q4_0"
+        elif is_q1_0_eligible(name, dims, include_embd=quantize_embd_q4):
             tgt = "Q1_0"
         elif enable_q4_fallback and is_q4_0_eligible(
-            name, dims, include_embd=quantize_embd_q4, promote_attn=False
+            name, dims, include_embd=quantize_embd_q4, promote_attn=True, allow_q2k_tensors=True
         ):
             tgt = "Q4_0"
     elif quant_format == "mixed_q1":
-        if promote_attn_q4 and _is_attention_projection(name) and is_q4_0_eligible(
-            name, dims, include_embd=quantize_embd_q4, promote_attn=True
+        if (_is_attention_projection(name) or _is_tiny_ssm_gate(name, dims)) and is_q4_0_eligible(
+            name, dims, include_embd=quantize_embd_q4, promote_attn=True, allow_q2k_tensors=True
         ):
             tgt = "Q4_0"
         elif _is_embd_or_head(name):
@@ -531,7 +573,7 @@ def target_quant_type(
         elif is_q1_0_eligible(name, dims, include_embd=False):
             tgt = "Q1_0"
         elif enable_q4_fallback and is_q4_0_eligible(
-            name, dims, include_embd=quantize_embd_q4, promote_attn=False
+            name, dims, include_embd=quantize_embd_q4, promote_attn=True, allow_q2k_tensors=True
         ):
             tgt = "Q4_0"
     elif quant_format == "q4_0":