Upload 6 files

gguf.py

@@ -556,7 +556,11 @@ def target_quant_type(
             name, dims, include_embd=quantize_embd_q4, promote_attn=True, allow_q2k_tensors=True
         ):
             tgt = "Q4_0"
-        elif is_q1_0_eligible(name, dims, include_embd=quantize_embd_q4):
+        elif _is_embd_or_head(name) and quantize_embd_q4 and enable_q4_fallback and is_q4_0_eligible(
+            name, dims, include_embd=True, allow_q2k_tensors=True
+        ):
+            tgt = "Q4_0"
+        elif is_q1_0_eligible(name, dims, include_embd=not quantize_embd_q4):
             tgt = "Q1_0"
         elif enable_q4_fallback and is_q4_0_eligible(
             name, dims, include_embd=quantize_embd_q4, promote_attn=True, allow_q2k_tensors=True

q1_0.py

@@ -152,42 +152,43 @@ def _quantize_q1_0_hpc_cpu(
     lam_vw = (float(cfg.vw_lambda) / QK1_0) if cfg.use_spectral else 0.0
     half = QK1_0 // 2
 
-    # 1. Analog + WLS seed for d* (testing mean, weighted WLS, and companded analog):
+    # 1. Analog + WLS seed for d* (bounded around d_mean so outlier imatrix channels cannot collapse/explode d):
     d_mean = np.mean(np.abs(X_f32), axis=1).astype(np.float32)
+    d_lo = (0.80 * d_mean).astype(np.float32)
+    d_hi = (1.25 * d_mean).astype(np.float32)
     wsum = np.maximum(np.sum(W_f32, axis=1, keepdims=True), 1.0e-30)
-    d_wls_init = (np.sum(W_f32 * np.abs(X_f32), axis=1, keepdims=True) / wsum)[:, 0].astype(np.float32)
+    d_wls_init = np.clip(
+        (np.sum(W_f32 * np.abs(X_f32), axis=1, keepdims=True) / wsum)[:, 0].astype(np.float32),
+        d_lo,
+        d_hi,
+    )
     mu = np.sum(W_f32 * X_f32, axis=1, keepdims=True) / wsum
     var = np.sum(W_f32 * (X_f32 - mu) ** 2, axis=1, keepdims=True) / wsum
     sig = np.sqrt(np.maximum(var, 1.0e-30))
     x_comp = mu + sig * 1.85 * np.tanh((X_f32 - mu) / (1.85 * sig))
-    d_analog = (np.sum(W_f32 * np.abs(x_comp), axis=1, keepdims=True) / wsum)[:, 0].astype(np.float32)
+    d_analog = np.clip(
+        (np.sum(W_f32 * np.abs(x_comp), axis=1, keepdims=True) / wsum)[:, 0].astype(np.float32),
+        d_lo,
+        d_hi,
+    )
 
+    q_sign = (X_f32 >= 0.0).astype(np.float32)
+    c_sign = 2.0 * q_sign - 1.0
     wls_d = d_mean.copy()
     best_seed_sse = np.full(B, np.inf, dtype=np.float64)
-    for d_start in (d_mean, d_wls_init, d_analog, -d_analog):
-        d = d_start.astype(np.float64)
-        for _ in range(2):
-            q = np.where(d[:, None] >= 0.0, X_f32 >= 0.0, X_f32 < 0.0).astype(np.int64)
-            c = (2 * q - 1).astype(np.float64)
-            num = np.sum(W_f32 * X_f32 * c, axis=1, dtype=np.float64)
-            den = np.sum(W_f32, axis=1, dtype=np.float64)
-            valid = den > 1.0e-30
-            d = np.where(valid, num / np.where(valid, den, 1.0), 0.0)
-        df = from_fp16_bits(to_fp16_bits(d.astype(np.float32)))
-        q = np.where(df[:, None] >= 0.0, X_f32 >= 0.0, X_f32 < 0.0).astype(np.float32)
-        c_f = 2.0 * q - 1.0
-        e = X_f32 - df[:, None] * c_f
+    for d_start in (d_mean, d_wls_init, d_analog):
+        df = from_fp16_bits(to_fp16_bits(np.clip(d_start, d_lo, d_hi).astype(np.float32)))
+        e = X_f32 - df[:, None] * c_sign
         sse = np.sum(W_f32 * e * e, axis=1, dtype=np.float64)
         take = sse < best_seed_sse
         best_seed_sse = np.where(take, sse, best_seed_sse)
         wls_d = np.where(take, df, wls_d)
 
-    # 2. 24-Candidate Field with Extended Spectral Objective:
-    cand_d16 = to_fp16_bits(wls_d[:, None] * MULTS_D[None, :])  # (B, 24)
+    # 2. 24-Candidate Field with Extended Spectral Objective (clamped to [0.80*d_mean, 1.25*d_mean]):
+    cand_raw = np.clip(wls_d[:, None] * MULTS_D[None, :], d_lo[:, None], d_hi[:, None]).astype(np.float32)
+    cand_d16 = to_fp16_bits(cand_raw)                           # (B, 24)
     cand_d = from_fp16_bits(cand_d16)                           # (B, 24)
-    q_c = np.where(cand_d[:, :, None] >= 0.0, X_f32[:, None, :] >= 0.0, X_f32[:, None, :] < 0.0).astype(np.float32)
-    c_c = 2.0 * q_c - 1.0
-    e_c = X_f32[:, None, :] - cand_d[:, :, None] * c_c          # (B, 24, 128)
+    e_c = X_f32[:, None, :] - cand_d[:, :, None] * c_sign[:, None, :]  # (B, 24, 128)
     sse_c = np.sum(W_f32[:, None, :] * e_c * e_c, axis=2)
     if cfg.use_spectral and (lam_dc > 0.0 or lam_vw > 0.0):
         v_c = e_c[:, :, :half] + e_c[:, :, half:]
@@ -201,19 +202,17 @@ def _quantize_q1_0_hpc_cpu(
     rows = np.arange(B)
     d_sel = cand_d[rows, best_cand]
 
-    # 4. Asymmetric Sign-Flip fp16-ULP Polish (+/-8 ULPs around +d_sel, -d_sel, wls_d, and d_mean):
+    # 4. FP16-ULP Polish (+/-rad ULPs around d_sel, wls_d, and d_mean within [d_lo, d_hi]):
     rad = max(int(cfg.ulp_radius), 4)
     deltas = np.arange(-rad, rad + 1, dtype=np.int32)
-    pos_b = np.clip(to_fp16_bits(np.abs(d_sel)).astype(np.int32)[:, None] + deltas[None, :], 0, 0x7BFF).astype(np.uint16)
-    neg_b = (pos_b | np.uint16(0x8000)).astype(np.uint16)
-    wls_b = np.clip(to_fp16_bits(np.abs(wls_d)).astype(np.int32)[:, None] + deltas[None, :], 0, 0x7BFF).astype(np.uint16)
-    wls_neg_b = (wls_b | np.uint16(0x8000)).astype(np.uint16)
-    ref_b = np.clip(to_fp16_bits(np.abs(d_mean)).astype(np.int32)[:, None] + deltas[None, :], 0, 0x7BFF).astype(np.uint16)
-    all_b = np.concatenate([pos_b, neg_b, wls_b, wls_neg_b, ref_b], axis=1)
+    lo_b = to_fp16_bits(d_lo).astype(np.int32)[:, None]
+    hi_b = np.maximum(to_fp16_bits(d_hi).astype(np.int32)[:, None], lo_b)
+    sel_b = np.clip(to_fp16_bits(np.abs(d_sel)).astype(np.int32)[:, None] + deltas[None, :], lo_b, hi_b).astype(np.uint16)
+    wls_b = np.clip(to_fp16_bits(np.abs(wls_d)).astype(np.int32)[:, None] + deltas[None, :], lo_b, hi_b).astype(np.uint16)
+    ref_b = np.clip(to_fp16_bits(np.abs(d_mean)).astype(np.int32)[:, None] + deltas[None, :], lo_b, hi_b).astype(np.uint16)
+    all_b = np.concatenate([sel_b, wls_b, ref_b], axis=1)
     all_d = from_fp16_bits(all_b)
-    q_u = np.where(all_d[:, :, None] >= 0.0, X_f32[:, None, :] >= 0.0, X_f32[:, None, :] < 0.0).astype(np.uint8)
-    c_u = 2.0 * q_u.astype(np.float32) - 1.0
-    e_u = X_f32[:, None, :] - all_d[:, :, None] * c_u
+    e_u = X_f32[:, None, :] - all_d[:, :, None] * c_sign[:, None, :]
     sse_u = np.sum(W_f32[:, None, :] * e_u * e_u, axis=2, dtype=np.float64)
     if cfg.use_spectral and (lam_dc > 0.0 or lam_vw > 0.0):
         v_u = e_u[:, :, :half] + e_u[:, :, half:]
@@ -226,9 +225,9 @@ def _quantize_q1_0_hpc_cpu(
         pick_u = np.argmin(sse_u, axis=1)
     best_d16 = all_b[rows, pick_u]
     best_d = all_d[rows, pick_u]
-    best_q = q_u[rows, pick_u].astype(np.int64)
+    best_q = (X_f32 >= 0.0).astype(np.int64)
 
-    # 5. Closed-Form 1-D Quadratic Vesica + DC Bit-Flip Projection (step = 2*d) within sse_budget:
+    # 5. Boundary-Guarded 1-D Quadratic Vesica + DC Bit-Flip Projection (only near-zero |x| <= 0.10*d):
     if cfg.use_spectral and (lam_vw > 0.0 or lam_dc > 0.0):
         c_f = (2 * best_q - 1).astype(np.float64)
         d_f64 = best_d.astype(np.float64)[:, None]
@@ -239,7 +238,8 @@ def _quantize_q1_0_hpc_cpu(
         v = e[:, :half] + e[:, half:]
         dc = np.sum(e, axis=1)
         denom = (W_f32 + lam_vw + lam_dc) * (step_f64 * step_f64)
-        act_b = np.abs(d_f64) > 1.0e-15
+        near_zero = np.abs(X_f32) <= 0.10 * d_f64
+        act_b = (d_f64 > 1.0e-15) & near_zero
         s_den = np.where(act_b & (denom > 1.0e-30), denom, 1.0)
         v_full = np.concatenate([v, v], axis=1)
         grad = step_f64 * (W_f32 * e + lam_vw * v_full + lam_dc * dc[:, None])
@@ -278,42 +278,42 @@ def _quantize_q1_0_hpc_cuda(
         X = torch.from_numpy(X_np).to(dev, dtype=torch.float32)
         W = torch.from_numpy(W_np).to(dev, dtype=torch.float32)
 
-        # 1. Analog + WLS seed (including negative-scale sign-flip starts):
+        # 1. Analog + WLS seed (bounded around d_mean so outlier imatrix channels cannot collapse/explode d):
         d_mean = torch.mean(torch.abs(X), dim=1)
+        d_lo = 0.80 * d_mean
+        d_hi = 1.25 * d_mean
         wsum = torch.clamp(torch.sum(W, dim=1, keepdim=True), min=1.0e-30)
-        d_wls_init = (torch.sum(W * torch.abs(X), dim=1, keepdim=True) / wsum).squeeze(1)
+        d_wls_init = torch.clamp(
+            (torch.sum(W * torch.abs(X), dim=1, keepdim=True) / wsum).squeeze(1),
+            min=d_lo,
+            max=d_hi,
+        )
         mu = torch.sum(W * X, dim=1, keepdim=True) / wsum
         var = torch.sum(W * (X - mu) ** 2, dim=1, keepdim=True) / wsum
         sig = torch.sqrt(torch.clamp(var, min=1.0e-30))
         x_comp = mu + sig * 1.85 * torch.tanh((X - mu) / (1.85 * sig))
-        d_analog = (torch.sum(W * torch.abs(x_comp), dim=1, keepdim=True) / wsum).squeeze(1)
+        d_analog = torch.clamp(
+            (torch.sum(W * torch.abs(x_comp), dim=1, keepdim=True) / wsum).squeeze(1),
+            min=d_lo,
+            max=d_hi,
+        )
 
+        c_sign = torch.where(X >= 0.0, 1.0, -1.0)
         wls_d = d_mean.clone()
         best_seed_sse = torch.full((B,), float("inf"), dtype=torch.float32, device=dev)
-        for d_start in (d_mean, d_wls_init, d_analog, -d_analog):
-            d = d_start.clone()
-            for _ in range(2):
-                q = torch.where(d[:, None] >= 0.0, X >= 0.0, X < 0.0)
-                c = torch.where(q, 1.0, -1.0)
-                num = torch.sum(W * X * c, dim=1)
-                den = torch.sum(W, dim=1)
-                valid = den > 1.0e-30
-                d = torch.where(valid, num / torch.where(valid, den, torch.ones_like(den)), torch.zeros_like(den))
-            df = d.to(torch.float16).to(torch.float32)
-            q = torch.where(df[:, None] >= 0.0, X >= 0.0, X < 0.0)
-            c = torch.where(q, 1.0, -1.0)
-            e = X - df[:, None] * c
+        for d_start in (d_mean, d_wls_init, d_analog):
+            df = torch.clamp(d_start, min=d_lo, max=d_hi).to(torch.float16).to(torch.float32)
+            e = X - df[:, None] * c_sign
             sse = torch.sum(W * e * e, dim=1)
             take = sse < best_seed_sse
             best_seed_sse = torch.where(take, sse, best_seed_sse)
             wls_d = torch.where(take, df, wls_d)
 
-        # 2. 24-Candidate Field with Extended Spectral Objective on GPU:
+        # 2. 24-Candidate Field with Extended Spectral Objective on GPU (clamped to [0.80*d_mean, 1.25*d_mean]):
         mults = torch.tensor(MULTS_D, dtype=torch.float32, device=dev)
-        cand_d = (wls_d[:, None] * mults[None, :]).to(torch.float16).to(torch.float32)  # (B, 24)
-        q_c = torch.where(cand_d[:, :, None] >= 0.0, X[:, None, :] >= 0.0, X[:, None, :] < 0.0)
-        c_c = torch.where(q_c, 1.0, -1.0)
-        e_c = X[:, None, :] - cand_d[:, :, None] * c_c
+        cand_raw = torch.clamp(wls_d[:, None] * mults[None, :], min=d_lo[:, None], max=d_hi[:, None])
+        cand_d = cand_raw.to(torch.float16).to(torch.float32)  # (B, 24)
+        e_c = X[:, None, :] - cand_d[:, :, None] * c_sign[:, None, :]
         sse_c = torch.sum(W[:, None, :] * e_c * e_c, dim=2)
         if cfg.use_spectral and (lam_dc > 0.0 or lam_vw > 0.0):
             v_c = e_c[:, :, :half] + e_c[:, :, half:]
@@ -328,22 +328,20 @@ def _quantize_q1_0_hpc_cuda(
         rows = torch.arange(B, device=dev)
         d_sel = cand_d[rows, best_cand_t]
 
-        # 4. Asymmetric Sign-Flip fp16-ULP Polish (+/-8 ULPs around +d_sel, -d_sel, wls_d, and d_mean):
+        # 4. FP16-ULP Polish (+/-rad ULPs around d_sel, wls_d, and d_mean within [d_lo, d_hi]):
         rad = max(int(cfg.ulp_radius), 4)
         deltas = torch.arange(-rad, rad + 1, dtype=torch.int32, device=dev)
+        lo_bits = (d_lo.to(torch.float16).view(torch.int16).to(torch.int32) & 0x7FFF)[:, None]
+        hi_bits = torch.maximum((d_hi.to(torch.float16).view(torch.int16).to(torch.int32) & 0x7FFF)[:, None], lo_bits)
         abs_bits = torch.abs(d_sel).to(torch.float16).view(torch.int16).to(torch.int32) & 0x7FFF
         wls_bits = torch.abs(wls_d).to(torch.float16).view(torch.int16).to(torch.int32) & 0x7FFF
         ref_bits = torch.abs(d_mean).to(torch.float16).view(torch.int16).to(torch.int32) & 0x7FFF
-        pos_b = torch.clamp(abs_bits[:, None] + deltas[None, :], 0, 0x7BFF)
-        neg_b = pos_b | 0x8000
-        wls_b = torch.clamp(wls_bits[:, None] + deltas[None, :], 0, 0x7BFF)
-        wls_neg_b = wls_b | 0x8000
-        mean_b = torch.clamp(ref_bits[:, None] + deltas[None, :], 0, 0x7BFF)
-        all_b = torch.cat([pos_b, neg_b, wls_b, wls_neg_b, mean_b], dim=1)
+        pos_b = torch.clamp(abs_bits[:, None] + deltas[None, :], lo_bits, hi_bits)
+        wls_b = torch.clamp(wls_bits[:, None] + deltas[None, :], lo_bits, hi_bits)
+        mean_b = torch.clamp(ref_bits[:, None] + deltas[None, :], lo_bits, hi_bits)
+        all_b = torch.cat([pos_b, wls_b, mean_b], dim=1)
         all_d = all_b.to(torch.int16).view(torch.float16).to(torch.float32)
-        q_u = torch.where(all_d[:, :, None] >= 0.0, X[:, None, :] >= 0.0, X[:, None, :] < 0.0).to(torch.uint8)
-        c_u = 2.0 * q_u.to(torch.float32) - 1.0
-        e_u = X[:, None, :] - all_d[:, :, None] * c_u
+        e_u = X[:, None, :] - all_d[:, :, None] * c_sign[:, None, :]
         sse_u = torch.sum(W[:, None, :] * e_u * e_u, dim=2)
         if cfg.use_spectral and (lam_dc > 0.0 or lam_vw > 0.0):
             v_u = e_u[:, :, :half] + e_u[:, :, half:]
@@ -356,9 +354,9 @@ def _quantize_q1_0_hpc_cuda(
             pick_u = torch.argmin(sse_u, dim=1)
         best_d16 = all_b[rows, pick_u]
         best_d = all_d[rows, pick_u]
-        best_q = q_u[rows, pick_u].to(torch.int64)
+        best_q = (X >= 0.0).to(torch.int64)
 
-        # 5. Closed-Form 1-D Quadratic Vesica + DC Bit-Flip Projection (step = 2*d) within sse_budget:
+        # 5. Boundary-Guarded 1-D Quadratic Vesica + DC Bit-Flip Projection (only near-zero |x| <= 0.10*d):
         if cfg.use_spectral and (lam_vw > 0.0 or lam_dc > 0.0):
             d_col = best_d[:, None]
             step_col = 2.0 * d_col
@@ -368,7 +366,8 @@ def _quantize_q1_0_hpc_cuda(
             v = e[:, :half] + e[:, half:]
             dc = torch.sum(e, dim=1)
             denom = (W + lam_vw + lam_dc) * (step_col * step_col)
-            act_b = torch.abs(d_col) > 1.0e-15
+            near_zero = torch.abs(X) <= 0.10 * d_col
+            act_b = (d_col > 1.0e-15) & near_zero
             s_den = torch.where(act_b & (denom > 1.0e-30), denom, torch.ones_like(denom))
             v_full = torch.cat([v, v], dim=1)
             grad = step_col * (W * e + lam_vw * v_full + lam_dc * dc[:, None])
@@ -436,8 +435,10 @@ def quantize_tensor_q1_0(
             Wb = np.ones_like(Xb, dtype=np.float32)
         else:
             wr = np.maximum(W_raw[start:stop], 1.0e-6)
-            wm = np.mean(wr, axis=1, keepdims=True)
-            Wb = np.where(wm > 1.0e-30, wr / np.maximum(wm, 1.0e-30), 1.0).astype(np.float32)
+            med = np.maximum(np.median(wr, axis=1, keepdims=True), 1.0e-6)
+            wt = np.clip((wr / med) ** 0.25 if cfg.temper_imatrix else wr / med, 0.5, 2.5)
+            wm = np.mean(wt, axis=1, keepdims=True)
+            Wb = np.where(wm > 1.0e-30, wt / np.maximum(wm, 1.0e-30), 1.0).astype(np.float32)
 
         if use_cuda and dev is not None:
             raw_b, sse_b = _quantize_q1_0_hpc_cuda(Xb, Wb, dev, cfg)