Upload 2 files

q3_k.py

@@ -79,6 +79,12 @@ _log = logging.getLogger(__name__)
 # The CPU default stays small (numpy temporaries stay cache-resident).
 Q3_K_CHUNK_BLOCKS_CUDA = 16384
 Q3_K_CHUNK_BLOCKS_CPU = 1024
+# Empirical peak VRAM per 256-weight super-block on the CUDA pipeline (same
+# 4 GiB per reference chunk as Q4_0, over an 8x-wider block — identical
+# 32768 bytes/block); used to auto-scale the GPU chunk to the free memory
+# actually available on the device.
+Q3_K_CUDA_BYTES_PER_BLOCK = 32768
+Q3_K_CUDA_VRAM_FRACTION = 0.45
 
 
 def _to_dev(x, dev: "torch.device") -> "torch.Tensor":
@@ -1011,7 +1017,20 @@ def quantize_tensor_q3_k(
     )
     dev = torch.device("cuda" if cfg.device == "auto" else cfg.device) if use_cuda else None
     if chunk_blocks is None:
-        chunk_blocks = Q3_K_CHUNK_BLOCKS_CUDA if use_cuda else Q3_K_CHUNK_BLOCKS_CPU
+        if use_cuda and dev is not None:
+            # VRAM-aware default: size the chunk from the free memory at
+            # start-up, never below the reference chunk (which always fits
+            # even on small cards).
+            try:
+                free_bytes, _ = torch.cuda.mem_get_info(dev)
+                chunk_blocks = max(
+                    Q3_K_CHUNK_BLOCKS_CUDA,
+                    int(Q3_K_CUDA_VRAM_FRACTION * free_bytes) // Q3_K_CUDA_BYTES_PER_BLOCK,
+                )
+            except Exception:
+                chunk_blocks = Q3_K_CHUNK_BLOCKS_CUDA
+        else:
+            chunk_blocks = Q3_K_CHUNK_BLOCKS_CPU
     chunk_blocks = max(1, int(chunk_blocks))
     _log.info(
         "Q3_K routing: cfg.device=%r cuda_is_available=%s n_blocks=%d -> %s (chunk_blocks=%d)",

q4_0.py

@@ -60,6 +60,11 @@ _log = logging.getLogger(__name__)
 # allocating multi-hundred-MB arrays per chunk.
 Q4_0_CHUNK_BLOCKS_CUDA = 131072
 Q4_0_CHUNK_BLOCKS_CPU = 8192
+# Empirical peak VRAM per 32-weight block on the CUDA pipeline (~4 GiB
+# observed for the 131072-block reference chunk); used to auto-scale the GPU
+# chunk to the free memory actually available on the device.
+Q4_0_CUDA_BYTES_PER_BLOCK = 32768
+Q4_0_CUDA_VRAM_FRACTION = 0.45
 
 
 def dequantize_q4_0_bytes(raw: bytes | np.ndarray, n_blocks: int) -> np.ndarray:
@@ -388,10 +393,10 @@ def _quantize_q4_0_hpc_cuda(
         e_u = X[:, None, :] - all_d[:, :, None] * (q_u.to(torch.float32) - 8.0)
         sse_u = torch.sum(W[:, None, :] * e_u * e_u, dim=2)
         pick_u = torch.argmin(sse_u, dim=1)
-        del e_u, q_u, qi_u, sh_u, sse_u
         best_d16 = all_b[rows, pick_u]
         best_d = all_d[rows, pick_u]
         best_q = q_u[rows, pick_u].to(torch.int64)
+        del e_u, q_u, qi_u, sh_u, sse_u
 
         # 5. Closed-Form 1-D Quadratic Vesica + DC Code Projection within sse_budget:
         if cfg.use_spectral and (lam_vw > 0.0 or lam_dc > 0.0):
@@ -609,7 +614,20 @@ def quantize_tensor_q4_0(
     )
     dev = torch.device("cuda" if cfg.device == "auto" else cfg.device) if use_cuda else None
     if chunk_blocks is None:
-        chunk_blocks = Q4_0_CHUNK_BLOCKS_CUDA if use_cuda else Q4_0_CHUNK_BLOCKS_CPU
+        if use_cuda and dev is not None:
+            # VRAM-aware default: size the chunk from the free memory at
+            # start-up, never below the reference chunk (which always fits
+            # even on small cards).
+            try:
+                free_bytes, _ = torch.cuda.mem_get_info(dev)
+                chunk_blocks = max(
+                    Q4_0_CHUNK_BLOCKS_CUDA,
+                    int(Q4_0_CUDA_VRAM_FRACTION * free_bytes) // Q4_0_CUDA_BYTES_PER_BLOCK,
+                )
+            except Exception:
+                chunk_blocks = Q4_0_CHUNK_BLOCKS_CUDA
+        else:
+            chunk_blocks = Q4_0_CHUNK_BLOCKS_CPU
     chunk_blocks = max(1, int(chunk_blocks))
     _log.info(
         "Q4_0 routing: cfg.device=%r cuda_is_available=%s n_blocks=%d -> %s (chunk_blocks=%d)",