{ "version": "2026-05-04", "source": "Unsloth Dynamic GGUF benchmarks (https://unsloth.ai) + llama.cpp perplexity discussions (github.com/ggerganov/llama.cpp/discussions/406)", "notes": "KLD = KL divergence vs FP16 baseline (Wikitext-2). ppl_delta_pct = perplexity increase vs FP16. Larger models (>30B) tolerate quantization better; smaller models (<3B) suffer more. Lookup falls back: exact family → size_class → default.", "size_class_thresholds_b": { "tiny": 2, "small": 9, "medium": 30 }, "families": { "default": { "size_class": "small", "quants": { "IQ1_S": { "kld": 0.250, "ppl_delta_pct": 30.0 }, "IQ1_M": { "kld": 0.180, "ppl_delta_pct": 22.0 }, "IQ2_XXS":{ "kld": 0.120, "ppl_delta_pct": 15.0 }, "IQ2_XS": { "kld": 0.080, "ppl_delta_pct": 10.0 }, "IQ2_M": { "kld": 0.055, "ppl_delta_pct": 7.0 }, "Q2_K": { "kld": 0.045, "ppl_delta_pct": 5.8 }, "Q2_K_L": { "kld": 0.041, "ppl_delta_pct": 5.4 }, "IQ3_XXS":{ "kld": 0.025, "ppl_delta_pct": 3.5 }, "IQ3_XS": { "kld": 0.018, "ppl_delta_pct": 2.5 }, "IQ3_M": { "kld": 0.014, "ppl_delta_pct": 1.8 }, "Q3_K_S": { "kld": 0.020, "ppl_delta_pct": 2.6 }, "Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 }, "Q3_K_L": { "kld": 0.011, "ppl_delta_pct": 1.2 }, "IQ4_XS": { "kld": 0.008, "ppl_delta_pct": 1.0 }, "IQ4_NL": { "kld": 0.006, "ppl_delta_pct": 0.7 }, "Q4_0": { "kld": 0.005, "ppl_delta_pct": 0.6 }, "Q4_K_S": { "kld": 0.0042, "ppl_delta_pct": 0.5 }, "Q4_K_M": { "kld": 0.0034, "ppl_delta_pct": 0.4 }, "Q5_K_S": { "kld": 0.0012, "ppl_delta_pct": 0.3 }, "Q5_K_M": { "kld": 0.0008, "ppl_delta_pct": 0.2 }, "Q6_K": { "kld": 0.00020, "ppl_delta_pct": 0.1 }, "Q8_0": { "kld": 0.00006, "ppl_delta_pct": 0.05 }, "F16": { "kld": 0.0, "ppl_delta_pct": 0.0 }, "BF16": { "kld": 0.0, "ppl_delta_pct": 0.0 }, "FP8": { "kld": 0.0001, "ppl_delta_pct": 0.08 }, "F32": { "kld": 0.0, "ppl_delta_pct": 0.0 } } }, "llama-3.1-8b": { "size_class": "small", "quants": { "Q2_K": { "kld": 0.041, "ppl_delta_pct": 5.4 }, "Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 }, "IQ4_XS": { "kld": 0.0064, "ppl_delta_pct": 0.85 }, "Q4_K_M": { "kld": 0.0034, "ppl_delta_pct": 0.42 }, "Q5_K_M": { "kld": 0.0008, "ppl_delta_pct": 0.19 }, "Q6_K": { "kld": 0.00020, "ppl_delta_pct": 0.08 }, "Q8_0": { "kld": 0.00006, "ppl_delta_pct": 0.04 } } }, "llama-3.1-70b": { "size_class": "medium", "quants": { "IQ2_XXS":{ "kld": 0.058, "ppl_delta_pct": 7.5 }, "IQ2_M": { "kld": 0.026, "ppl_delta_pct": 3.4 }, "Q2_K": { "kld": 0.020, "ppl_delta_pct": 2.6 }, "IQ3_M": { "kld": 0.0072, "ppl_delta_pct": 0.95 }, "Q3_K_M": { "kld": 0.0066, "ppl_delta_pct": 0.72 }, "IQ4_XS": { "kld": 0.0028, "ppl_delta_pct": 0.38 }, "Q4_K_M": { "kld": 0.0014, "ppl_delta_pct": 0.18 }, "Q5_K_M": { "kld": 0.00032, "ppl_delta_pct": 0.08 }, "Q6_K": { "kld": 0.000080,"ppl_delta_pct": 0.03 }, "Q8_0": { "kld": 0.000022,"ppl_delta_pct": 0.015 } } }, "llama-3.2-3b": { "size_class": "tiny", "quants": { "Q2_K": { "kld": 0.080, "ppl_delta_pct": 10.0 }, "Q3_K_M": { "kld": 0.028, "ppl_delta_pct": 3.1 }, "Q4_K_M": { "kld": 0.0070, "ppl_delta_pct": 0.85 }, "Q5_K_M": { "kld": 0.0017, "ppl_delta_pct": 0.40 }, "Q6_K": { "kld": 0.00040, "ppl_delta_pct": 0.16 }, "Q8_0": { "kld": 0.00012, "ppl_delta_pct": 0.08 } } }, "llama-3.2-1b": { "size_class": "tiny", "quants": { "Q2_K": { "kld": 0.140, "ppl_delta_pct": 17.0 }, "Q3_K_M": { "kld": 0.045, "ppl_delta_pct": 5.4 }, "Q4_K_M": { "kld": 0.012, "ppl_delta_pct": 1.5 }, "Q5_K_M": { "kld": 0.0030, "ppl_delta_pct": 0.70 }, "Q6_K": { "kld": 0.00072, "ppl_delta_pct": 0.30 }, "Q8_0": { "kld": 0.00021, "ppl_delta_pct": 0.14 } } }, "qwen2.5-7b": { "size_class": "small", "quants": { "Q2_K": { "kld": 0.044, "ppl_delta_pct": 5.8 }, "Q3_K_M": { "kld": 0.015, "ppl_delta_pct": 1.6 }, "Q4_K_M": { "kld": 0.0036, "ppl_delta_pct": 0.45 }, "Q5_K_M": { "kld": 0.0009, "ppl_delta_pct": 0.21 }, "Q6_K": { "kld": 0.00022, "ppl_delta_pct": 0.09 }, "Q8_0": { "kld": 0.00007, "ppl_delta_pct": 0.05 } } }, "qwen2.5-14b": { "size_class": "small", "quants": { "Q2_K": { "kld": 0.030, "ppl_delta_pct": 4.0 }, "Q3_K_M": { "kld": 0.010, "ppl_delta_pct": 1.1 }, "Q4_K_M": { "kld": 0.0024, "ppl_delta_pct": 0.30 }, "Q5_K_M": { "kld": 0.0006, "ppl_delta_pct": 0.14 }, "Q6_K": { "kld": 0.00015, "ppl_delta_pct": 0.06 }, "Q8_0": { "kld": 0.00005, "ppl_delta_pct": 0.03 } } }, "qwen2.5-32b": { "size_class": "medium", "quants": { "IQ2_M": { "kld": 0.034, "ppl_delta_pct": 4.4 }, "Q2_K": { "kld": 0.026, "ppl_delta_pct": 3.4 }, "Q3_K_M": { "kld": 0.0086, "ppl_delta_pct": 0.94 }, "IQ4_XS": { "kld": 0.0036, "ppl_delta_pct": 0.49 }, "Q4_K_M": { "kld": 0.0018, "ppl_delta_pct": 0.23 }, "Q5_K_M": { "kld": 0.00042, "ppl_delta_pct": 0.10 }, "Q6_K": { "kld": 0.00010, "ppl_delta_pct": 0.04 }, "Q8_0": { "kld": 0.00003, "ppl_delta_pct": 0.02 }, "FP8": { "kld": 0.00006, "ppl_delta_pct": 0.04 } } }, "qwen2.5-72b": { "size_class": "large", "quants": { "IQ2_M": { "kld": 0.022, "ppl_delta_pct": 2.9 }, "Q2_K": { "kld": 0.017, "ppl_delta_pct": 2.2 }, "Q3_K_M": { "kld": 0.0058, "ppl_delta_pct": 0.62 }, "Q4_K_M": { "kld": 0.0012, "ppl_delta_pct": 0.15 }, "Q5_K_M": { "kld": 0.00028, "ppl_delta_pct": 0.07 }, "Q6_K": { "kld": 0.00007, "ppl_delta_pct": 0.025}, "Q8_0": { "kld": 0.000018,"ppl_delta_pct": 0.013} } }, "mistral-7b": { "size_class": "small", "quants": { "Q2_K": { "kld": 0.046, "ppl_delta_pct": 6.1 }, "Q3_K_M": { "kld": 0.016, "ppl_delta_pct": 1.7 }, "Q4_K_M": { "kld": 0.0040, "ppl_delta_pct": 0.50 }, "Q5_K_M": { "kld": 0.0010, "ppl_delta_pct": 0.23 }, "Q6_K": { "kld": 0.00024, "ppl_delta_pct": 0.10 }, "Q8_0": { "kld": 0.00008, "ppl_delta_pct": 0.05 } } }, "mixtral-8x7b": { "size_class": "medium", "quants": { "Q2_K": { "kld": 0.024, "ppl_delta_pct": 3.1 }, "Q3_K_M": { "kld": 0.0078, "ppl_delta_pct": 0.85 }, "Q4_K_M": { "kld": 0.0017, "ppl_delta_pct": 0.21 }, "Q5_K_M": { "kld": 0.00040, "ppl_delta_pct": 0.10 }, "Q6_K": { "kld": 0.00010, "ppl_delta_pct": 0.04 }, "Q8_0": { "kld": 0.00003, "ppl_delta_pct": 0.02 } } }, "phi-3-mini": { "size_class": "tiny", "quants": { "Q2_K": { "kld": 0.110, "ppl_delta_pct": 13.5 }, "Q3_K_M": { "kld": 0.034, "ppl_delta_pct": 4.1 }, "Q4_K_M": { "kld": 0.0090, "ppl_delta_pct": 1.1 }, "Q5_K_M": { "kld": 0.0022, "ppl_delta_pct": 0.50 }, "Q6_K": { "kld": 0.00056, "ppl_delta_pct": 0.22 }, "Q8_0": { "kld": 0.00017, "ppl_delta_pct": 0.10 } } }, "gemma-2-9b": { "size_class": "small", "quants": { "Q2_K": { "kld": 0.043, "ppl_delta_pct": 5.6 }, "Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 }, "Q4_K_M": { "kld": 0.0034, "ppl_delta_pct": 0.42 }, "Q5_K_M": { "kld": 0.0009, "ppl_delta_pct": 0.20 }, "Q6_K": { "kld": 0.00021, "ppl_delta_pct": 0.09 }, "Q8_0": { "kld": 0.00007, "ppl_delta_pct": 0.05 } } }, "gemma-2-27b": { "size_class": "medium", "quants": { "Q2_K": { "kld": 0.028, "ppl_delta_pct": 3.7 }, "Q3_K_M": { "kld": 0.0094, "ppl_delta_pct": 1.0 }, "Q4_K_M": { "kld": 0.0021, "ppl_delta_pct": 0.27 }, "Q5_K_M": { "kld": 0.00050, "ppl_delta_pct": 0.12 }, "Q6_K": { "kld": 0.00012, "ppl_delta_pct": 0.05 }, "Q8_0": { "kld": 0.00004, "ppl_delta_pct": 0.025} } }, "deepseek-r1-distill-llama-8b": { "size_class": "small", "quants": { "Q2_K": { "kld": 0.042, "ppl_delta_pct": 5.5 }, "Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 }, "Q4_K_M": { "kld": 0.0035, "ppl_delta_pct": 0.43 }, "Q5_K_M": { "kld": 0.0008, "ppl_delta_pct": 0.19 }, "Q6_K": { "kld": 0.00020, "ppl_delta_pct": 0.09 }, "Q8_0": { "kld": 0.00006, "ppl_delta_pct": 0.04 } } } } }