File size: 9,023 Bytes
01560d7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | {
"version": "2026-05-04",
"source": "Unsloth Dynamic GGUF benchmarks (https://unsloth.ai) + llama.cpp perplexity discussions (github.com/ggerganov/llama.cpp/discussions/406)",
"notes": "KLD = KL divergence vs FP16 baseline (Wikitext-2). ppl_delta_pct = perplexity increase vs FP16. Larger models (>30B) tolerate quantization better; smaller models (<3B) suffer more. Lookup falls back: exact family → size_class → default.",
"size_class_thresholds_b": {
"tiny": 2,
"small": 9,
"medium": 30
},
"families": {
"default": {
"size_class": "small",
"quants": {
"IQ1_S": { "kld": 0.250, "ppl_delta_pct": 30.0 },
"IQ1_M": { "kld": 0.180, "ppl_delta_pct": 22.0 },
"IQ2_XXS":{ "kld": 0.120, "ppl_delta_pct": 15.0 },
"IQ2_XS": { "kld": 0.080, "ppl_delta_pct": 10.0 },
"IQ2_M": { "kld": 0.055, "ppl_delta_pct": 7.0 },
"Q2_K": { "kld": 0.045, "ppl_delta_pct": 5.8 },
"Q2_K_L": { "kld": 0.041, "ppl_delta_pct": 5.4 },
"IQ3_XXS":{ "kld": 0.025, "ppl_delta_pct": 3.5 },
"IQ3_XS": { "kld": 0.018, "ppl_delta_pct": 2.5 },
"IQ3_M": { "kld": 0.014, "ppl_delta_pct": 1.8 },
"Q3_K_S": { "kld": 0.020, "ppl_delta_pct": 2.6 },
"Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 },
"Q3_K_L": { "kld": 0.011, "ppl_delta_pct": 1.2 },
"IQ4_XS": { "kld": 0.008, "ppl_delta_pct": 1.0 },
"IQ4_NL": { "kld": 0.006, "ppl_delta_pct": 0.7 },
"Q4_0": { "kld": 0.005, "ppl_delta_pct": 0.6 },
"Q4_K_S": { "kld": 0.0042, "ppl_delta_pct": 0.5 },
"Q4_K_M": { "kld": 0.0034, "ppl_delta_pct": 0.4 },
"Q5_K_S": { "kld": 0.0012, "ppl_delta_pct": 0.3 },
"Q5_K_M": { "kld": 0.0008, "ppl_delta_pct": 0.2 },
"Q6_K": { "kld": 0.00020, "ppl_delta_pct": 0.1 },
"Q8_0": { "kld": 0.00006, "ppl_delta_pct": 0.05 },
"F16": { "kld": 0.0, "ppl_delta_pct": 0.0 },
"BF16": { "kld": 0.0, "ppl_delta_pct": 0.0 },
"FP8": { "kld": 0.0001, "ppl_delta_pct": 0.08 },
"F32": { "kld": 0.0, "ppl_delta_pct": 0.0 }
}
},
"llama-3.1-8b": {
"size_class": "small",
"quants": {
"Q2_K": { "kld": 0.041, "ppl_delta_pct": 5.4 },
"Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 },
"IQ4_XS": { "kld": 0.0064, "ppl_delta_pct": 0.85 },
"Q4_K_M": { "kld": 0.0034, "ppl_delta_pct": 0.42 },
"Q5_K_M": { "kld": 0.0008, "ppl_delta_pct": 0.19 },
"Q6_K": { "kld": 0.00020, "ppl_delta_pct": 0.08 },
"Q8_0": { "kld": 0.00006, "ppl_delta_pct": 0.04 }
}
},
"llama-3.1-70b": {
"size_class": "medium",
"quants": {
"IQ2_XXS":{ "kld": 0.058, "ppl_delta_pct": 7.5 },
"IQ2_M": { "kld": 0.026, "ppl_delta_pct": 3.4 },
"Q2_K": { "kld": 0.020, "ppl_delta_pct": 2.6 },
"IQ3_M": { "kld": 0.0072, "ppl_delta_pct": 0.95 },
"Q3_K_M": { "kld": 0.0066, "ppl_delta_pct": 0.72 },
"IQ4_XS": { "kld": 0.0028, "ppl_delta_pct": 0.38 },
"Q4_K_M": { "kld": 0.0014, "ppl_delta_pct": 0.18 },
"Q5_K_M": { "kld": 0.00032, "ppl_delta_pct": 0.08 },
"Q6_K": { "kld": 0.000080,"ppl_delta_pct": 0.03 },
"Q8_0": { "kld": 0.000022,"ppl_delta_pct": 0.015 }
}
},
"llama-3.2-3b": {
"size_class": "tiny",
"quants": {
"Q2_K": { "kld": 0.080, "ppl_delta_pct": 10.0 },
"Q3_K_M": { "kld": 0.028, "ppl_delta_pct": 3.1 },
"Q4_K_M": { "kld": 0.0070, "ppl_delta_pct": 0.85 },
"Q5_K_M": { "kld": 0.0017, "ppl_delta_pct": 0.40 },
"Q6_K": { "kld": 0.00040, "ppl_delta_pct": 0.16 },
"Q8_0": { "kld": 0.00012, "ppl_delta_pct": 0.08 }
}
},
"llama-3.2-1b": {
"size_class": "tiny",
"quants": {
"Q2_K": { "kld": 0.140, "ppl_delta_pct": 17.0 },
"Q3_K_M": { "kld": 0.045, "ppl_delta_pct": 5.4 },
"Q4_K_M": { "kld": 0.012, "ppl_delta_pct": 1.5 },
"Q5_K_M": { "kld": 0.0030, "ppl_delta_pct": 0.70 },
"Q6_K": { "kld": 0.00072, "ppl_delta_pct": 0.30 },
"Q8_0": { "kld": 0.00021, "ppl_delta_pct": 0.14 }
}
},
"qwen2.5-7b": {
"size_class": "small",
"quants": {
"Q2_K": { "kld": 0.044, "ppl_delta_pct": 5.8 },
"Q3_K_M": { "kld": 0.015, "ppl_delta_pct": 1.6 },
"Q4_K_M": { "kld": 0.0036, "ppl_delta_pct": 0.45 },
"Q5_K_M": { "kld": 0.0009, "ppl_delta_pct": 0.21 },
"Q6_K": { "kld": 0.00022, "ppl_delta_pct": 0.09 },
"Q8_0": { "kld": 0.00007, "ppl_delta_pct": 0.05 }
}
},
"qwen2.5-14b": {
"size_class": "small",
"quants": {
"Q2_K": { "kld": 0.030, "ppl_delta_pct": 4.0 },
"Q3_K_M": { "kld": 0.010, "ppl_delta_pct": 1.1 },
"Q4_K_M": { "kld": 0.0024, "ppl_delta_pct": 0.30 },
"Q5_K_M": { "kld": 0.0006, "ppl_delta_pct": 0.14 },
"Q6_K": { "kld": 0.00015, "ppl_delta_pct": 0.06 },
"Q8_0": { "kld": 0.00005, "ppl_delta_pct": 0.03 }
}
},
"qwen2.5-32b": {
"size_class": "medium",
"quants": {
"IQ2_M": { "kld": 0.034, "ppl_delta_pct": 4.4 },
"Q2_K": { "kld": 0.026, "ppl_delta_pct": 3.4 },
"Q3_K_M": { "kld": 0.0086, "ppl_delta_pct": 0.94 },
"IQ4_XS": { "kld": 0.0036, "ppl_delta_pct": 0.49 },
"Q4_K_M": { "kld": 0.0018, "ppl_delta_pct": 0.23 },
"Q5_K_M": { "kld": 0.00042, "ppl_delta_pct": 0.10 },
"Q6_K": { "kld": 0.00010, "ppl_delta_pct": 0.04 },
"Q8_0": { "kld": 0.00003, "ppl_delta_pct": 0.02 },
"FP8": { "kld": 0.00006, "ppl_delta_pct": 0.04 }
}
},
"qwen2.5-72b": {
"size_class": "large",
"quants": {
"IQ2_M": { "kld": 0.022, "ppl_delta_pct": 2.9 },
"Q2_K": { "kld": 0.017, "ppl_delta_pct": 2.2 },
"Q3_K_M": { "kld": 0.0058, "ppl_delta_pct": 0.62 },
"Q4_K_M": { "kld": 0.0012, "ppl_delta_pct": 0.15 },
"Q5_K_M": { "kld": 0.00028, "ppl_delta_pct": 0.07 },
"Q6_K": { "kld": 0.00007, "ppl_delta_pct": 0.025},
"Q8_0": { "kld": 0.000018,"ppl_delta_pct": 0.013}
}
},
"mistral-7b": {
"size_class": "small",
"quants": {
"Q2_K": { "kld": 0.046, "ppl_delta_pct": 6.1 },
"Q3_K_M": { "kld": 0.016, "ppl_delta_pct": 1.7 },
"Q4_K_M": { "kld": 0.0040, "ppl_delta_pct": 0.50 },
"Q5_K_M": { "kld": 0.0010, "ppl_delta_pct": 0.23 },
"Q6_K": { "kld": 0.00024, "ppl_delta_pct": 0.10 },
"Q8_0": { "kld": 0.00008, "ppl_delta_pct": 0.05 }
}
},
"mixtral-8x7b": {
"size_class": "medium",
"quants": {
"Q2_K": { "kld": 0.024, "ppl_delta_pct": 3.1 },
"Q3_K_M": { "kld": 0.0078, "ppl_delta_pct": 0.85 },
"Q4_K_M": { "kld": 0.0017, "ppl_delta_pct": 0.21 },
"Q5_K_M": { "kld": 0.00040, "ppl_delta_pct": 0.10 },
"Q6_K": { "kld": 0.00010, "ppl_delta_pct": 0.04 },
"Q8_0": { "kld": 0.00003, "ppl_delta_pct": 0.02 }
}
},
"phi-3-mini": {
"size_class": "tiny",
"quants": {
"Q2_K": { "kld": 0.110, "ppl_delta_pct": 13.5 },
"Q3_K_M": { "kld": 0.034, "ppl_delta_pct": 4.1 },
"Q4_K_M": { "kld": 0.0090, "ppl_delta_pct": 1.1 },
"Q5_K_M": { "kld": 0.0022, "ppl_delta_pct": 0.50 },
"Q6_K": { "kld": 0.00056, "ppl_delta_pct": 0.22 },
"Q8_0": { "kld": 0.00017, "ppl_delta_pct": 0.10 }
}
},
"gemma-2-9b": {
"size_class": "small",
"quants": {
"Q2_K": { "kld": 0.043, "ppl_delta_pct": 5.6 },
"Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 },
"Q4_K_M": { "kld": 0.0034, "ppl_delta_pct": 0.42 },
"Q5_K_M": { "kld": 0.0009, "ppl_delta_pct": 0.20 },
"Q6_K": { "kld": 0.00021, "ppl_delta_pct": 0.09 },
"Q8_0": { "kld": 0.00007, "ppl_delta_pct": 0.05 }
}
},
"gemma-2-27b": {
"size_class": "medium",
"quants": {
"Q2_K": { "kld": 0.028, "ppl_delta_pct": 3.7 },
"Q3_K_M": { "kld": 0.0094, "ppl_delta_pct": 1.0 },
"Q4_K_M": { "kld": 0.0021, "ppl_delta_pct": 0.27 },
"Q5_K_M": { "kld": 0.00050, "ppl_delta_pct": 0.12 },
"Q6_K": { "kld": 0.00012, "ppl_delta_pct": 0.05 },
"Q8_0": { "kld": 0.00004, "ppl_delta_pct": 0.025}
}
},
"deepseek-r1-distill-llama-8b": {
"size_class": "small",
"quants": {
"Q2_K": { "kld": 0.042, "ppl_delta_pct": 5.5 },
"Q3_K_M": { "kld": 0.014, "ppl_delta_pct": 1.5 },
"Q4_K_M": { "kld": 0.0035, "ppl_delta_pct": 0.43 },
"Q5_K_M": { "kld": 0.0008, "ppl_delta_pct": 0.19 },
"Q6_K": { "kld": 0.00020, "ppl_delta_pct": 0.09 },
"Q8_0": { "kld": 0.00006, "ppl_delta_pct": 0.04 }
}
}
}
}
|