File size: 1,228 Bytes
2523b1a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | [
{
"quant_type": "FP16",
"file_suffix": "fp16.gguf",
"size_mb": 3100,
"ram_required_mb": 3800,
"core_accuracy_pct": 94.77,
"real_accuracy_pct": 60.42,
"recovery_rate_pct": 53.85,
"tokens_per_sec": 14.2,
"p50_latency_ms": 320.0,
"p95_latency_ms": 480.0
},
{
"quant_type": "Q8_0",
"file_suffix": "q8_0.gguf",
"size_mb": 1650,
"ram_required_mb": 2100,
"core_accuracy_pct": 94.72,
"real_accuracy_pct": 60.35,
"recovery_rate_pct": 53.8,
"tokens_per_sec": 22.5,
"p50_latency_ms": 210.0,
"p95_latency_ms": 310.0
},
{
"quant_type": "Q6_K",
"file_suffix": "q6_k.gguf",
"size_mb": 1280,
"ram_required_mb": 1650,
"core_accuracy_pct": 94.6,
"real_accuracy_pct": 60.1,
"recovery_rate_pct": 53.5,
"tokens_per_sec": 28.1,
"p50_latency_ms": 175.0,
"p95_latency_ms": 250.0
},
{
"quant_type": "Q4_K_M",
"file_suffix": "q4_k_m.gguf",
"size_mb": 940,
"ram_required_mb": 1200,
"core_accuracy_pct": 94.45,
"real_accuracy_pct": 59.85,
"recovery_rate_pct": 53.1,
"tokens_per_sec": 34.8,
"p50_latency_ms": 140.0,
"p95_latency_ms": 195.0,
"recommended_production": true
}
] |