{ "model": "gemma-4-12B-it-OnlyText", "llama_cpp_commit": "9d286e1b315c", "date": "2026-09-24", "host": { "os": "Linux-7.0.0-31-generic-x86_64-with-glibc2.39", "cpu": "x86_64", "gpus": "2x AMD Radeon AI PRO R9700 32GB (gfx1201)" }, "protocol": { "baseline": "llama-bench -p 512,1024,2048 -n 128,256 -ngl 99 -fa on -r 3", "chat_control": "llama-cli -n 256 -ngl 99 -fa on -np 1 --temp 0 -st, ~90-token fixed instruction prompt (the no-draft decode control)", "quant_damage": "llama-perplexity --kl-divergence against the F16 master's logits (ctx 1024, 40 windows, 20440 scored tokens)" }, "results": { "single": { "Q4_K_M": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q4_K_M.gguf", "model_type": "gemma4 ?B Q4_K - Medium", "model_size": 7365536414, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:30:10Z", "avg_ns": 258935603, "stddev_ns": 52911841, "avg_ts": 2029.527797, "stddev_ts": 384.580842, "samples_ns": [ 318293196, 241793548, 216720066 ], "samples_ts": [ 1608.58, 2117.51, 2362.49 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q4_K_M.gguf", "model_type": "gemma4 ?B Q4_K - Medium", "model_size": 7365536414, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:30:11Z", "avg_ns": 441297406, "stddev_ns": 1690875, "avg_ts": 2320.453251, "stddev_ts": 8.87146, "samples_ns": [ 440288358, 443249489, 440354371 ], "samples_ts": [ 2325.75, 2310.21, 2325.4 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q4_K_M.gguf", "model_type": "gemma4 ?B Q4_K - Medium", "model_size": 7365536414, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:30:13Z", "avg_ns": 955422741, "stddev_ns": 2495364, "avg_ts": 2143.5635, "stddev_ts": 5.60102, "samples_ns": [ 952756842, 955809822, 957701561 ], "samples_ts": [ 2149.55, 2142.69, 2138.45 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q4_K_M.gguf", "model_type": "gemma4 ?B Q4_K - Medium", "model_size": 7365536414, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-24T18:30:17Z", "avg_ns": 2212306617, "stddev_ns": 3044648, "avg_ts": 57.858237, "stddev_ts": 0.07958, "samples_ns": [ 2209849196, 2211358013, 2215712642 ], "samples_ts": [ 57.9225, 57.883, 57.7692 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q4_K_M.gguf", "model_type": "gemma4 ?B Q4_K - Medium", "model_size": 7365536414, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-24T18:30:24Z", "avg_ns": 4444979464, "stddev_ns": 10640940, "avg_ts": 57.593287, "stddev_ts": 0.137773, "samples_ns": [ 4435659069, 4442705882, 4456573441 ], "samples_ts": [ 57.7141, 57.6225, 57.4432 ] } ], "mtp_n0": { "prompt_tokens_per_second": 456.0, "decode_tokens_per_second": 54.4 } }, "Q6_K": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q6_K.gguf", "model_type": "gemma4 ?B Q6_K", "model_size": 9770175134, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:30:48Z", "avg_ns": 387973301, "stddev_ns": 70895840, "avg_ts": 1346.80069, "stddev_ts": 222.639541, "samples_ns": [ 469830504, 347921232, 346168169 ], "samples_ts": [ 1089.75, 1471.6, 1479.05 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q6_K.gguf", "model_type": "gemma4 ?B Q6_K", "model_size": 9770175134, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:30:50Z", "avg_ns": 711583389, "stddev_ns": 2424602, "avg_ts": 1439.055387, "stddev_ts": 4.893909, "samples_ns": [ 710449135, 709933968, 714367065 ], "samples_ts": [ 1441.34, 1442.39, 1433.44 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q6_K.gguf", "model_type": "gemma4 ?B Q6_K", "model_size": 9770175134, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:30:52Z", "avg_ns": 1501614846, "stddev_ns": 2135355, "avg_ts": 1363.866884, "stddev_ts": 1.940643, "samples_ns": [ 1499167467, 1503095424, 1502581648 ], "samples_ts": [ 1366.09, 1362.52, 1362.99 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q6_K.gguf", "model_type": "gemma4 ?B Q6_K", "model_size": 9770175134, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-24T18:30:58Z", "avg_ns": 2690318366, "stddev_ns": 16206237, "avg_ts": 47.579161, "stddev_ts": 0.286139, "samples_ns": [ 2675886517, 2687217764, 2707850818 ], "samples_ts": [ 47.8346, 47.6329, 47.27 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q6_K.gguf", "model_type": "gemma4 ?B Q6_K", "model_size": 9770175134, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-24T18:31:07Z", "avg_ns": 5364116844, "stddev_ns": 26814720, "avg_ts": 47.725336, "stddev_ts": 0.239211, "samples_ns": [ 5375977574, 5382955631, 5333417329 ], "samples_ts": [ 47.6192, 47.5575, 47.9992 ] } ], "mtp_n0": { "prompt_tokens_per_second": 380.2, "decode_tokens_per_second": 46.2 } }, "Q8_0": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q8_0.gguf", "model_type": "gemma4 ?B Q8_0", "model_size": 12653793584, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:31:35Z", "avg_ns": 311788306, "stddev_ns": 122221229, "avg_ts": 1797.687289, "stddev_ts": 600.634017, "samples_ns": [ 450962891, 262470565, 221931464 ], "samples_ts": [ 1135.35, 1950.69, 2307.02 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q8_0.gguf", "model_type": "gemma4 ?B Q8_0", "model_size": 12653793584, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:31:36Z", "avg_ns": 383045480, "stddev_ns": 2155902, "avg_ts": 2673.368291, "stddev_ts": 15.057889, "samples_ns": [ 383245800, 385094143, 380796498 ], "samples_ts": [ 2671.91, 2659.09, 2689.1 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q8_0.gguf", "model_type": "gemma4 ?B Q8_0", "model_size": 12653793584, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-24T18:31:38Z", "avg_ns": 834600943, "stddev_ns": 3388654, "avg_ts": 2453.894301, "stddev_ts": 9.958651, "samples_ns": [ 831353789, 834333760, 838115280 ], "samples_ts": [ 2463.45, 2454.65, 2443.58 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q8_0.gguf", "model_type": "gemma4 ?B Q8_0", "model_size": 12653793584, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-24T18:31:41Z", "avg_ns": 3240397399, "stddev_ns": 2479796, "avg_ts": 39.501343, "stddev_ts": 0.030203, "samples_ns": [ 3243188806, 3239548458, 3238454935 ], "samples_ts": [ 39.4673, 39.5117, 39.525 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/gemma-4-12B-it-OnlyText-GGUF/gemma-4-12B-it-OnlyText-Q8_0.gguf", "model_type": "gemma4 ?B Q8_0", "model_size": 12653793584, "model_n_params": 11907323696, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-24T18:31:51Z", "avg_ns": 6548554623, "stddev_ns": 144946322, "avg_ts": 39.105208, "stddev_ts": 0.854685, "samples_ns": [ 6460441963, 6469377236, 6715844671 ], "samples_ts": [ 39.6258, 39.571, 38.1188 ] } ], "mtp_n0": { "prompt_tokens_per_second": 578.4, "decode_tokens_per_second": 38.5 } } }, "dual": {} }, "ppl": {}, "quant_kl": { "Q4_K_M": { "mean_kld": 1.47213, "median_kld": 0.781188, "max_kld": 29.328627, "same_top_p": 56.272 }, "Q6_K": { "mean_kld": 0.413062, "median_kld": 0.086526, "max_kld": 22.610075, "same_top_p": 77.926 }, "Q8_0": { "mean_kld": 0.206998, "median_kld": 0.022358, "max_kld": 24.440359, "same_top_p": 85.802 } }, "ppl_withheld": "Perplexity is not reported for this family, but not because the measurement is broken: llama-perplexity adds the leading BOS this architecture needs (llama.cpp forces it for gemma4), and an independent HF implementation of the same protocol agrees to within 0.08 PPL (160.9418 vs 160.8636) with identical tokenization, so the number is a faithful property of the checkpoint. It is withheld because raw-wikitext perplexity is not a meaningful quality signal for an instruct-only model: the median token is predicted well (1.88 nats, 44% top-1) while the mean is dominated by a heavy tail where the chat-tuned model is out of distribution on plain text, and the result is not monotone in quantization width. Upstream guidance is to measure the base checkpoint or chat-formatted data. The raw ppl-*.log files are still on disk. See docs/gemma4-plan.md." }