{ "model": "Qwen3.5-2B-OnlyText", "llama_cpp_commit": "f280b26983ad", "date": "2026-09-22", "host": { "os": "Linux-7.0.0-31-generic-x86_64-with-glibc2.39", "cpu": "x86_64", "gpus": "2x AMD Radeon AI PRO R9700 32GB (gfx1201)" }, "protocol": { "baseline": "llama-bench -p 512,1024,2048 -n 128,256 -ngl 99 -fa on -r 3", "chat_control": "llama-cli -n 256 -ngl 99 -fa on -np 1 --temp 0 -st, ~90-token fixed instruction prompt (the no-draft decode control)", "mtp": "llama-cli -n 256 -ngl 99 -fa on -np 1 --temp 0 --spec-type draft-mtp --spec-draft-n-max N", "quant_damage": "llama-perplexity --kl-divergence against the F16 master's logits (ctx 1024, 40 windows, 20440 scored tokens)" }, "results": { "single": { "Q4_K_M": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 2B Q4_K - Medium", "model_size": 1301171936, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:02:23Z", "avg_ns": 71079982, "stddev_ns": 5410282, "avg_ts": 7229.9025, "stddev_ts": 527.146313, "samples_ns": [ 77327111, 67921902, 67990933 ], "samples_ts": [ 6621.22, 7538.07, 7530.42 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 2B Q4_K - Medium", "model_size": 1301171936, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:02:23Z", "avg_ns": 105296606, "stddev_ns": 11882965, "avg_ts": 9806.166685, "stddev_ts": 1082.013635, "samples_ns": [ 118066968, 103258062, 94564788 ], "samples_ts": [ 8673.04, 9916.9, 10828.6 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 2B Q4_K - Medium", "model_size": 1301171936, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:02:23Z", "avg_ns": 184416623, "stddev_ns": 1707812, "avg_ts": 11105.92636, "stddev_ts": 103.158802, "samples_ns": [ 184812806, 182545650, 185891415 ], "samples_ts": [ 11081.5, 11219.1, 11017.2 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 2B Q4_K - Medium", "model_size": 1301171936, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-22T18:02:24Z", "avg_ns": 707397911, "stddev_ns": 694008, "avg_ts": 180.944954, "stddev_ts": 0.177501, "samples_ns": [ 707345709, 708116546, 706731478 ], "samples_ts": [ 180.958, 180.761, 181.115 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 2B Q4_K - Medium", "model_size": 1301171936, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-22T18:02:26Z", "avg_ns": 1410435669, "stddev_ns": 1810122, "avg_ts": 181.504401, "stddev_ts": 0.233037, "samples_ns": [ 1408380077, 1411137830, 1411789101 ], "samples_ts": [ 181.769, 181.414, 181.33 ] } ], "mtp_n0": { "prompt_tokens_per_second": 1567.6, "decode_tokens_per_second": 143.6 }, "mtp_n2": { "prompt_tokens_per_second": 1351.9, "decode_tokens_per_second": 157.1 }, "mtp_n3": { "prompt_tokens_per_second": 1250.9, "decode_tokens_per_second": 145.0 }, "mtp_n4": { "prompt_tokens_per_second": 1340.5, "decode_tokens_per_second": 128.7 } }, "Q6_K": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 2B Q6_K", "model_size": 1595331296, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:02:44Z", "avg_ns": 95042184, "stddev_ns": 10759994, "avg_ts": 5434.367177, "stddev_ts": 627.745898, "samples_ns": [ 105329228, 95932734, 83864590 ], "samples_ts": [ 4860.95, 5337.07, 6105.08 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 2B Q6_K", "model_size": 1595331296, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:02:44Z", "avg_ns": 127621393, "stddev_ns": 4308085, "avg_ts": 8029.870222, "stddev_ts": 272.751498, "samples_ns": [ 131663972, 123089572, 128110635 ], "samples_ts": [ 7777.37, 8319.15, 7993.09 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 2B Q6_K", "model_size": 1595331296, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:02:45Z", "avg_ns": 252188234, "stddev_ns": 94242, "avg_ts": 8120.918847, "stddev_ts": 2.99104, "samples_ns": [ 252107502, 252289766, 252167435 ], "samples_ts": [ 8123.52, 8117.65, 8121.59 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 2B Q6_K", "model_size": 1595331296, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-22T18:02:46Z", "avg_ns": 709443040, "stddev_ns": 1248134, "avg_ts": 180.423595, "stddev_ts": 0.317577, "samples_ns": [ 710316515, 708014233, 709998374 ], "samples_ts": [ 180.201, 180.787, 180.282 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 2B Q6_K", "model_size": 1595331296, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-22T18:02:48Z", "avg_ns": 1420251972, "stddev_ns": 3039743, "avg_ts": 180.250256, "stddev_ts": 0.385751, "samples_ns": [ 1420161903, 1417258264, 1423335749 ], "samples_ts": [ 180.261, 180.63, 179.859 ] } ], "mtp_n0": { "prompt_tokens_per_second": 1638.2, "decode_tokens_per_second": 147.7 }, "mtp_n2": { "prompt_tokens_per_second": 1254.5, "decode_tokens_per_second": 143.8 }, "mtp_n3": { "prompt_tokens_per_second": 1260.6, "decode_tokens_per_second": 134.4 }, "mtp_n4": { "prompt_tokens_per_second": 1293.9, "decode_tokens_per_second": 137.9 } }, "Q8_0": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 2B Q8_0", "model_size": 2065673216, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:03:06Z", "avg_ns": 50831806, "stddev_ns": 6860199, "avg_ts": 10187.226486, "stddev_ts": 1275.896867, "samples_ns": [ 58750087, 47067483, 46677848 ], "samples_ts": [ 8714.88, 10878, 10968.8 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 2B Q8_0", "model_size": 2065673216, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:03:06Z", "avg_ns": 91451448, "stddev_ns": 4850571, "avg_ts": 11218.732198, "stddev_ts": 609.11956, "samples_ns": [ 92972281, 95359356, 86022708 ], "samples_ts": [ 11014, 10738.3, 11903.8 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 2B Q8_0", "model_size": 2065673216, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:03:07Z", "avg_ns": 246669691, "stddev_ns": 88583054, "avg_ts": 8939.189127, "stddev_ts": 2658.995691, "samples_ns": [ 348955583, 195920087, 195133403 ], "samples_ts": [ 5868.94, 10453.2, 10495.4 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 2B Q8_0", "model_size": 2065673216, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-22T18:03:07Z", "avg_ns": 782557850, "stddev_ns": 20037620, "avg_ts": 163.63667, "stddev_ts": 4.129274, "samples_ns": [ 805680855, 771703757, 770288939 ], "samples_ts": [ 158.872, 165.867, 166.171 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-2B-OnlyText-GGUF/Qwen3.5-2B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 2B Q8_0", "model_size": 2065673216, "model_n_params": 1942616384, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-22T18:03:10Z", "avg_ns": 1535635835, "stddev_ns": 2431673, "avg_ts": 166.706469, "stddev_ts": 0.264132, "samples_ns": [ 1537685262, 1532948918, 1536273325 ], "samples_ts": [ 166.484, 166.998, 166.637 ] } ], "mtp_n0": { "prompt_tokens_per_second": 1816.9, "decode_tokens_per_second": 142.8 }, "mtp_n2": { "prompt_tokens_per_second": 1559.3, "decode_tokens_per_second": 157.2 }, "mtp_n3": { "prompt_tokens_per_second": 1549.5, "decode_tokens_per_second": 151.5 }, "mtp_n4": { "prompt_tokens_per_second": 1492.2, "decode_tokens_per_second": 127.1 } } }, "dual": {} }, "ppl": { "Q4_K_M": 12.2515, "Q6_K": 11.5519, "Q8_0": 11.2841 }, "quant_kl": { "Q4_K_M": { "mean_kld": 0.051197, "median_kld": 0.033427, "max_kld": 19.147791, "same_top_p": 88.576 }, "Q6_K": { "mean_kld": 0.005935, "median_kld": 0.003854, "max_kld": 5.406886, "same_top_p": 95.822 }, "Q8_0": { "mean_kld": 0.001069, "median_kld": 0.000713, "max_kld": 0.867766, "same_top_p": 98.312 } } }