{ "model": "Qwen3.5-4B-OnlyText", "llama_cpp_commit": "f280b26983ad", "date": "2026-09-22", "host": { "os": "Linux-7.0.0-31-generic-x86_64-with-glibc2.39", "cpu": "x86_64", "gpus": "2x AMD Radeon AI PRO R9700 32GB (gfx1201)" }, "protocol": { "baseline": "llama-bench -p 512,1024,2048 -n 128,256 -ngl 99 -fa on -r 3", "chat_control": "llama-cli -n 256 -ngl 99 -fa on -np 1 --temp 0 -st, ~90-token fixed instruction prompt (the no-draft decode control)", "mtp": "llama-cli -n 256 -ngl 99 -fa on -np 1 --temp 0 --spec-type draft-mtp --spec-draft-n-max N", "quant_damage": "llama-perplexity --kl-divergence against the F16 master's logits (ctx 1024, 40 windows, 20440 scored tokens)" }, "results": { "single": { "Q4_K_M": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 4B Q4_K - Medium", "model_size": 2772440152, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:06:42Z", "avg_ns": 157664281, "stddev_ns": 15253833, "avg_ts": 3268.87538, "stddev_ts": 332.961824, "samples_ns": [ 168850252, 163854121, 140288470 ], "samples_ts": [ 3032.27, 3124.73, 3649.62 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 4B Q4_K - Medium", "model_size": 2772440152, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:06:43Z", "avg_ns": 223363279, "stddev_ns": 2925343, "avg_ts": 4584.980623, "stddev_ts": 59.637458, "samples_ns": [ 226708297, 222097924, 221283616 ], "samples_ts": [ 4516.82, 4610.58, 4627.55 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 4B Q4_K - Medium", "model_size": 2772440152, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:06:44Z", "avg_ns": 456350160, "stddev_ns": 935293, "avg_ts": 4487.794498, "stddev_ts": 9.200938, "samples_ns": [ 457228374, 455366691, 456455415 ], "samples_ts": [ 4479.16, 4497.47, 4486.75 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 4B Q4_K - Medium", "model_size": 2772440152, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-22T18:06:45Z", "avg_ns": 1229945526, "stddev_ns": 1334002, "avg_ts": 104.069731, "stddev_ts": 0.112871, "samples_ns": [ 1231290800, 1228623089, 1229922689 ], "samples_ts": [ 103.956, 104.182, 104.072 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q4_K_M.gguf", "model_type": "qwen35 4B Q4_K - Medium", "model_size": 2772440152, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-22T18:06:49Z", "avg_ns": 2460154625, "stddev_ns": 4888149, "avg_ts": 104.058774, "stddev_ts": 0.206532, "samples_ns": [ 2456649962, 2458075567, 2465738347 ], "samples_ts": [ 104.207, 104.147, 103.823 ] } ], "mtp_n0": { "prompt_tokens_per_second": 897.4, "decode_tokens_per_second": 93.7 }, "mtp_n2": { "prompt_tokens_per_second": 697.5, "decode_tokens_per_second": 113.4 }, "mtp_n3": { "prompt_tokens_per_second": 797.9, "decode_tokens_per_second": 103.4 }, "mtp_n4": { "prompt_tokens_per_second": 672.0, "decode_tokens_per_second": 98.2 } }, "Q6_K": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 4B Q6_K", "model_size": 3552021592, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:07:15Z", "avg_ns": 222804969, "stddev_ns": 29882719, "avg_ts": 2327.792983, "stddev_ts": 333.817901, "samples_ns": [ 246051808, 233264694, 189098405 ], "samples_ts": [ 2080.86, 2194.93, 2707.58 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 4B Q6_K", "model_size": 3552021592, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:07:16Z", "avg_ns": 316753507, "stddev_ns": 1602738, "avg_ts": 3232.852796, "stddev_ts": 16.341433, "samples_ns": [ 318454512, 315271572, 316534437 ], "samples_ts": [ 3215.53, 3247.99, 3235.04 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 4B Q6_K", "model_size": 3552021592, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:07:18Z", "avg_ns": 643404770, "stddev_ns": 1686905, "avg_ts": 3183.080797, "stddev_ts": 8.332196, "samples_ns": [ 642536383, 645348744, 642329184 ], "samples_ts": [ 3187.37, 3173.48, 3188.4 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 4B Q6_K", "model_size": 3552021592, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-22T18:07:20Z", "avg_ns": 1289547927, "stddev_ns": 1909433, "avg_ts": 99.259736, "stddev_ts": 0.147, "samples_ns": [ 1289858080, 1291282957, 1287502745 ], "samples_ts": [ 99.2357, 99.1262, 99.4173 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q6_K.gguf", "model_type": "qwen35 4B Q6_K", "model_size": 3552021592, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-22T18:07:24Z", "avg_ns": 2576203704, "stddev_ns": 5537829, "avg_ts": 99.37133, "stddev_ts": 0.21352, "samples_ns": [ 2570981520, 2582009948, 2575619646 ], "samples_ts": [ 99.5729, 99.1476, 99.3936 ] } ], "mtp_n0": { "prompt_tokens_per_second": 939.8, "decode_tokens_per_second": 90.6 }, "mtp_n2": { "prompt_tokens_per_second": 661.4, "decode_tokens_per_second": 115.4 }, "mtp_n3": { "prompt_tokens_per_second": 640.3, "decode_tokens_per_second": 112.7 }, "mtp_n4": { "prompt_tokens_per_second": 655.0, "decode_tokens_per_second": 100.4 } }, "Q8_0": { "baseline": [ { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 4B Q8_0", "model_size": 4599562432, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:07:51Z", "avg_ns": 201969284, "stddev_ns": 127466427, "avg_ts": 3147.645337, "stddev_ts": 1456.00331, "samples_ns": [ 349154829, 128335855, 128417168 ], "samples_ts": [ 1466.4, 3989.53, 3987.01 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 4B Q8_0", "model_size": 4599562432, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 1024, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:07:51Z", "avg_ns": 183554313, "stddev_ns": 3499307, "avg_ts": 5580.069014, "stddev_ts": 105.338353, "samples_ns": [ 187549494, 182080046, 181033400 ], "samples_ts": [ 5459.89, 5623.9, 5656.41 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 4B Q8_0", "model_size": 4599562432, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 2048, "n_gen": 0, "n_depth": 0, "test_time": "2026-09-22T18:07:52Z", "avg_ns": 374194528, "stddev_ns": 1204705, "avg_ts": 5473.126933, "stddev_ts": 17.622062, "samples_ns": [ 374247151, 372964529, 375371905 ], "samples_ts": [ 5472.32, 5491.14, 5455.92 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 4B Q8_0", "model_size": 4599562432, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2026-09-22T18:07:54Z", "avg_ns": 1429022381, "stddev_ns": 2036861, "avg_ts": 89.571846, "stddev_ts": 0.127545, "samples_ns": [ 1431372231, 1427769796, 1427925117 ], "samples_ts": [ 89.4247, 89.6503, 89.6406 ] }, { "build_commit": "f280b26", "build_number": 1, "cpu_info": "AMD Ryzen 7 7700 8-Core Processor", "gpu_info": "AMD Radeon AI Pro R9700", "backends": "ROCm", "model_filename": "/mnt/data/models/Qwen3.5-4B-OnlyText-GGUF/Qwen3.5-4B-OnlyText-MTP-Q8_0.gguf", "model_type": "qwen35 4B Q8_0", "model_size": 4599562432, "model_n_params": 4326304768, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 256, "n_depth": 0, "test_time": "2026-09-22T18:07:58Z", "avg_ns": 2843873461, "stddev_ns": 3672408, "avg_ts": 90.01817, "stddev_ts": 0.116159, "samples_ns": [ 2848105159, 2841520682, 2841994542 ], "samples_ts": [ 89.8843, 90.0926, 90.0776 ] } ], "mtp_n0": { "prompt_tokens_per_second": 1137.0, "decode_tokens_per_second": 83.7 }, "mtp_n2": { "prompt_tokens_per_second": 818.8, "decode_tokens_per_second": 113.4 }, "mtp_n3": { "prompt_tokens_per_second": 828.9, "decode_tokens_per_second": 107.7 }, "mtp_n4": { "prompt_tokens_per_second": 787.7, "decode_tokens_per_second": 104.7 } } }, "dual": {} }, "ppl": { "Q4_K_M": 9.609, "Q6_K": 9.4519, "Q8_0": 9.3587 }, "quant_kl": { "Q4_K_M": { "mean_kld": 0.063803, "median_kld": 0.022137, "max_kld": 22.240232, "same_top_p": 90.411 }, "Q6_K": { "mean_kld": 0.010235, "median_kld": 0.002395, "max_kld": 14.572444, "same_top_p": 96.487 }, "Q8_0": { "mean_kld": 0.005509, "median_kld": 0.000555, "max_kld": 18.441305, "same_top_p": 98.371 } } }