diff --git "a/benchmarks.json" "b/benchmarks.json" new file mode 100644--- /dev/null +++ "b/benchmarks.json" @@ -0,0 +1,10012 @@ +[ + { + "round": "airllm", + "source_file": "airllm_14b_4bit_code_edit_warm.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "airllm", + "engine_version": "4.0.0", + "git_rev": null, + "mode": "airllm_4bit_prefetch", + "status": "ok", + "model": "Qwen/Qwen2.5-14B-Instruct", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 113, + "completion_tokens": 8, + "load_time_s": 2.55, + "ttft_s": 33.371, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 214.529, + "total_time_s": 247.9, + "decode_tok_per_s": 0.033, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.032, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 3536.4, + "current_rss_mib": 626.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": 1936.2, + "page_cache_delta_mib": -503.0, + "page_cache_delta_since_start_mib": -458.6, + "idle_watts": null, + "avg_watts": 9.3, + "net_watts": null, + "joules_per_tok": 288.21, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct" + }, + { + "round": "airllm", + "source_file": "airllm_14b_4bit_open_ended_warm.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "airllm", + "engine_version": "4.0.0", + "git_rev": null, + "mode": "airllm_4bit_prefetch", + "status": "ok", + "model": "Qwen/Qwen2.5-14B-Instruct", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 8, + "load_time_s": 2.573, + "ttft_s": 32.863, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 213.357, + "total_time_s": 246.219, + "decode_tok_per_s": 0.033, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.032, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 3531.2, + "current_rss_mib": 617.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": 1920.8, + "page_cache_delta_mib": -640.9, + "page_cache_delta_since_start_mib": -591.8, + "idle_watts": null, + "avg_watts": 9.33, + "net_watts": null, + "joules_per_tok": 287.1, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When discussing the interaction between page-aligned O" + }, + { + "round": "airllm", + "source_file": "airllm_14b_4bit_prompt_heavy.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "airllm", + "engine_version": "4.0.0", + "git_rev": null, + "mode": "airllm_4bit_prefetch", + "status": "ok", + "model": "Qwen/Qwen2.5-14B-Instruct", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 365.529, + "ttft_s": 32.093, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 954.918, + "total_time_s": 987.011, + "decode_tok_per_s": 0.032, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.032, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5390.5, + "current_rss_mib": 689.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": 1983.3, + "page_cache_delta_mib": 248.0, + "page_cache_delta_since_start_mib": -751.8, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described is layer-streaming inference, which minimizes memory usage by allocating a small fixed pool of reusable buffers " + }, + { + "round": "airllm", + "source_file": "airllm_14b_bf16_open_ended.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "airllm", + "engine_version": "4.0.0", + "git_rev": null, + "mode": "airllm_none_prefetch", + "status": "ok", + "model": "Qwen/Qwen2.5-14B-Instruct", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 4, + "load_time_s": 81.81, + "ttft_s": 45.413, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 131.774, + "total_time_s": 177.187, + "decode_tok_per_s": 0.023, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.023, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5217.6, + "current_rss_mib": 3489.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": 1502.0, + "page_cache_delta_mib": -4310.7, + "page_cache_delta_since_start_mib": -3140.9, + "idle_watts": null, + "avg_watts": 9.31, + "net_watts": null, + "joules_per_tok": 412.35, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " In the context of" + }, + { + "round": "competitors", + "source_file": "hf_accelerate_14b_4bit.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "hf-accelerate", + "engine_version": "transformers-5.23.0+bnb-0.50.2", + "git_rev": null, + "mode": "hf_accelerate_14b_4bit_disk", + "status": "timeout_300s", + "model": "Qwen2.5-14B-Instruct", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": 144.0, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": null, + "total_time_s": 300.0, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": null, + "current_rss_mib": null, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": null, + "page_cache_delta_since_start_mib": null, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": "Loaded and disk-offloaded 579 tensors (4-bit NF4, gpu_max_mem=3GiB, cpu_max_mem=1GiB) in 144.0s (2m24s), then hit 300s wall-clock timeout during generation of 16 tokens. PyTorch 2.14.1+cu126 wheel warns Orin CC 8.7 unsupported (same venv as AirLLM 4.0.0).", + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_cpu_mmap.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_cpu_mmap", + "status": "timeout", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 300.458, + "total_time_s": 300.458, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 6109.6, + "current_rss_mib": null, + "mem_avail_before_mib": 5040.2, + "mem_avail_after_mib": null, + "swap_delta_mib": 362.6, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 5648.3, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.87, + "avg_watts": 10.61, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cru" + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl0", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 22.832, + "total_time_s": 22.832, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5991.0, + "current_rss_mib": null, + "mem_avail_before_mib": 4984.7, + "mem_avail_after_mib": null, + "swap_delta_mib": 3683.5, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 135.7, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.79, + "avg_watts": 9.9, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl12_b512.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl12_b512", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 23.562, + "total_time_s": 23.562, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5977.9, + "current_rss_mib": null, + "mem_avail_before_mib": 5021.0, + "mem_avail_after_mib": null, + "swap_delta_mib": 3676.6, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 76.5, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.97, + "avg_watts": 9.84, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl12_b512.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl12_b512", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 23.752, + "total_time_s": 23.752, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5742.9, + "current_rss_mib": null, + "mem_avail_before_mib": 5163.8, + "mem_avail_after_mib": null, + "swap_delta_mib": 3684.2, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -5.9, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.97, + "avg_watts": 9.89, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl12_b512.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl12_b512", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 23.445, + "total_time_s": 23.445, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5723.5, + "current_rss_mib": null, + "mem_avail_before_mib": 5125.5, + "mem_avail_after_mib": null, + "swap_delta_mib": 3710.8, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 53.4, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.97, + "avg_watts": 9.95, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl18.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl18", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 56.129, + "total_time_s": 56.129, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5776.3, + "current_rss_mib": null, + "mem_avail_before_mib": 4972.8, + "mem_avail_after_mib": null, + "swap_delta_mib": 3634.6, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -2317.2, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.92, + "avg_watts": 9.23, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl18.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl18", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 22.412, + "total_time_s": 22.412, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5736.2, + "current_rss_mib": null, + "mem_avail_before_mib": 5027.8, + "mem_avail_after_mib": null, + "swap_delta_mib": 3626.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 114.3, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.92, + "avg_watts": 9.97, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_14b_ngl18.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_14b_ngl18", + "status": "error_rc_1", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 54.08, + "total_time_s": 54.08, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5715.3, + "current_rss_mib": null, + "mem_avail_before_mib": 5078.5, + "mem_avail_after_mib": null, + "swap_delta_mib": 3556.4, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -86.6, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.92, + "avg_watts": 8.73, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_32b_ngl10.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_32b_ngl10", + "status": "timeout", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 302.828, + "total_time_s": 302.828, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 5714.7, + "current_rss_mib": null, + "mem_avail_before_mib": 5176.0, + "mem_avail_after_mib": null, + "swap_delta_mib": 17242.8, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -305.4, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.89, + "avg_watts": 10.29, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "When streaming transformer layers" + }, + { + "round": "competitors", + "source_file": "llama_cpp_qwen3_30b_cpu_mmap.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_qwen3_30b_cpu_mmap", + "status": "error_rc_1", + "model": "Qwen3-30B-A3B-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 66.223, + "total_time_s": 66.223, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 6044.1, + "current_rss_mib": null, + "mem_avail_before_mib": 5115.9, + "mem_avail_after_mib": null, + "swap_delta_mib": 12324.5, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -62.8, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.92, + "avg_watts": 10.16, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "competitors", + "source_file": "llama_cpp_qwen3_30b_cpu_mmap_b512.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_qwen3_30b_cpu_mmap_b512", + "status": "ok", + "model": "Qwen3-30B-A3B-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 23, + "load_time_s": 25.34, + "ttft_s": 17.367, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 18.113, + "total_time_s": 62.445, + "decode_tok_per_s": 1.27, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.648, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 6070.9, + "current_rss_mib": null, + "mem_avail_before_mib": 6137.6, + "mem_avail_after_mib": null, + "swap_delta_mib": 247.9, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -100.1, + "page_cache_delta_since_start_mib": null, + "idle_watts": 8.07, + "avg_watts": 9.89, + "net_watts": null, + "joules_per_tok": 15.26, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "the problem is that the GPU is not able to read the data from the host memory as it is being written by the" + }, + { + "round": "competitors", + "source_file": "llama_cpp_qwen3_30b_gpu_offload.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "llama.cpp", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "llama_cpp_qwen3_30b_gpu_moe_offload", + "status": "error_rc_1", + "model": "Qwen3-30B-A3B-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": "q8_0", + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": 0, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 54.427, + "total_time_s": 54.427, + "decode_tok_per_s": 0.0, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.0, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": 6014.0, + "current_rss_mib": null, + "mem_avail_before_mib": 6081.8, + "mem_avail_after_mib": null, + "swap_delta_mib": 12342.2, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 81.1, + "page_cache_delta_since_start_mib": null, + "idle_watts": 7.95, + "avg_watts": 10.12, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": null + }, + { + "round": "results", + "source_file": "r0_prompt_heavy.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 2.968, + "ttft_s": 9.257, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 228.946, + "total_time_s": 238.203, + "decode_tok_per_s": 0.135, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.134, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 241263.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3225.7, + "current_rss_mib": 3225.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 37.9, + "page_cache_delta_since_start_mib": -764.6, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" + }, + { + "round": "results", + "source_file": "r14_draft_prompt_heavy.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 52.833, + "ttft_s": 5.752, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 39.649, + "total_time_s": 45.401, + "decode_tok_per_s": 0.782, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.705, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 63, + "accepted_tokens": 17, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 79826.54, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5877.1, + "current_rss_mib": 5666.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -251.4, + "page_cache_delta_since_start_mib": -2667.7, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round2", + "source_file": "hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 53.386, + "ttft_s": 5.665, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 41.96, + "total_time_s": 47.624, + "decode_tok_per_s": 0.739, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.672, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 63, + "accepted_tokens": 17, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 79826.54, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6257.4, + "current_rss_mib": 5668.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -198.3, + "page_cache_delta_since_start_mib": -2235.9, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round2", + "source_file": "hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 53.386, + "ttft_s": 3.438, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 24.011, + "total_time_s": 27.449, + "decode_tok_per_s": 1.957, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.749, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 9, + "drafted_tokens": 40, + "accepted_tokens": 39, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 47895.93, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6257.4, + "current_rss_mib": 5669.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -4.3, + "page_cache_delta_since_start_mib": -2240.2, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round2", + "source_file": "hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 53.386, + "ttft_s": 3.155, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 36.017, + "total_time_s": 39.172, + "decode_tok_per_s": 1.305, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.225, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 13, + "drafted_tokens": 59, + "accepted_tokens": 32, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 69183.0, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6257.4, + "current_rss_mib": 5669.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -14.6, + "page_cache_delta_since_start_mib": -2254.7, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round2", + "source_file": "hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 75.668, + "ttft_s": 5.074, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 63.663, + "total_time_s": 68.736, + "decode_tok_per_s": 0.487, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.466, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 25, + "drafted_tokens": 26, + "accepted_tokens": 7, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 125190.63, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5982.3, + "current_rss_mib": 5152.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 57.9, + "page_cache_delta_since_start_mib": -181.0, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" + }, + { + "round": "round2", + "source_file": "hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 75.668, + "ttft_s": 3.087, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 23.822, + "total_time_s": 26.909, + "decode_tok_per_s": 1.973, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.784, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 10, + "drafted_tokens": 45, + "accepted_tokens": 38, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 50076.25, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5982.3, + "current_rss_mib": 5153.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 3.7, + "page_cache_delta_since_start_mib": -177.3, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round2", + "source_file": "hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 75.668, + "ttft_s": 2.75, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 79.476, + "total_time_s": 82.225, + "decode_tok_per_s": 0.591, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.584, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 31, + "drafted_tokens": 25, + "accepted_tokens": 17, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 155236.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5982.3, + "current_rss_mib": 5155.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 2.4, + "page_cache_delta_since_start_mib": -174.9, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round2", + "source_file": "stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 2.4, + "ttft_s": 9.753, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 247.344, + "total_time_s": 257.097, + "decode_tok_per_s": 0.125, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.124, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 241263.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3249.2, + "current_rss_mib": 3249.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 2.0, + "page_cache_delta_since_start_mib": 2.3, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" + }, + { + "round": "round2", + "source_file": "stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 2.4, + "ttft_s": 8.395, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 380.266, + "total_time_s": 388.66, + "decode_tok_per_s": 0.124, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.124, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 48, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 361895.06, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3249.2, + "current_rss_mib": 3249.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 2.1, + "page_cache_delta_since_start_mib": 4.4, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round2", + "source_file": "stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 2.4, + "ttft_s": 8.088, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 385.094, + "total_time_s": 393.183, + "decode_tok_per_s": 0.122, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.122, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 48, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 361895.06, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3249.2, + "current_rss_mib": 3249.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 9.7, + "page_cache_delta_since_start_mib": 14.1, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round2", + "source_file": "stream_r0_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 2.102, + "ttft_s": 10.312, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 204.874, + "total_time_s": 215.185, + "decode_tok_per_s": 0.151, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.149, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 25, + "drafted_tokens": 26, + "accepted_tokens": 7, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 188487.01, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3253.6, + "current_rss_mib": 3253.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 34.6, + "page_cache_delta_since_start_mib": 46.1, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" + }, + { + "round": "round2", + "source_file": "stream_r0_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 2.102, + "ttft_s": 8.983, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 76.835, + "total_time_s": 85.818, + "decode_tok_per_s": 0.612, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.559, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 10, + "drafted_tokens": 45, + "accepted_tokens": 38, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 75394.8, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3253.6, + "current_rss_mib": 3253.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 3.6, + "page_cache_delta_since_start_mib": 49.7, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round2", + "source_file": "stream_r0_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "stream_r0_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 2.102, + "ttft_s": 8.666, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 256.328, + "total_time_s": 264.994, + "decode_tok_per_s": 0.183, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.181, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 31, + "drafted_tokens": 25, + "accepted_tokens": 17, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 233723.89, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3255.7, + "current_rss_mib": 3255.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 39.6, + "page_cache_delta_since_start_mib": 89.3, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round3", + "source_file": "r3_hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r3_hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.886, + "ttft_s": 5.916, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 69.787, + "total_time_s": 75.704, + "decode_tok_per_s": 0.444, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.423, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 62, + "accepted_tokens": 16, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 79826.54, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6079.0, + "current_rss_mib": 6079.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -57.5, + "page_cache_delta_since_start_mib": -1016.1, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round3", + "source_file": "r3_hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r3_hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.886, + "ttft_s": 3.49, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 18.261, + "total_time_s": 21.75, + "decode_tok_per_s": 2.574, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.207, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 31930.62, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6083.3, + "current_rss_mib": 6083.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.3, + "page_cache_delta_since_start_mib": -1015.8, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round3", + "source_file": "r3_hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r3_hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.886, + "ttft_s": 3.289, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 31.885, + "total_time_s": 35.174, + "decode_tok_per_s": 1.474, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.365, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 64, + "accepted_tokens": 34, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 58539.46, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6083.6, + "current_rss_mib": 6083.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 1.2, + "page_cache_delta_since_start_mib": -1014.6, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round3", + "source_file": "r3_stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r3_stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 1.859, + "ttft_s": 7.419, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 260.939, + "total_time_s": 268.358, + "decode_tok_per_s": 0.119, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.119, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 241263.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3168.0, + "current_rss_mib": 3168.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.6, + "page_cache_delta_since_start_mib": 1.2, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round4", + "source_file": "r4_auto_budget_5800_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_auto_budget_5800_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 8.611, + "ttft_s": 4.823, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 38.613, + "total_time_s": 43.437, + "decode_tok_per_s": 0.803, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.737, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 62, + "accepted_tokens": 16, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 68186.31, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6482.0, + "current_rss_mib": 5978.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 7.5, + "page_cache_delta_since_start_mib": -178.0, + "idle_watts": null, + "avg_watts": 12.65, + "net_watts": null, + "joules_per_tok": 17.17, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round4", + "source_file": "r4_auto_budget_5800_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_auto_budget_5800_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 8.611, + "ttft_s": 3.137, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 13.633, + "total_time_s": 16.77, + "decode_tok_per_s": 3.447, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.862, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 27274.52, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6482.0, + "current_rss_mib": 5971.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -1.5, + "page_cache_delta_since_start_mib": -179.5, + "idle_watts": null, + "avg_watts": 12.59, + "net_watts": null, + "joules_per_tok": 4.4, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round4", + "source_file": "r4_auto_budget_5800_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_auto_budget_5800_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 8.611, + "ttft_s": 2.907, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 27.967, + "total_time_s": 30.874, + "decode_tok_per_s": 1.681, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.555, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 81, + "accepted_tokens": 34, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 50003.29, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6482.0, + "current_rss_mib": 5939.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -23.4, + "page_cache_delta_since_start_mib": -202.8, + "idle_watts": null, + "avg_watts": 12.2, + "net_watts": null, + "joules_per_tok": 7.85, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 7.8, + "ttft_s": 5.093, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 39.028, + "total_time_s": 44.12, + "decode_tok_per_s": 0.794, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.725, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 62, + "accepted_tokens": 16, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 79992.96, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5708.1, + "current_rss_mib": 5348.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -87.8, + "page_cache_delta_since_start_mib": -690.5, + "idle_watts": null, + "avg_watts": 12.85, + "net_watts": null, + "joules_per_tok": 17.72, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 7.8, + "ttft_s": 3.155, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 13.958, + "total_time_s": 17.113, + "decode_tok_per_s": 3.367, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.805, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 31930.62, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5708.1, + "current_rss_mib": 5352.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -14.7, + "page_cache_delta_since_start_mib": -705.2, + "idle_watts": null, + "avg_watts": 12.36, + "net_watts": null, + "joules_per_tok": 4.41, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r14_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r14_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 7.8, + "ttft_s": 2.936, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 28.241, + "total_time_s": 31.177, + "decode_tok_per_s": 1.664, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.54, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 81, + "accepted_tokens": 34, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 58539.46, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5708.1, + "current_rss_mib": 5353.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 5.8, + "page_cache_delta_since_start_mib": -699.4, + "idle_watts": null, + "avg_watts": 12.27, + "net_watts": null, + "joules_per_tok": 7.97, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 5.469, + "ttft_s": 4.708, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 38.462, + "total_time_s": 43.17, + "decode_tok_per_s": 0.806, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.741, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 62, + "accepted_tokens": 16, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 75114.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6201.6, + "current_rss_mib": 6201.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -54.8, + "page_cache_delta_since_start_mib": -451.7, + "idle_watts": null, + "avg_watts": 12.67, + "net_watts": null, + "joules_per_tok": 17.09, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 5.469, + "ttft_s": 3.142, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 13.715, + "total_time_s": 16.857, + "decode_tok_per_s": 3.427, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.848, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 30193.48, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.8, + "current_rss_mib": 6198.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -11.1, + "page_cache_delta_since_start_mib": -462.8, + "idle_watts": null, + "avg_watts": 12.09, + "net_watts": null, + "joules_per_tok": 4.25, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 5.469, + "ttft_s": 2.91, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 27.794, + "total_time_s": 30.704, + "decode_tok_per_s": 1.691, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.563, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 81, + "accepted_tokens": 34, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55083.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.8, + "current_rss_mib": 6179.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 7.0, + "page_cache_delta_since_start_mib": -455.8, + "idle_watts": null, + "avg_watts": 12.86, + "net_watts": null, + "joules_per_tok": 8.23, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b_t2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b_t2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 5.178, + "ttft_s": 5.791, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 53.552, + "total_time_s": 59.343, + "decode_tok_per_s": 0.579, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.539, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 62, + "accepted_tokens": 16, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 75262.1, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6199.6, + "current_rss_mib": 6199.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -53.2, + "page_cache_delta_since_start_mib": 4.1, + "idle_watts": null, + "avg_watts": 11.14, + "net_watts": null, + "joules_per_tok": 20.65, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b_t2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b_t2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 5.178, + "ttft_s": 3.989, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 18.402, + "total_time_s": 22.391, + "decode_tok_per_s": 2.554, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.144, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 29898.02, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.1, + "current_rss_mib": 6203.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -18.3, + "page_cache_delta_since_start_mib": -14.1, + "idle_watts": null, + "avg_watts": 11.43, + "net_watts": null, + "joules_per_tok": 5.33, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b_t2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b_t2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 5.178, + "ttft_s": 3.805, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 38.511, + "total_time_s": 42.316, + "decode_tok_per_s": 1.22, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.134, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 81, + "accepted_tokens": 34, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55083.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.1, + "current_rss_mib": 6203.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -9.2, + "page_cache_delta_since_start_mib": -23.4, + "idle_watts": null, + "avg_watts": 11.34, + "net_watts": null, + "joules_per_tok": 10.0, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b_t4.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b_t4", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 5.561, + "ttft_s": 5.592, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 51.504, + "total_time_s": 57.095, + "decode_tok_per_s": 0.602, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.56, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 15, + "drafted_tokens": 62, + "accepted_tokens": 16, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 75262.1, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.0, + "current_rss_mib": 6186.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -245.6, + "page_cache_delta_since_start_mib": -166.0, + "idle_watts": null, + "avg_watts": 11.47, + "net_watts": null, + "joules_per_tok": 20.47, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b_t4.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b_t4", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 5.561, + "ttft_s": 6.269, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 22.093, + "total_time_s": 28.362, + "decode_tok_per_s": 2.127, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.692, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 30045.75, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.0, + "current_rss_mib": 6174.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 3.3, + "page_cache_delta_since_start_mib": -162.6, + "idle_watts": null, + "avg_watts": 10.5, + "net_watts": null, + "joules_per_tok": 6.21, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_draft05b_t4.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_draft05b_t4", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 5.561, + "ttft_s": 3.665, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 65.359, + "total_time_s": 69.025, + "decode_tok_per_s": 0.719, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.695, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 81, + "accepted_tokens": 34, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55083.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6203.0, + "current_rss_mib": 6172.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 3.4, + "page_cache_delta_since_start_mib": -159.2, + "idle_watts": null, + "avg_watts": 9.77, + "net_watts": null, + "joules_per_tok": 14.04, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 4.735, + "ttft_s": 7.575, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 147.639, + "total_time_s": 155.213, + "decode_tok_per_s": 0.21, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.206, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 25, + "drafted_tokens": 29, + "accepted_tokens": 7, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 125190.63, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5596.0, + "current_rss_mib": 5596.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 58.6, + "page_cache_delta_since_start_mib": 73.6, + "idle_watts": null, + "avg_watts": 9.86, + "net_watts": null, + "joules_per_tok": 47.84, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping critical layers in memory, and asynchrono" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 4.735, + "ttft_s": 6.145, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 28.747, + "total_time_s": 34.892, + "decode_tok_per_s": 1.635, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.376, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 43, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 30045.75, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5599.5, + "current_rss_mib": 5599.5, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 12.2, + "page_cache_delta_since_start_mib": 85.7, + "idle_watts": null, + "avg_watts": 10.26, + "net_watts": null, + "joules_per_tok": 7.45, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round4", + "source_file": "r4_hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 4.735, + "ttft_s": 5.783, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 196.684, + "total_time_s": 202.467, + "decode_tok_per_s": 0.239, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.237, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 30, + "drafted_tokens": 32, + "accepted_tokens": 18, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 150228.75, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5601.2, + "current_rss_mib": 5164.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 166.6, + "page_cache_delta_since_start_mib": 252.4, + "idle_watts": null, + "avg_watts": 9.69, + "net_watts": null, + "joules_per_tok": 40.88, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round4", + "source_file": "r4_stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r4_stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 2.595, + "ttft_s": 10.378, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 265.468, + "total_time_s": 275.846, + "decode_tok_per_s": 0.117, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.116, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 241429.79, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 3073.8, + "current_rss_mib": 3073.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 21.8, + "page_cache_delta_since_start_mib": 35.4, + "idle_watts": null, + "avg_watts": 9.86, + "net_watts": null, + "joules_per_tok": 85.0, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_auto_budget_5800_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_auto_budget_5800_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.523, + "ttft_s": 5.366, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 51.725, + "total_time_s": 57.091, + "decode_tok_per_s": 0.599, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.561, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 80269.73, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5580.8, + "current_rss_mib": 5580.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -58.6, + "page_cache_delta_since_start_mib": 369.9, + "idle_watts": null, + "avg_watts": 11.19, + "net_watts": null, + "joules_per_tok": 19.97, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_auto_budget_5800_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_auto_budget_5800_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.523, + "ttft_s": 3.662, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 13.455, + "total_time_s": 17.117, + "decode_tok_per_s": 3.493, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.804, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 24890.4, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5584.6, + "current_rss_mib": 5584.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -12.0, + "page_cache_delta_since_start_mib": 357.9, + "idle_watts": null, + "avg_watts": 11.84, + "net_watts": null, + "joules_per_tok": 4.22, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_auto_budget_5800_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_auto_budget_5800_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.523, + "ttft_s": 3.534, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 33.908, + "total_time_s": 37.442, + "decode_tok_per_s": 1.386, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.282, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 115, + "accepted_tokens": 34, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55231.6, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5584.6, + "current_rss_mib": 5584.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -12.8, + "page_cache_delta_since_start_mib": 345.1, + "idle_watts": null, + "avg_watts": 11.27, + "net_watts": null, + "joules_per_tok": 8.79, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r14_draft15b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r14_draft15b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 7.065, + "ttft_s": 7.763, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 83.656, + "total_time_s": 91.419, + "decode_tok_per_s": 0.371, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.35, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 103, + "accepted_tokens": 16, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 85314.73, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5865.6, + "current_rss_mib": 5865.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -41.6, + "page_cache_delta_since_start_mib": -289.5, + "idle_watts": null, + "avg_watts": 10.33, + "net_watts": null, + "joules_per_tok": 29.51, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r14_draft15b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r14_draft15b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 7.065, + "ttft_s": 5.808, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 22.176, + "total_time_s": 27.984, + "decode_tok_per_s": 2.119, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.715, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 26608.85, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5870.3, + "current_rss_mib": 5870.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 4.3, + "page_cache_delta_since_start_mib": -285.2, + "idle_watts": null, + "avg_watts": 9.81, + "net_watts": null, + "joules_per_tok": 5.72, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r14_draft15b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r14_draft15b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 7.065, + "ttft_s": 5.726, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 50.56, + "total_time_s": 56.285, + "decode_tok_per_s": 0.93, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.853, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 10, + "drafted_tokens": 94, + "accepted_tokens": 35, + "resident_layers": 14, + "streamed_layers": 34, + "n_layers": 48, + "resident_mib": 3244.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 53051.28, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5870.3, + "current_rss_mib": 5870.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -14.2, + "page_cache_delta_since_start_mib": -299.4, + "idle_watts": null, + "avg_watts": 10.43, + "net_watts": null, + "joules_per_tok": 12.23, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.533, + "ttft_s": 5.432, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 50.131, + "total_time_s": 55.563, + "decode_tok_per_s": 0.618, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.576, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 80269.73, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5542.5, + "current_rss_mib": 5542.5, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -51.2, + "page_cache_delta_since_start_mib": -1137.8, + "idle_watts": null, + "avg_watts": 11.45, + "net_watts": null, + "joules_per_tok": 19.88, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.533, + "ttft_s": 3.664, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 13.386, + "total_time_s": 17.05, + "decode_tok_per_s": 3.511, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.815, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 25038.13, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5547.2, + "current_rss_mib": 5547.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -13.0, + "page_cache_delta_since_start_mib": -1150.8, + "idle_watts": null, + "avg_watts": 10.73, + "net_watts": null, + "joules_per_tok": 3.81, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.533, + "ttft_s": 3.452, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 33.828, + "total_time_s": 37.281, + "decode_tok_per_s": 1.389, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.288, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 115, + "accepted_tokens": 34, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55083.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5547.2, + "current_rss_mib": 5547.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 6.8, + "page_cache_delta_since_start_mib": -1144.0, + "idle_watts": null, + "avg_watts": 10.97, + "net_watts": null, + "joules_per_tok": 8.52, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.341, + "ttft_s": 7.225, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 52.134, + "total_time_s": 59.359, + "decode_tok_per_s": 0.595, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.539, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 75, + "accepted_tokens": 21, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55083.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5575.7, + "current_rss_mib": 5575.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -24.5, + "page_cache_delta_since_start_mib": 257.5, + "idle_watts": null, + "avg_watts": 10.25, + "net_watts": null, + "joules_per_tok": 19.02, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described in the document is layer-streaming inference, which manages to fit large language models into limited memory by " + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.341, + "ttft_s": 5.607, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 20.974, + "total_time_s": 26.581, + "decode_tok_per_s": 2.241, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.806, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 25038.13, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5580.9, + "current_rss_mib": 5580.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.2, + "page_cache_delta_since_start_mib": 257.7, + "idle_watts": null, + "avg_watts": 10.46, + "net_watts": null, + "joules_per_tok": 5.79, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.341, + "ttft_s": 5.345, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 52.259, + "total_time_s": 57.603, + "decode_tok_per_s": 0.899, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.833, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 107, + "accepted_tokens": 32, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 55083.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5580.9, + "current_rss_mib": 5580.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.2, + "page_cache_delta_since_start_mib": 257.9, + "idle_watts": null, + "avg_watts": 9.78, + "net_watts": null, + "joules_per_tok": 11.73, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers in the context of streaming transformer layers on an 8 GB edge " + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 5.227, + "ttft_s": 6.954, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 125.207, + "total_time_s": 132.161, + "decode_tok_per_s": 0.248, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.242, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 25, + "drafted_tokens": 37, + "accepted_tokens": 7, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 125338.35, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 4992.8, + "current_rss_mib": 4992.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.9, + "page_cache_delta_since_start_mib": -11.4, + "idle_watts": null, + "avg_watts": 9.98, + "net_watts": null, + "joules_per_tok": 41.23, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 5.227, + "ttft_s": 5.484, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 26.205, + "total_time_s": 31.689, + "decode_tok_per_s": 1.794, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.515, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 31, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 29898.02, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 4995.1, + "current_rss_mib": 4995.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.7, + "page_cache_delta_since_start_mib": -10.7, + "idle_watts": null, + "avg_watts": 9.8, + "net_watts": null, + "joules_per_tok": 6.47, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r16_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r16_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 5.227, + "ttft_s": 5.319, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 145.983, + "total_time_s": 151.302, + "decode_tok_per_s": 0.322, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.317, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 29, + "drafted_tokens": 30, + "accepted_tokens": 19, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 145368.85, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 4997.1, + "current_rss_mib": 4997.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 3.4, + "page_cache_delta_since_start_mib": -7.2, + "idle_watts": null, + "avg_watts": 9.8, + "net_watts": null, + "joules_per_tok": 30.88, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r19_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r19_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 7.737, + "ttft_s": 4.043, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 31.922, + "total_time_s": 35.965, + "decode_tok_per_s": 0.971, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.89, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 72732.06, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5890.0, + "current_rss_mib": 5890.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -77.4, + "page_cache_delta_since_start_mib": -448.5, + "idle_watts": null, + "avg_watts": 12.96, + "net_watts": null, + "joules_per_tok": 14.57, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r19_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r19_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 7.737, + "ttft_s": 2.45, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 8.27, + "total_time_s": 10.72, + "decode_tok_per_s": 5.684, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 4.478, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 22728.77, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5894.9, + "current_rss_mib": 5894.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.4, + "page_cache_delta_since_start_mib": -448.1, + "idle_watts": null, + "avg_watts": 12.7, + "net_watts": null, + "joules_per_tok": 2.84, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_hybrid_r19_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_hybrid_r19_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 7.737, + "ttft_s": 2.262, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 21.636, + "total_time_s": 23.898, + "decode_tok_per_s": 2.172, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.009, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 115, + "accepted_tokens": 34, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 50003.29, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5895.1, + "current_rss_mib": 5895.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -33.9, + "page_cache_delta_since_start_mib": -482.0, + "idle_watts": null, + "avg_watts": 12.48, + "net_watts": null, + "joules_per_tok": 6.21, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round5", + "source_file": "r5_zc_multiturn_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_multiturn_r16_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 5.897, + "ttft_s": 7.188, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 78.222, + "total_time_s": 85.41, + "decode_tok_per_s": 0.396, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.375, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 16, + "streamed_layers": 32, + "n_layers": 48, + "resident_mib": 3558.5, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 80122.0, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5574.5, + "current_rss_mib": 5574.5, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -74.1, + "page_cache_delta_since_start_mib": 16.3, + "idle_watts": null, + "avg_watts": 10.19, + "net_watts": null, + "joules_per_tok": 27.19, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_multiturn_r16_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_multiturn_r16_draft05b_multiturn", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "multi_turn_prefix_reuse", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": null, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": null, + "total_time_s": null, + "decode_tok_per_s": null, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": null, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": null, + "current_rss_mib": null, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": null, + "page_cache_delta_since_start_mib": null, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": 340, + "turn1_ttft_s": 5.174, + "turn2_prompt_tokens": 377, + "turn2_ttft_s": 5.4, + "ttft_speedup_x": 0.96, + "notes": null, + "output_preview": " O_DIRECT prevents page-cache thrashing by bypassing the operating system's page cache" + }, + { + "round": "round5", + "source_file": "r5_zc_stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 1.99, + "ttft_s": 6.96, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 163.279, + "total_time_s": 170.239, + "decode_tok_per_s": 0.19, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.188, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 241429.79, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2492.4, + "current_rss_mib": 2492.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.8, + "page_cache_delta_since_start_mib": 1.3, + "idle_watts": null, + "avg_watts": 10.71, + "net_watts": null, + "joules_per_tok": 56.97, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 1.99, + "ttft_s": 5.618, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 247.709, + "total_time_s": 253.327, + "decode_tok_per_s": 0.19, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.189, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 48, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 361728.64, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2492.4, + "current_rss_mib": 2492.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 36.1, + "page_cache_delta_since_start_mib": 37.4, + "idle_watts": null, + "avg_watts": 10.67, + "net_watts": null, + "joules_per_tok": 56.33, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_stream_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 1.99, + "ttft_s": 5.368, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 247.761, + "total_time_s": 253.129, + "decode_tok_per_s": 0.19, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.19, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 48, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 362061.48, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2492.4, + "current_rss_mib": 2492.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 26.8, + "page_cache_delta_since_start_mib": 64.2, + "idle_watts": null, + "avg_watts": 10.7, + "net_watts": null, + "joules_per_tok": 56.41, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round5", + "source_file": "r5_zc_stream_r0_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_stream_r0_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 2.011, + "ttft_s": 6.246, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 109.781, + "total_time_s": 116.027, + "decode_tok_per_s": 0.282, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.276, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 25, + "drafted_tokens": 37, + "accepted_tokens": 7, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 188653.43, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2500.8, + "current_rss_mib": 2500.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.9, + "page_cache_delta_since_start_mib": 1.3, + "idle_watts": null, + "avg_watts": 11.16, + "net_watts": null, + "joules_per_tok": 40.47, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round5", + "source_file": "r5_zc_stream_r0_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_stream_r0_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 2.011, + "ttft_s": 4.906, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 22.969, + "total_time_s": 27.875, + "decode_tok_per_s": 2.046, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.722, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 31, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 45236.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2503.1, + "current_rss_mib": 2503.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.2, + "page_cache_delta_since_start_mib": 1.5, + "idle_watts": null, + "avg_watts": 11.03, + "net_watts": null, + "joules_per_tok": 6.41, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round5", + "source_file": "r5_zc_stream_r0_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r5_zc_stream_r0_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 2.011, + "ttft_s": 4.612, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 128.056, + "total_time_s": 132.668, + "decode_tok_per_s": 0.367, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.362, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 29, + "drafted_tokens": 30, + "accepted_tokens": 19, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 218644.93, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2505.1, + "current_rss_mib": 2505.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 1.5, + "page_cache_delta_since_start_mib": 3.0, + "idle_watts": null, + "avg_watts": 10.97, + "net_watts": null, + "joules_per_tok": 30.31, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round6", + "source_file": "r6_auto_budget_6000_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_auto_budget_6000_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.615, + "ttft_s": 7.057, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 77.232, + "total_time_s": 84.289, + "decode_tok_per_s": 0.401, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.38, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 17, + "streamed_layers": 31, + "n_layers": 48, + "resident_mib": 3706.3, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 77758.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5715.0, + "current_rss_mib": 5715.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -91.1, + "page_cache_delta_since_start_mib": 151.9, + "idle_watts": null, + "avg_watts": 10.11, + "net_watts": null, + "joules_per_tok": 26.62, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round6", + "source_file": "r6_auto_budget_6000_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_auto_budget_6000_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.615, + "ttft_s": 5.474, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 20.38, + "total_time_s": 25.854, + "decode_tok_per_s": 2.306, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.857, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 17, + "streamed_layers": 31, + "n_layers": 48, + "resident_mib": 3706.3, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 24299.49, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5719.7, + "current_rss_mib": 5719.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.4, + "page_cache_delta_since_start_mib": 152.4, + "idle_watts": null, + "avg_watts": 10.14, + "net_watts": null, + "joules_per_tok": 5.46, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round6", + "source_file": "r6_auto_budget_6000_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_auto_budget_6000_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.615, + "ttft_s": 5.291, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 51.813, + "total_time_s": 57.105, + "decode_tok_per_s": 0.907, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.841, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 115, + "accepted_tokens": 34, + "resident_layers": 17, + "streamed_layers": 31, + "n_layers": 48, + "resident_mib": 3706.3, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 53458.88, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5719.7, + "current_rss_mib": 5719.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.1, + "page_cache_delta_since_start_mib": 152.5, + "idle_watts": null, + "avg_watts": 9.78, + "net_watts": null, + "joules_per_tok": 11.64, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r12_draft15b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r12_draft15b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.343, + "ttft_s": 8.156, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 88.085, + "total_time_s": 96.241, + "decode_tok_per_s": 0.352, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.332, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 103, + "accepted_tokens": 16, + "resident_layers": 12, + "streamed_layers": 36, + "n_layers": 48, + "resident_mib": 2949.0, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 90023.29, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5580.4, + "current_rss_mib": 5580.4, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -82.5, + "page_cache_delta_since_start_mib": 391.5, + "idle_watts": null, + "avg_watts": 10.05, + "net_watts": null, + "joules_per_tok": 30.22, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r12_draft15b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r12_draft15b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.343, + "ttft_s": 6.099, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 23.446, + "total_time_s": 29.545, + "decode_tok_per_s": 2.005, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.625, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 12, + "streamed_layers": 36, + "n_layers": 48, + "resident_mib": 2949.0, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 28086.11, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5585.0, + "current_rss_mib": 5585.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -69.6, + "page_cache_delta_since_start_mib": 321.9, + "idle_watts": null, + "avg_watts": 9.65, + "net_watts": null, + "joules_per_tok": 5.94, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r12_draft15b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r12_draft15b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.343, + "ttft_s": 6.014, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 63.741, + "total_time_s": 69.755, + "decode_tok_per_s": 0.737, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.688, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 10, + "drafted_tokens": 94, + "accepted_tokens": 35, + "resident_layers": 12, + "streamed_layers": 36, + "n_layers": 48, + "resident_mib": 2949.0, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 56172.23, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5585.0, + "current_rss_mib": 5585.0, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 2.4, + "page_cache_delta_since_start_mib": 324.3, + "idle_watts": null, + "avg_watts": 9.78, + "net_watts": null, + "joules_per_tok": 14.22, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 7.145, + "ttft_s": 3.976, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 20.457, + "total_time_s": 24.432, + "decode_tok_per_s": 1.515, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.31, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 75, + "accepted_tokens": 21, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 50003.29, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6037.1, + "current_rss_mib": 6037.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -33.1, + "page_cache_delta_since_start_mib": -382.9, + "idle_watts": null, + "avg_watts": 13.05, + "net_watts": null, + "joules_per_tok": 9.97, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described in the document is layer-streaming inference, which manages to fit large language models into limited memory by " + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 7.145, + "ttft_s": 2.358, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 7.928, + "total_time_s": 10.286, + "decode_tok_per_s": 5.928, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 4.666, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 22728.77, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6042.3, + "current_rss_mib": 6042.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -35.3, + "page_cache_delta_since_start_mib": -418.2, + "idle_watts": null, + "avg_watts": 13.81, + "net_watts": null, + "joules_per_tok": 2.96, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 7.145, + "ttft_s": 2.168, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 20.461, + "total_time_s": 22.629, + "decode_tok_per_s": 2.297, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.121, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 107, + "accepted_tokens": 32, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 50003.29, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6042.3, + "current_rss_mib": 6042.3, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 4.3, + "page_cache_delta_since_start_mib": -413.9, + "idle_watts": null, + "avg_watts": 13.11, + "net_watts": null, + "joules_per_tok": 6.18, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers in the context of streaming transformer layers on an 8 GB edge " + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_draft05b_th2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_draft05b_th2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 7.943, + "ttft_s": 3.957, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 31.355, + "total_time_s": 35.312, + "decode_tok_per_s": 0.989, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.906, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 72732.06, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5927.6, + "current_rss_mib": 5927.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -76.5, + "page_cache_delta_since_start_mib": -829.4, + "idle_watts": null, + "avg_watts": 12.61, + "net_watts": null, + "joules_per_tok": 13.92, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_draft05b_th2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_draft05b_th2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 7.943, + "ttft_s": 2.355, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 7.874, + "total_time_s": 10.23, + "decode_tok_per_s": 5.969, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 4.692, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 22728.77, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5932.5, + "current_rss_mib": 5932.5, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -11.2, + "page_cache_delta_since_start_mib": -840.6, + "idle_watts": null, + "avg_watts": 13.02, + "net_watts": null, + "joules_per_tok": 2.78, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_draft05b_th2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_draft05b_th2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 7.943, + "ttft_s": 2.189, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 32.682, + "total_time_s": 34.871, + "decode_tok_per_s": 1.438, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.377, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 115, + "accepted_tokens": 34, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 50003.29, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5932.7, + "current_rss_mib": 5932.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -2.4, + "page_cache_delta_since_start_mib": -843.0, + "idle_watts": null, + "avg_watts": 11.23, + "net_watts": null, + "joules_per_tok": 8.16, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.029, + "ttft_s": 4.353, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 64.64, + "total_time_s": 68.993, + "decode_tok_per_s": 0.48, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.464, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 25, + "drafted_tokens": 37, + "accepted_tokens": 7, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 113643.85, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5458.6, + "current_rss_mib": 5458.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 3.9, + "page_cache_delta_since_start_mib": -45.7, + "idle_watts": null, + "avg_watts": 11.94, + "net_watts": null, + "joules_per_tok": 25.74, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.029, + "ttft_s": 3.152, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 13.547, + "total_time_s": 16.699, + "decode_tok_per_s": 3.469, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 2.874, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 31, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 27274.52, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5460.9, + "current_rss_mib": 5460.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.7, + "page_cache_delta_since_start_mib": -45.0, + "idle_watts": null, + "avg_watts": 11.95, + "net_watts": null, + "joules_per_tok": 4.16, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r19_ngram.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r19_ngram", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.029, + "ttft_s": 2.87, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 75.455, + "total_time_s": 78.325, + "decode_tok_per_s": 0.623, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.613, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 29, + "drafted_tokens": 30, + "accepted_tokens": 19, + "resident_layers": 19, + "streamed_layers": 29, + "n_layers": 48, + "resident_mib": 4020.4, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 131826.86, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5463.0, + "current_rss_mib": 5462.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 22.1, + "page_cache_delta_since_start_mib": -23.0, + "idle_watts": null, + "avg_watts": 11.6, + "net_watts": null, + "joules_per_tok": 18.93, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r20_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r20_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.749, + "ttft_s": 4.078, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 30.016, + "total_time_s": 34.094, + "decode_tok_per_s": 1.033, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.939, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 16, + "drafted_tokens": 107, + "accepted_tokens": 16, + "resident_layers": 20, + "streamed_layers": 28, + "n_layers": 48, + "resident_mib": 4168.1, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 70534.86, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6152.5, + "current_rss_mib": 6069.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -74.6, + "page_cache_delta_since_start_mib": -171.5, + "idle_watts": null, + "avg_watts": 12.89, + "net_watts": null, + "joules_per_tok": 13.73, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r20_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r20_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.749, + "ttft_s": 2.305, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 7.868, + "total_time_s": 10.173, + "decode_tok_per_s": 5.974, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 4.718, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 36, + "resident_layers": 20, + "streamed_layers": 28, + "n_layers": 48, + "resident_mib": 4168.1, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 21990.14, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6152.5, + "current_rss_mib": 6074.6, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.7, + "page_cache_delta_since_start_mib": -170.8, + "idle_watts": null, + "avg_watts": 12.89, + "net_watts": null, + "joules_per_tok": 2.73, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round6", + "source_file": "r6_hybrid_r20_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_hybrid_r20_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.749, + "ttft_s": 2.124, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 22.288, + "total_time_s": 24.412, + "decode_tok_per_s": 2.109, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 1.966, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 115, + "accepted_tokens": 34, + "resident_layers": 20, + "streamed_layers": 28, + "n_layers": 48, + "resident_mib": 4168.1, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 48378.3, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 6152.5, + "current_rss_mib": 6074.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -2.6, + "page_cache_delta_since_start_mib": -173.4, + "idle_watts": null, + "avg_watts": 12.84, + "net_watts": null, + "joules_per_tok": 6.53, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round6", + "source_file": "r6_multiturn_long_r19_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_multiturn_long_r19_draft05b_multiturn", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "multi_turn_prefix_reuse", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": null, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": null, + "total_time_s": null, + "decode_tok_per_s": null, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": null, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": null, + "current_rss_mib": null, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": null, + "page_cache_delta_since_start_mib": null, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": 1377, + "turn1_ttft_s": 16.705, + "turn2_prompt_tokens": 1414, + "turn2_ttft_s": 3.416, + "ttft_speedup_x": 4.89, + "notes": null, + "output_preview": " O_DIRECT prevents page-cache thrashing by bypassing the operating system's page cache" + }, + { + "round": "round6", + "source_file": "r6_stream_r0_th2.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_stream_r0_th2", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 1.913, + "ttft_s": 7.374, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 174.278, + "total_time_s": 181.652, + "decode_tok_per_s": 0.178, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.176, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": null, + "io_mib_read": 241429.79, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2484.7, + "current_rss_mib": 2484.7, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 19.2, + "page_cache_delta_since_start_mib": 25.6, + "idle_watts": null, + "avg_watts": 10.58, + "net_watts": null, + "joules_per_tok": 60.05, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" + }, + { + "round": "round6_32b", + "source_file": "r6_32b_auto5600_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_auto5600_draft05b", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 6.838, + "ttft_s": 11.69, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 92.127, + "total_time_s": 103.817, + "decode_tok_per_s": 0.336, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.308, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 13, + "drafted_tokens": 87, + "accepted_tokens": 19, + "resident_layers": 6, + "streamed_layers": 58, + "n_layers": 64, + "resident_mib": 2813.1, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 209768.71, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5289.1, + "current_rss_mib": 5289.1, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -59.0, + "page_cache_delta_since_start_mib": -4749.6, + "idle_watts": null, + "avg_watts": 12.24, + "net_watts": null, + "joules_per_tok": 39.7, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described involves layer-streaming inference, which uses a small fixed pool of reusable compute buffers and asynchronously" + }, + { + "round": "round6_32b", + "source_file": "r6_32b_auto5600_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_auto5600_draft05b", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 6.838, + "ttft_s": 8.416, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 49.8, + "total_time_s": 58.216, + "decode_tok_per_s": 0.944, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.825, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 7, + "drafted_tokens": 66, + "accepted_tokens": 41, + "resident_layers": 6, + "streamed_layers": 58, + "n_layers": 64, + "resident_mib": 2813.1, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 112605.38, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5293.8, + "current_rss_mib": 5293.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -16.0, + "page_cache_delta_since_start_mib": -4765.5, + "idle_watts": null, + "avg_watts": 11.79, + "net_watts": null, + "joules_per_tok": 14.3, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\n pub model_path: PathBuf,\n pub n_ctx: u32,\n pub n_batch: u32,\n pub resident_layers: Option,\n " + }, + { + "round": "round6_32b", + "source_file": "r6_32b_auto5600_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_auto5600_draft05b", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 6.838, + "ttft_s": 7.916, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 92.905, + "total_time_s": 100.821, + "decode_tok_per_s": 0.506, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.476, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 13, + "drafted_tokens": 127, + "accepted_tokens": 35, + "resident_layers": 6, + "streamed_layers": 58, + "n_layers": 64, + "resident_mib": 2813.1, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 209657.66, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5293.8, + "current_rss_mib": 5293.8, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -0.1, + "page_cache_delta_since_start_mib": -4765.6, + "idle_watts": null, + "avg_watts": 11.96, + "net_watts": null, + "joules_per_tok": 25.12, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers can b" + }, + { + "round": "round6_32b", + "source_file": "r6_32b_auto5600_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_auto5600_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 5.459, + "ttft_s": 16.181, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 146.116, + "total_time_s": 162.297, + "decode_tok_per_s": 0.212, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.197, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 13, + "drafted_tokens": 89, + "accepted_tokens": 19, + "resident_layers": 6, + "streamed_layers": 58, + "n_layers": 64, + "resident_mib": 2813.1, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 209582.03, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5293.9, + "current_rss_mib": 5293.9, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -95.5, + "page_cache_delta_since_start_mib": 5.7, + "idle_watts": null, + "avg_watts": 10.75, + "net_watts": null, + "joules_per_tok": 54.52, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described involves layer-streaming inference, where a fixed pool of reusable compute buffers is allocated for transformer " + }, + { + "round": "round6_32b", + "source_file": "r6_32b_auto5600_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_auto5600_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 5.459, + "ttft_s": 12.943, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 73.28, + "total_time_s": 86.223, + "decode_tok_per_s": 0.641, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.557, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 7, + "drafted_tokens": 66, + "accepted_tokens": 41, + "resident_layers": 6, + "streamed_layers": 58, + "n_layers": 64, + "resident_mib": 2813.1, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 112792.07, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5299.2, + "current_rss_mib": 5299.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.0, + "page_cache_delta_since_start_mib": 5.7, + "idle_watts": null, + "avg_watts": 10.84, + "net_watts": null, + "joules_per_tok": 19.47, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\n pub model_path: PathBuf,\n pub n_ctx: u32,\n pub n_batch: u32,\n pub resident_layers: Option,\n " + }, + { + "round": "round6_32b", + "source_file": "r6_32b_auto5600_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_auto5600_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 5.459, + "ttft_s": 12.177, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 121.648, + "total_time_s": 133.825, + "decode_tok_per_s": 0.386, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.359, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 11, + "drafted_tokens": 98, + "accepted_tokens": 37, + "resident_layers": 6, + "streamed_layers": 58, + "n_layers": 64, + "resident_mib": 2813.1, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 177244.68, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 5299.2, + "current_rss_mib": 5299.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -16.2, + "page_cache_delta_since_start_mib": -10.5, + "idle_watts": null, + "avg_watts": 10.53, + "net_watts": null, + "joules_per_tok": 29.35, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " In the context of streaming transformer layers on an 8 GB edge device, understanding how page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffer" + }, + { + "round": "round6_32b", + "source_file": "r6_32b_stream_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r6_32b_stream_r0", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 2.137, + "ttft_s": 14.479, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": 330.254, + "total_time_s": 344.733, + "decode_tok_per_s": 0.094, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": 0.093, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 64, + "n_layers": 64, + "resident_mib": 1026.8, + "slot_mib": 297.7, + "expert_slot_mib": null, + "io_mib_read": 573082.1, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": false, + "is_unified_memory": null, + "direct_io_active": true, + "peak_rss_mib": 2957.2, + "current_rss_mib": 2957.2, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 6.9, + "page_cache_delta_since_start_mib": 8.2, + "idle_watts": null, + "avg_watts": 11.06, + "net_watts": null, + "joules_per_tok": 119.1, + "net_joules_per_tok": null, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described involves layer-streaming inference, which uses a small fixed pool of reusable compute buffers and asynchronously" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": true, + "vram_budget_mb": 0, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 4.936, + "ttft_s": 4.467, + "ttft_s_min": 4.464, + "ttft_s_max": 4.504, + "decode_time_s": 40.794, + "total_time_s": 45.286, + "decode_tok_per_s": 0.76, + "decode_tok_per_s_min": 0.755, + "decode_tok_per_s_max": 0.76, + "total_tok_per_s": 0.707, + "total_tok_per_s_min": 0.703, + "total_tok_per_s_max": 0.707, + "sweeps": 15, + "drafted_tokens": 77, + "accepted_tokens": 18, + "resident_layers": 4, + "streamed_layers": 44, + "n_layers": 48, + "resident_mib": 1692.4, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 103107.13, + "pread_mib_s": 2796.4, + "effective_io_mib_s": 2276.8, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3709.2, + "current_rss_mib": 3709.2, + "mem_avail_before_mib": 2397.2, + "mem_avail_after_mib": 2403.7, + "swap_delta_mib": -0.3, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -0.4, + "page_cache_delta_since_start_mib": -1657.4, + "idle_watts": 7.77, + "avg_watts": 13.21, + "net_watts": 5.43, + "joules_per_tok": 18.69, + "net_joules_per_tok": 7.69, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": true, + "vram_budget_mb": 0, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 4.936, + "ttft_s": 3.258, + "ttft_s_min": 3.236, + "ttft_s_max": 3.281, + "decode_time_s": 14.884, + "total_time_s": 18.158, + "decode_tok_per_s": 3.158, + "decode_tok_per_s_min": 2.244, + "decode_tok_per_s_max": 3.159, + "total_tok_per_s": 2.643, + "total_tok_per_s_min": 1.985, + "total_tok_per_s_max": 2.646, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 43, + "resident_layers": 4, + "streamed_layers": 44, + "n_layers": 48, + "resident_mib": 1692.4, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 41242.85, + "pread_mib_s": 2799.5, + "effective_io_mib_s": 2273.3, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3713.9, + "current_rss_mib": 3713.9, + "mem_avail_before_mib": 2382.9, + "mem_avail_after_mib": 2385.3, + "swap_delta_mib": -0.3, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.2, + "page_cache_delta_since_start_mib": -1654.7, + "idle_watts": 7.77, + "avg_watts": 12.88, + "net_watts": 5.11, + "joules_per_tok": 4.87, + "net_joules_per_tok": 1.93, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": true, + "vram_budget_mb": 0, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 4.936, + "ttft_s": 2.952, + "ttft_s_min": 2.921, + "ttft_s_max": 2.995, + "decode_time_s": 29.778, + "total_time_s": 32.699, + "decode_tok_per_s": 1.578, + "decode_tok_per_s_min": 1.564, + "decode_tok_per_s_max": 1.582, + "total_tok_per_s": 1.468, + "total_tok_per_s_min": 1.452, + "total_tok_per_s_max": 1.469, + "sweeps": 11, + "drafted_tokens": 97, + "accepted_tokens": 38, + "resident_layers": 4, + "streamed_layers": 44, + "n_layers": 48, + "resident_mib": 1692.4, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 75611.89, + "pread_mib_s": 2794.3, + "effective_io_mib_s": 2312.4, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3715.9, + "current_rss_mib": 3715.9, + "mem_avail_before_mib": 2387.5, + "mem_avail_after_mib": 2388.8, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.8, + "page_cache_delta_since_start_mib": -1654.0, + "idle_watts": 7.77, + "avg_watts": 12.84, + "net_watts": 5.06, + "joules_per_tok": 8.75, + "net_joules_per_tok": 3.45, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b_multiturn", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "multi_turn_prefix_reuse", + "chat_template_applied": null, + "warmup": null, + "reps": null, + "temperature": null, + "seed": null, + "threads": null, + "kv_cache_type": null, + "vram_budget_auto": null, + "vram_budget_mb": null, + "prompt_tokens": null, + "completion_tokens": null, + "load_time_s": null, + "ttft_s": null, + "ttft_s_min": null, + "ttft_s_max": null, + "decode_time_s": null, + "total_time_s": null, + "decode_tok_per_s": null, + "decode_tok_per_s_min": null, + "decode_tok_per_s_max": null, + "total_tok_per_s": null, + "total_tok_per_s_min": null, + "total_tok_per_s_max": null, + "sweeps": null, + "drafted_tokens": null, + "accepted_tokens": null, + "resident_layers": null, + "streamed_layers": null, + "n_layers": null, + "resident_mib": null, + "slot_mib": null, + "expert_slot_mib": null, + "io_mib_read": null, + "pread_mib_s": null, + "effective_io_mib_s": null, + "expert_cache_hits": null, + "expert_cache_misses": null, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": null, + "moe_layers": null, + "moe_active_streaming": null, + "uma_zero_copy": null, + "is_unified_memory": null, + "direct_io_active": null, + "peak_rss_mib": null, + "current_rss_mib": null, + "mem_avail_before_mib": null, + "mem_avail_after_mib": null, + "swap_delta_mib": null, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": null, + "page_cache_delta_since_start_mib": null, + "idle_watts": null, + "avg_watts": null, + "net_watts": null, + "joules_per_tok": null, + "net_joules_per_tok": null, + "turn1_prompt_tokens": 1377, + "turn1_ttft_s": 14.877, + "turn2_prompt_tokens": 1414, + "turn2_ttft_s": 2.993, + "ttft_speedup_x": 4.97, + "notes": null, + "output_preview": " O_DIRECT prevents page-cache thrashing by bypassing the operating system's page cache" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": false, + "warmup": false, + "reps": 3, + "temperature": 0.7, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": true, + "vram_budget_mb": 0, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 3.799, + "ttft_s": 8.945, + "ttft_s_min": 8.935, + "ttft_s_max": 9.013, + "decode_time_s": 103.065, + "total_time_s": 112.011, + "decode_tok_per_s": 0.301, + "decode_tok_per_s_min": 0.263, + "decode_tok_per_s_max": 0.42, + "total_tok_per_s": 0.286, + "total_tok_per_s_min": 0.252, + "total_tok_per_s_max": 0.386, + "sweeps": 15, + "drafted_tokens": 72, + "accepted_tokens": 18, + "resident_layers": 3, + "streamed_layers": 45, + "n_layers": 48, + "resident_mib": 1526.0, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 105603.4, + "pread_mib_s": 1031.5, + "effective_io_mib_s": 942.8, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3542.2, + "current_rss_mib": 3542.2, + "mem_avail_before_mib": 2538.7, + "mem_avail_after_mib": 2522.2, + "swap_delta_mib": -0.5, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 1.1, + "page_cache_delta_since_start_mib": 11.2, + "idle_watts": 7.84, + "avg_watts": 9.95, + "net_watts": 2.11, + "joules_per_tok": 34.83, + "net_joules_per_tok": 7.37, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of compute buffers and asynchronously streaming transformer blocks from NVMe storage" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": false, + "warmup": false, + "reps": 3, + "temperature": 0.7, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": true, + "vram_budget_mb": 0, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 3.799, + "ttft_s": 7.714, + "ttft_s_min": 7.712, + "ttft_s_max": 7.719, + "decode_time_s": 37.27, + "total_time_s": 44.989, + "decode_tok_per_s": 1.261, + "decode_tok_per_s_min": 1.261, + "decode_tok_per_s_max": 1.262, + "total_tok_per_s": 1.067, + "total_tok_per_s_min": 1.067, + "total_tok_per_s_max": 1.068, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 43, + "resident_layers": 3, + "streamed_layers": 45, + "n_layers": 48, + "resident_mib": 1526.0, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 42241.36, + "pread_mib_s": 1031.2, + "effective_io_mib_s": 938.9, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3546.9, + "current_rss_mib": 3546.9, + "mem_avail_before_mib": 2540.0, + "mem_avail_after_mib": 2537.3, + "swap_delta_mib": -0.5, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.1, + "page_cache_delta_since_start_mib": 11.9, + "idle_watts": 7.84, + "avg_watts": 9.98, + "net_watts": 2.14, + "joules_per_tok": 9.35, + "net_joules_per_tok": 2.0, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_auto_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_auto_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": false, + "warmup": false, + "reps": 3, + "temperature": 0.7, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": true, + "vram_budget_mb": 0, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 3.799, + "ttft_s": 7.421, + "ttft_s_min": 7.406, + "ttft_s_max": 7.424, + "decode_time_s": 111.118, + "total_time_s": 118.524, + "decode_tok_per_s": 0.423, + "decode_tok_per_s_min": 0.373, + "decode_tok_per_s_max": 0.454, + "total_tok_per_s": 0.405, + "total_tok_per_s_min": 0.36, + "total_tok_per_s_max": 0.432, + "sweeps": 15, + "drafted_tokens": 112, + "accepted_tokens": 34, + "resident_layers": 3, + "streamed_layers": 45, + "n_layers": 48, + "resident_mib": 1526.0, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 105603.4, + "pread_mib_s": 1031.8, + "effective_io_mib_s": 950.4, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3548.9, + "current_rss_mib": 3548.9, + "mem_avail_before_mib": 2527.6, + "mem_avail_after_mib": 2521.9, + "swap_delta_mib": -0.5, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.5, + "page_cache_delta_since_start_mib": 34.2, + "idle_watts": 7.84, + "avg_watts": 9.97, + "net_watts": 2.12, + "joules_per_tok": 24.61, + "net_joules_per_tok": 5.24, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " In the context of running transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU bu" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_b5400_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_b5400_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 4.581, + "ttft_s": 4.238, + "ttft_s_min": 4.23, + "ttft_s_max": 7.883, + "decode_time_s": 37.618, + "total_time_s": 42.173, + "decode_tok_per_s": 0.824, + "decode_tok_per_s_min": 0.817, + "decode_tok_per_s_max": 0.827, + "total_tok_per_s": 0.759, + "total_tok_per_s_min": 0.706, + "total_tok_per_s_max": 0.765, + "sweeps": 15, + "drafted_tokens": 77, + "accepted_tokens": 18, + "resident_layers": 7, + "streamed_layers": 41, + "n_layers": 48, + "resident_mib": 2155.6, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 96159.67, + "pread_mib_s": 2923.7, + "effective_io_mib_s": 2280.1, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4171.4, + "current_rss_mib": 4171.1, + "mem_avail_before_mib": 1915.6, + "mem_avail_after_mib": 1914.0, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -0.8, + "page_cache_delta_since_start_mib": -59.6, + "idle_watts": 7.7, + "avg_watts": 13.29, + "net_watts": 5.59, + "joules_per_tok": 17.51, + "net_joules_per_tok": 7.37, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_b5400_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_b5400_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 4.581, + "ttft_s": 3.006, + "ttft_s_min": 3.006, + "ttft_s_max": 5.221, + "decode_time_s": 13.803, + "total_time_s": 16.809, + "decode_tok_per_s": 3.405, + "decode_tok_per_s_min": 3.284, + "decode_tok_per_s_max": 3.414, + "total_tok_per_s": 2.856, + "total_tok_per_s_min": 2.457, + "total_tok_per_s_max": 2.862, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 43, + "resident_layers": 7, + "streamed_layers": 41, + "n_layers": 48, + "resident_mib": 2155.6, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 38463.87, + "pread_mib_s": 2961.1, + "effective_io_mib_s": 2288.3, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4175.9, + "current_rss_mib": 4175.8, + "mem_avail_before_mib": 1909.0, + "mem_avail_after_mib": 1907.4, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.2, + "page_cache_delta_since_start_mib": -66.9, + "idle_watts": 7.7, + "avg_watts": 13.16, + "net_watts": 5.47, + "joules_per_tok": 4.61, + "net_joules_per_tok": 1.91, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_b5400_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_b5400_draft05b", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 4.581, + "ttft_s": 2.736, + "ttft_s_min": 2.709, + "ttft_s_max": 2.765, + "decode_time_s": 27.66, + "total_time_s": 30.396, + "decode_tok_per_s": 1.699, + "decode_tok_per_s_min": 1.685, + "decode_tok_per_s_max": 1.713, + "total_tok_per_s": 1.579, + "total_tok_per_s_min": 1.568, + "total_tok_per_s_max": 1.589, + "sweeps": 11, + "drafted_tokens": 97, + "accepted_tokens": 38, + "resident_layers": 7, + "streamed_layers": 41, + "n_layers": 48, + "resident_mib": 2155.6, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 70517.09, + "pread_mib_s": 2931.5, + "effective_io_mib_s": 2319.9, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4177.9, + "current_rss_mib": 4177.8, + "mem_avail_before_mib": 1906.0, + "mem_avail_after_mib": 1904.2, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -0.3, + "page_cache_delta_since_start_mib": -68.6, + "idle_watts": 7.7, + "avg_watts": 13.29, + "net_watts": 5.59, + "joules_per_tok": 8.42, + "net_joules_per_tok": 3.54, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_b5400_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_b5400_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": false, + "warmup": false, + "reps": 3, + "temperature": 0.7, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 4.333, + "ttft_s": 8.337, + "ttft_s_min": 8.329, + "ttft_s_max": 8.556, + "decode_time_s": 95.405, + "total_time_s": 103.742, + "decode_tok_per_s": 0.325, + "decode_tok_per_s_min": 0.284, + "decode_tok_per_s_max": 0.454, + "total_tok_per_s": 0.308, + "total_tok_per_s_min": 0.273, + "total_tok_per_s_max": 0.417, + "sweeps": 15, + "drafted_tokens": 72, + "accepted_tokens": 18, + "resident_layers": 6, + "streamed_layers": 42, + "n_layers": 48, + "resident_mib": 2006.6, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 98543.92, + "pread_mib_s": 1026.8, + "effective_io_mib_s": 948.4, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4023.3, + "current_rss_mib": 4023.3, + "mem_avail_before_mib": 2060.9, + "mem_avail_after_mib": 2060.2, + "swap_delta_mib": -0.3, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.3, + "page_cache_delta_since_start_mib": 22.2, + "idle_watts": 7.72, + "avg_watts": 10.03, + "net_watts": 2.31, + "joules_per_tok": 32.52, + "net_joules_per_tok": 7.47, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of compute buffers and asynchronously streaming transformer blocks from NVMe storage" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_b5400_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_b5400_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": false, + "warmup": false, + "reps": 3, + "temperature": 0.7, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 4.333, + "ttft_s": 7.122, + "ttft_s_min": 7.121, + "ttft_s_max": 7.142, + "decode_time_s": 34.313, + "total_time_s": 41.434, + "decode_tok_per_s": 1.37, + "decode_tok_per_s_min": 1.369, + "decode_tok_per_s_max": 1.372, + "total_tok_per_s": 1.158, + "total_tok_per_s_min": 1.158, + "total_tok_per_s_max": 1.16, + "sweeps": 6, + "drafted_tokens": 50, + "accepted_tokens": 43, + "resident_layers": 6, + "streamed_layers": 42, + "n_layers": 48, + "resident_mib": 2006.6, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 39208.95, + "pread_mib_s": 1025.5, + "effective_io_mib_s": 946.3, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4028.0, + "current_rss_mib": 4028.0, + "mem_avail_before_mib": 2052.5, + "mem_avail_after_mib": 2056.1, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.1, + "page_cache_delta_since_start_mib": 22.9, + "idle_watts": 7.72, + "avg_watts": 9.92, + "net_watts": 2.2, + "joules_per_tok": 8.56, + "net_joules_per_tok": 1.9, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option,\npub vram_budget_b" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_b5400_draft05b_temp07.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_b5400_draft05b_temp07", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": false, + "warmup": false, + "reps": 3, + "temperature": 0.7, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 4.333, + "ttft_s": 6.816, + "ttft_s_min": 6.803, + "ttft_s_max": 6.841, + "decode_time_s": 102.588, + "total_time_s": 109.39, + "decode_tok_per_s": 0.458, + "decode_tok_per_s_min": 0.405, + "decode_tok_per_s_max": 0.491, + "total_tok_per_s": 0.439, + "total_tok_per_s_min": 0.391, + "total_tok_per_s_max": 0.468, + "sweeps": 15, + "drafted_tokens": 112, + "accepted_tokens": 34, + "resident_layers": 6, + "streamed_layers": 42, + "n_layers": 48, + "resident_mib": 2006.6, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 98543.92, + "pread_mib_s": 1027.3, + "effective_io_mib_s": 961.5, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4030.0, + "current_rss_mib": 4030.0, + "mem_avail_before_mib": 2047.3, + "mem_avail_after_mib": 2046.9, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 1.2, + "page_cache_delta_since_start_mib": 25.8, + "idle_watts": 7.72, + "avg_watts": 9.77, + "net_watts": 2.05, + "joules_per_tok": 22.27, + "net_joules_per_tok": 4.67, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " In the context of running transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU bu" + }, + { + "round": "round8_v010", + "source_file": "r8_14b_r0.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_14b_r0", + "status": "ok", + "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": false, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 31, + "completion_tokens": 16, + "load_time_s": 2.993, + "ttft_s": 5.966, + "ttft_s_min": 5.783, + "ttft_s_max": 6.149, + "decode_time_s": 86.265, + "total_time_s": 92.231, + "decode_tok_per_s": 0.174, + "decode_tok_per_s_min": 0.174, + "decode_tok_per_s_max": 0.174, + "total_tok_per_s": 0.173, + "total_tok_per_s_min": 0.173, + "total_tok_per_s_max": 0.174, + "sweeps": 16, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 1026.8, + "slot_mib": 166.4, + "expert_slot_mib": 0.0, + "io_mib_read": 120631.69, + "pread_mib_s": 1426.9, + "effective_io_mib_s": 1308.8, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 2458.0, + "current_rss_mib": 2457.9, + "mem_avail_before_mib": 3656.6, + "mem_avail_after_mib": 3632.7, + "swap_delta_mib": -3.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 11.8, + "page_cache_delta_since_start_mib": -677.4, + "idle_watts": 7.71, + "avg_watts": 10.37, + "net_watts": 2.66, + "joules_per_tok": 59.75, + "net_joules_per_tok": 15.33, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page" + }, + { + "round": "round8_v010", + "source_file": "r8_32b_b5400_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_32b_b5400_draft05b", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "prompt_heavy", + "chat_template_applied": false, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 340, + "completion_tokens": 32, + "load_time_s": 3.764, + "ttft_s": 18.01, + "ttft_s_min": 17.468, + "ttft_s_max": 18.552, + "decode_time_s": 183.023, + "total_time_s": 201.033, + "decode_tok_per_s": 0.171, + "decode_tok_per_s_min": 0.156, + "decode_tok_per_s_max": 0.185, + "total_tok_per_s": 0.16, + "total_tok_per_s_min": 0.148, + "total_tok_per_s_max": 0.172, + "sweeps": 13, + "drafted_tokens": 67, + "accepted_tokens": 19, + "resident_layers": 2, + "streamed_layers": 62, + "n_layers": 64, + "resident_mib": 1622.2, + "slot_mib": 297.7, + "expert_slot_mib": 0.0, + "io_mib_read": 224952.76, + "pread_mib_s": 1229.4, + "effective_io_mib_s": 1125.0, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4108.7, + "current_rss_mib": 4108.6, + "mem_avail_before_mib": 1961.5, + "mem_avail_after_mib": 1964.1, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -10.4, + "page_cache_delta_since_start_mib": -21.0, + "idle_watts": 7.94, + "avg_watts": 10.21, + "net_watts": 2.26, + "joules_per_tok": 64.13, + "net_joules_per_tok": 14.21, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " The core memory-management mechanism described involves layer-streaming inference, which uses a small fixed pool of reusable compute buffers and asynchronously" + }, + { + "round": "round8_v010", + "source_file": "r8_32b_b5400_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_32b_b5400_draft05b", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "code_edit", + "chat_template_applied": false, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 106, + "completion_tokens": 48, + "load_time_s": 3.764, + "ttft_s": 14.704, + "ttft_s_min": 14.671, + "ttft_s_max": 14.738, + "decode_time_s": 55.967, + "total_time_s": 70.672, + "decode_tok_per_s": 0.84, + "decode_tok_per_s_min": 0.838, + "decode_tok_per_s_max": 0.841, + "total_tok_per_s": 0.679, + "total_tok_per_s_min": 0.678, + "total_tok_per_s_max": 0.681, + "sweeps": 5, + "drafted_tokens": 44, + "accepted_tokens": 44, + "resident_layers": 2, + "streamed_layers": 62, + "n_layers": 64, + "resident_mib": 1622.2, + "slot_mib": 297.7, + "expert_slot_mib": 0.0, + "io_mib_read": 86818.02, + "pread_mib_s": 1337.6, + "effective_io_mib_s": 1224.3, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4113.2, + "current_rss_mib": 4113.2, + "mem_avail_before_mib": 1958.5, + "mem_avail_after_mib": 1951.6, + "swap_delta_mib": -0.5, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -1.9, + "page_cache_delta_since_start_mib": -24.5, + "idle_watts": 7.94, + "avg_watts": 10.54, + "net_watts": 2.59, + "joules_per_tok": 15.51, + "net_joules_per_tok": 3.82, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\n pub model_path: PathBuf,\n pub n_ctx: u32,\n pub n_batch: u32,\n pub resident_layers: Option,\n " + }, + { + "round": "round8_v010", + "source_file": "r8_32b_b5400_draft05b.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_32b_b5400_draft05b", + "status": "ok", + "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", + "workload": "open_ended", + "chat_template_applied": false, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 5400, + "prompt_tokens": 31, + "completion_tokens": 48, + "load_time_s": 3.764, + "ttft_s": 16.558, + "ttft_s_min": 14.005, + "ttft_s_max": 19.111, + "decode_time_s": 154.474, + "total_time_s": 171.031, + "decode_tok_per_s": 0.304, + "decode_tok_per_s_min": 0.304, + "decode_tok_per_s_max": 0.304, + "total_tok_per_s": 0.281, + "total_tok_per_s_min": 0.277, + "total_tok_per_s_max": 0.285, + "sweeps": 12, + "drafted_tokens": 104, + "accepted_tokens": 37, + "resident_layers": 2, + "streamed_layers": 62, + "n_layers": 64, + "resident_mib": 1622.2, + "slot_mib": 297.7, + "expert_slot_mib": 0.0, + "io_mib_read": 207648.7, + "pread_mib_s": 1311.4, + "effective_io_mib_s": 1213.5, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 4115.2, + "current_rss_mib": 4115.2, + "mem_avail_before_mib": 1930.7, + "mem_avail_after_mib": 1955.5, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -12.6, + "page_cache_delta_since_start_mib": -37.9, + "idle_watts": 7.94, + "avg_watts": 10.27, + "net_watts": 2.33, + "joules_per_tok": 36.61, + "net_joules_per_tok": 8.3, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers can b" + }, + { + "round": "round8_v010", + "source_file": "r8_olmoe_r0_cached.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_olmoe_r0_sparse_cached", + "status": "ok", + "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", + "draft_model": null, + "workload": "prompt_heavy", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 367, + "completion_tokens": 32, + "load_time_s": 0.681, + "ttft_s": 2.078, + "ttft_s_min": 2.078, + "ttft_s_max": 2.08, + "decode_time_s": 2.268, + "total_time_s": 4.346, + "decode_tok_per_s": 13.667, + "decode_tok_per_s_min": 13.652, + "decode_tok_per_s_max": 13.922, + "total_tok_per_s": 7.362, + "total_tok_per_s_min": 7.356, + "total_tok_per_s_max": 7.433, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 16, + "n_layers": 16, + "resident_mib": 218.3, + "slot_mib": 0.0, + "expert_slot_mib": 139.0, + "io_mib_read": 2085.12, + "pread_mib_s": 1404.5, + "effective_io_mib_s": 479.7, + "expert_cache_hits": 11928, + "expert_cache_misses": 0, + "expert_cache_hit_rate": 1.0, + "expert_cache_mib_saved": 8635.38, + "moe_layers": 16, + "moe_active_streaming": true, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 2921.4, + "current_rss_mib": 2921.4, + "mem_avail_before_mib": 2467.0, + "mem_avail_after_mib": 2465.2, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.4, + "page_cache_delta_since_start_mib": -90.3, + "idle_watts": 8.13, + "avg_watts": 12.32, + "net_watts": 4.2, + "joules_per_tok": 1.67, + "net_joules_per_tok": 0.57, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "\nThe core memory-management mechanism involves a combination of zero-copy staging from NVMe with page-aligned DMA, layer-streaming inference with reusable" + }, + { + "round": "round8_v010", + "source_file": "r8_olmoe_r0_cached.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_olmoe_r0_sparse_cached", + "status": "ok", + "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", + "draft_model": null, + "workload": "code_edit", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 133, + "completion_tokens": 48, + "load_time_s": 0.681, + "ttft_s": 1.865, + "ttft_s_min": 1.863, + "ttft_s_max": 1.868, + "decode_time_s": 3.601, + "total_time_s": 5.465, + "decode_tok_per_s": 13.054, + "decode_tok_per_s_min": 12.451, + "decode_tok_per_s_max": 13.24, + "total_tok_per_s": 8.783, + "total_tok_per_s_min": 8.513, + "total_tok_per_s_max": 8.86, + "sweeps": 48, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 16, + "n_layers": 16, + "resident_mib": 218.3, + "slot_mib": 0.0, + "expert_slot_mib": 139.0, + "io_mib_read": 2180.74, + "pread_mib_s": 1269.1, + "effective_io_mib_s": 409.4, + "expert_cache_hits": 17941, + "expert_cache_misses": 131, + "expert_cache_hit_rate": 0.993, + "expert_cache_mib_saved": 12988.27, + "moe_layers": 16, + "moe_active_streaming": true, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 2953.3, + "current_rss_mib": 2953.3, + "mem_avail_before_mib": 2438.6, + "mem_avail_after_mib": 2437.3, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.1, + "page_cache_delta_since_start_mib": -103.2, + "idle_watts": 8.13, + "avg_watts": 12.61, + "net_watts": 4.49, + "joules_per_tok": 1.44, + "net_joules_per_tok": 0.51, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers" + }, + { + "round": "round8_v010", + "source_file": "r8_olmoe_r0_cached.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_olmoe_r0_sparse_cached", + "status": "ok", + "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": false, + "warmup": true, + "reps": 3, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 33, + "completion_tokens": 48, + "load_time_s": 0.681, + "ttft_s": 1.72, + "ttft_s_min": 1.718, + "ttft_s_max": 1.724, + "decode_time_s": 3.901, + "total_time_s": 5.622, + "decode_tok_per_s": 12.047, + "decode_tok_per_s_min": 11.45, + "decode_tok_per_s_max": 12.505, + "total_tok_per_s": 8.538, + "total_tok_per_s_min": 8.235, + "total_tok_per_s_max": 8.764, + "sweeps": 48, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 16, + "n_layers": 16, + "resident_mib": 218.3, + "slot_mib": 0.0, + "expert_slot_mib": 139.0, + "io_mib_read": 2353.59, + "pread_mib_s": 1167.9, + "effective_io_mib_s": 436.5, + "expert_cache_hits": 17705, + "expert_cache_misses": 367, + "expert_cache_hit_rate": 0.98, + "expert_cache_mib_saved": 12816.33, + "moe_layers": 16, + "moe_active_streaming": true, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 2970.7, + "current_rss_mib": 2970.7, + "mem_avail_before_mib": 2415.1, + "mem_avail_after_mib": 2413.6, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 0.0, + "page_cache_delta_since_start_mib": -114.5, + "idle_watts": 8.13, + "avg_watts": 11.98, + "net_watts": 3.86, + "joules_per_tok": 1.4, + "net_joules_per_tok": 0.45, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "\n\n1. **Page-Aligned O_DIRECT NVMe Reads:**\n - **Explanation:** When performing an O_DIRECT read on an NVMe storage device, the read operation is aligned with " + }, + { + "round": "round8_v010", + "source_file": "r8_olmoe_r0_full.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_olmoe_r0_full", + "status": "ok", + "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", + "draft_model": null, + "workload": "open_ended", + "chat_template_applied": false, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 33, + "completion_tokens": 32, + "load_time_s": 0.507, + "ttft_s": 1.448, + "ttft_s_min": 1.355, + "ttft_s_max": 1.54, + "decode_time_s": 40.615, + "total_time_s": 42.062, + "decode_tok_per_s": 0.763, + "decode_tok_per_s_min": 0.763, + "decode_tok_per_s_max": 0.763, + "total_tok_per_s": 0.761, + "total_tok_per_s_min": 0.759, + "total_tok_per_s_max": 0.762, + "sweeps": 32, + "drafted_tokens": 0, + "accepted_tokens": 0, + "resident_layers": 0, + "streamed_layers": 16, + "n_layers": 16, + "resident_mib": 112.8, + "slot_mib": 145.6, + "expert_slot_mib": 0.0, + "io_mib_read": 74545.94, + "pread_mib_s": 1942.9, + "effective_io_mib_s": 1774.0, + "expert_cache_hits": 0, + "expert_cache_misses": 0, + "expert_cache_hit_rate": null, + "expert_cache_mib_saved": 0.0, + "moe_layers": 0, + "moe_active_streaming": false, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 1312.2, + "current_rss_mib": 1312.2, + "mem_avail_before_mib": 4053.9, + "mem_avail_after_mib": 4054.1, + "swap_delta_mib": -2.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": 1.0, + "page_cache_delta_since_start_mib": 1.0, + "idle_watts": 7.86, + "avg_watts": 10.91, + "net_watts": 3.05, + "joules_per_tok": 14.34, + "net_joules_per_tok": 4.01, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "\n\n1. **Page-Aligned O_DIRECT NVMe Reads:**\n - **Explanation:** When performing an O_DIRECT read on" + }, + { + "round": "round8_v010", + "source_file": "r8_qwen3_30b_a3b_draft06b_code.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_qwen3_30b_a3b_r0_draft06b", + "status": "ok", + "model": "Qwen3-30B-A3B-Q4_K_M.gguf", + "draft_model": "Qwen3-0.6B-Q4_K_M.gguf", + "workload": "code_edit", + "chat_template_applied": true, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 114, + "completion_tokens": 32, + "load_time_s": 33.063, + "ttft_s": 13.596, + "ttft_s_min": 13.512, + "ttft_s_max": 13.681, + "decode_time_s": 331.875, + "total_time_s": 345.471, + "decode_tok_per_s": 0.113, + "decode_tok_per_s_min": 0.066, + "decode_tok_per_s_max": 0.159, + "total_tok_per_s": 0.11, + "total_tok_per_s_min": 0.066, + "total_tok_per_s_max": 0.154, + "sweeps": 11, + "drafted_tokens": 62, + "accepted_tokens": 22, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 951.3, + "slot_mib": 0.0, + "expert_slot_mib": 373.5, + "io_mib_read": 74126.45, + "pread_mib_s": 267.3, + "effective_io_mib_s": 257.1, + "expert_cache_hits": 4218, + "expert_cache_misses": 22119, + "expert_cache_hit_rate": 0.16, + "expert_cache_mib_saved": 3767.38, + "moe_layers": 48, + "moe_active_streaming": true, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3974.3, + "current_rss_mib": 3973.1, + "mem_avail_before_mib": 1549.3, + "mem_avail_after_mib": 1539.5, + "swap_delta_mib": -2.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -667.3, + "page_cache_delta_since_start_mib": -1066.2, + "idle_watts": 7.85, + "avg_watts": 8.33, + "net_watts": 0.48, + "joules_per_tok": 89.98, + "net_joules_per_tok": 5.21, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "\nOkay, let me try to figure out how to do this. The user wants me to take the given Rust struct and add a new field called `" + }, + { + "round": "round8_v010", + "source_file": "r8_qwen3_30b_a3b_draft06b_open.json", + "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", + "engine": "layerstream-plugin", + "engine_version": "0.1.0", + "git_rev": null, + "mode": "r8_qwen3_30b_a3b_r0_draft06b", + "status": "ok", + "model": "Qwen3-30B-A3B-Q4_K_M.gguf", + "draft_model": "Qwen3-0.6B-Q4_K_M.gguf", + "workload": "open_ended", + "chat_template_applied": true, + "warmup": false, + "reps": 2, + "temperature": 0.0, + "seed": 42, + "threads": 4, + "kv_cache_type": "q8_0", + "vram_budget_auto": false, + "vram_budget_mb": 0, + "prompt_tokens": 39, + "completion_tokens": 32, + "load_time_s": 3.263, + "ttft_s": 11.238, + "ttft_s_min": 11.08, + "ttft_s_max": 11.395, + "decode_time_s": 105.204, + "total_time_s": 116.442, + "decode_tok_per_s": 0.317, + "decode_tok_per_s_min": 0.233, + "decode_tok_per_s_max": 0.4, + "total_tok_per_s": 0.291, + "total_tok_per_s_min": 0.222, + "total_tok_per_s_max": 0.361, + "sweeps": 6, + "drafted_tokens": 48, + "accepted_tokens": 27, + "resident_layers": 0, + "streamed_layers": 48, + "n_layers": 48, + "resident_mib": 951.3, + "slot_mib": 0.0, + "expert_slot_mib": 373.5, + "io_mib_read": 73026.22, + "pread_mib_s": 708.0, + "effective_io_mib_s": 667.7, + "expert_cache_hits": 1826, + "expert_cache_misses": 6019, + "expert_cache_hit_rate": 0.233, + "expert_cache_mib_saved": 1555.77, + "moe_layers": 48, + "moe_active_streaming": true, + "uma_zero_copy": false, + "is_unified_memory": true, + "direct_io_active": true, + "peak_rss_mib": 3969.2, + "current_rss_mib": 3969.2, + "mem_avail_before_mib": 1661.1, + "mem_avail_after_mib": 1629.2, + "swap_delta_mib": 0.0, + "cuda_max_allocated_mib": null, + "page_cache_delta_mib": -24.4, + "page_cache_delta_since_start_mib": -236.8, + "idle_watts": 8.18, + "avg_watts": 9.06, + "net_watts": 0.88, + "joules_per_tok": 32.98, + "net_joules_per_tok": 3.21, + "turn1_prompt_tokens": null, + "turn1_ttft_s": null, + "turn2_prompt_tokens": null, + "turn2_ttft_s": null, + "ttft_speedup_x": null, + "notes": null, + "output_preview": "\nOkay, so I need to explain how page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers interact when streaming transformer layers on an " + } +]