layerstream-benchmarks / benchmarks.json
kleinbem's picture
Add hardware_id multi-hardware schema and Space Hardware filter
e53c046 verified
Raw History Blame Contribute Delete
317 kB
[ { "round": "airllm", "source_file": "airllm_14b_4bit_code_edit_warm.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "airllm", "engine_version": "4.0.0", "git_rev": null, "mode": "airllm_4bit_prefetch", "status": "ok", "model": "Qwen/Qwen2.5-14B-Instruct", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 113, "completion_tokens": 8, "load_time_s": 2.55, "ttft_s": 33.371, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 214.529, "total_time_s": 247.9, "decode_tok_per_s": 0.033, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.032, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 3536.4, "current_rss_mib": 626.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": 1936.2, "page_cache_delta_mib": -503.0, "page_cache_delta_since_start_mib": -458.6, "idle_watts": null, "avg_watts": 9.3, "net_watts": null, "joules_per_tok": 288.21, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct" }, { "round": "airllm", "source_file": "airllm_14b_4bit_open_ended_warm.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "airllm", "engine_version": "4.0.0", "git_rev": null, "mode": "airllm_4bit_prefetch", "status": "ok", "model": "Qwen/Qwen2.5-14B-Instruct", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 8, "load_time_s": 2.573, "ttft_s": 32.863, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 213.357, "total_time_s": 246.219, "decode_tok_per_s": 0.033, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.032, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 3531.2, "current_rss_mib": 617.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": 1920.8, "page_cache_delta_mib": -640.9, "page_cache_delta_since_start_mib": -591.8, "idle_watts": null, "avg_watts": 9.33, "net_watts": null, "joules_per_tok": 287.1, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When discussing the interaction between page-aligned O" }, { "round": "airllm", "source_file": "airllm_14b_4bit_prompt_heavy.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "airllm", "engine_version": "4.0.0", "git_rev": null, "mode": "airllm_4bit_prefetch", "status": "ok", "model": "Qwen/Qwen2.5-14B-Instruct", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 365.529, "ttft_s": 32.093, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 954.918, "total_time_s": 987.011, "decode_tok_per_s": 0.032, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.032, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5390.5, "current_rss_mib": 689.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": 1983.3, "page_cache_delta_mib": 248.0, "page_cache_delta_since_start_mib": -751.8, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described is layer-streaming inference, which minimizes memory usage by allocating a small fixed pool of reusable buffers " }, { "round": "airllm", "source_file": "airllm_14b_bf16_open_ended.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "airllm", "engine_version": "4.0.0", "git_rev": null, "mode": "airllm_none_prefetch", "status": "ok", "model": "Qwen/Qwen2.5-14B-Instruct", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 4, "load_time_s": 81.81, "ttft_s": 45.413, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 131.774, "total_time_s": 177.187, "decode_tok_per_s": 0.023, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.023, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5217.6, "current_rss_mib": 3489.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": 1502.0, "page_cache_delta_mib": -4310.7, "page_cache_delta_since_start_mib": -3140.9, "idle_watts": null, "avg_watts": 9.31, "net_watts": null, "joules_per_tok": 412.35, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " In the context of" }, { "round": "competitors", "source_file": "hf_accelerate_14b_4bit.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "hf-accelerate", "engine_version": "transformers-5.23.0+bnb-0.50.2", "git_rev": null, "mode": "hf_accelerate_14b_4bit_disk", "status": "timeout_300s", "model": "Qwen2.5-14B-Instruct", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": 144.0, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": null, "total_time_s": 300.0, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": null, "current_rss_mib": null, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": null, "page_cache_delta_since_start_mib": null, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": "Loaded and disk-offloaded 579 tensors (4-bit NF4, gpu_max_mem=3GiB, cpu_max_mem=1GiB) in 144.0s (2m24s), then hit 300s wall-clock timeout during generation of 16 tokens. PyTorch 2.14.1+cu126 wheel warns Orin CC 8.7 unsupported (same venv as AirLLM 4.0.0).", "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_cpu_mmap.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_cpu_mmap", "status": "timeout", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 300.458, "total_time_s": 300.458, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 6109.6, "current_rss_mib": null, "mem_avail_before_mib": 5040.2, "mem_avail_after_mib": null, "swap_delta_mib": 362.6, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 5648.3, "page_cache_delta_since_start_mib": null, "idle_watts": 7.87, "avg_watts": 10.61, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cru" }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl0", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 22.832, "total_time_s": 22.832, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5991.0, "current_rss_mib": null, "mem_avail_before_mib": 4984.7, "mem_avail_after_mib": null, "swap_delta_mib": 3683.5, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 135.7, "page_cache_delta_since_start_mib": null, "idle_watts": 7.79, "avg_watts": 9.9, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl12_b512.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl12_b512", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 23.562, "total_time_s": 23.562, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5977.9, "current_rss_mib": null, "mem_avail_before_mib": 5021.0, "mem_avail_after_mib": null, "swap_delta_mib": 3676.6, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 76.5, "page_cache_delta_since_start_mib": null, "idle_watts": 7.97, "avg_watts": 9.84, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl12_b512.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl12_b512", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 23.752, "total_time_s": 23.752, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5742.9, "current_rss_mib": null, "mem_avail_before_mib": 5163.8, "mem_avail_after_mib": null, "swap_delta_mib": 3684.2, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -5.9, "page_cache_delta_since_start_mib": null, "idle_watts": 7.97, "avg_watts": 9.89, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl12_b512.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl12_b512", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 23.445, "total_time_s": 23.445, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5723.5, "current_rss_mib": null, "mem_avail_before_mib": 5125.5, "mem_avail_after_mib": null, "swap_delta_mib": 3710.8, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 53.4, "page_cache_delta_since_start_mib": null, "idle_watts": 7.97, "avg_watts": 9.95, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl18.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl18", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 56.129, "total_time_s": 56.129, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5776.3, "current_rss_mib": null, "mem_avail_before_mib": 4972.8, "mem_avail_after_mib": null, "swap_delta_mib": 3634.6, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -2317.2, "page_cache_delta_since_start_mib": null, "idle_watts": 7.92, "avg_watts": 9.23, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl18.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl18", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 22.412, "total_time_s": 22.412, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5736.2, "current_rss_mib": null, "mem_avail_before_mib": 5027.8, "mem_avail_after_mib": null, "swap_delta_mib": 3626.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 114.3, "page_cache_delta_since_start_mib": null, "idle_watts": 7.92, "avg_watts": 9.97, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_14b_ngl18.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_14b_ngl18", "status": "error_rc_1", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 54.08, "total_time_s": 54.08, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5715.3, "current_rss_mib": null, "mem_avail_before_mib": 5078.5, "mem_avail_after_mib": null, "swap_delta_mib": 3556.4, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -86.6, "page_cache_delta_since_start_mib": null, "idle_watts": 7.92, "avg_watts": 8.73, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_32b_ngl10.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_32b_ngl10", "status": "timeout", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 302.828, "total_time_s": 302.828, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 5714.7, "current_rss_mib": null, "mem_avail_before_mib": 5176.0, "mem_avail_after_mib": null, "swap_delta_mib": 17242.8, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -305.4, "page_cache_delta_since_start_mib": null, "idle_watts": 7.89, "avg_watts": 10.29, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "When streaming transformer layers" }, { "round": "competitors", "source_file": "llama_cpp_qwen3_30b_cpu_mmap.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_qwen3_30b_cpu_mmap", "status": "error_rc_1", "model": "Qwen3-30B-A3B-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 66.223, "total_time_s": 66.223, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 6044.1, "current_rss_mib": null, "mem_avail_before_mib": 5115.9, "mem_avail_after_mib": null, "swap_delta_mib": 12324.5, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -62.8, "page_cache_delta_since_start_mib": null, "idle_watts": 7.92, "avg_watts": 10.16, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "competitors", "source_file": "llama_cpp_qwen3_30b_cpu_mmap_b512.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_qwen3_30b_cpu_mmap_b512", "status": "ok", "model": "Qwen3-30B-A3B-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 23, "load_time_s": 25.34, "ttft_s": 17.367, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 18.113, "total_time_s": 62.445, "decode_tok_per_s": 1.27, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.648, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 6070.9, "current_rss_mib": null, "mem_avail_before_mib": 6137.6, "mem_avail_after_mib": null, "swap_delta_mib": 247.9, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -100.1, "page_cache_delta_since_start_mib": null, "idle_watts": 8.07, "avg_watts": 9.89, "net_watts": null, "joules_per_tok": 15.26, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "the problem is that the GPU is not able to read the data from the host memory as it is being written by the" }, { "round": "competitors", "source_file": "llama_cpp_qwen3_30b_gpu_offload.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "llama.cpp", "engine_version": "0.1.0", "git_rev": null, "mode": "llama_cpp_qwen3_30b_gpu_moe_offload", "status": "error_rc_1", "model": "Qwen3-30B-A3B-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": "q8_0", "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": 0, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 54.427, "total_time_s": 54.427, "decode_tok_per_s": 0.0, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.0, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": 6014.0, "current_rss_mib": null, "mem_avail_before_mib": 6081.8, "mem_avail_after_mib": null, "swap_delta_mib": 12342.2, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 81.1, "page_cache_delta_since_start_mib": null, "idle_watts": 7.95, "avg_watts": 10.12, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": null }, { "round": "results", "source_file": "r0_prompt_heavy.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 2.968, "ttft_s": 9.257, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 228.946, "total_time_s": 238.203, "decode_tok_per_s": 0.135, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.134, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 241263.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3225.7, "current_rss_mib": 3225.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 37.9, "page_cache_delta_since_start_mib": -764.6, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" }, { "round": "results", "source_file": "r14_draft_prompt_heavy.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 52.833, "ttft_s": 5.752, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 39.649, "total_time_s": 45.401, "decode_tok_per_s": 0.782, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.705, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 63, "accepted_tokens": 17, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 79826.54, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5877.1, "current_rss_mib": 5666.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -251.4, "page_cache_delta_since_start_mib": -2667.7, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round2", "source_file": "hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 53.386, "ttft_s": 5.665, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 41.96, "total_time_s": 47.624, "decode_tok_per_s": 0.739, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.672, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 63, "accepted_tokens": 17, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 79826.54, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6257.4, "current_rss_mib": 5668.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -198.3, "page_cache_delta_since_start_mib": -2235.9, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round2", "source_file": "hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 53.386, "ttft_s": 3.438, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 24.011, "total_time_s": 27.449, "decode_tok_per_s": 1.957, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.749, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 9, "drafted_tokens": 40, "accepted_tokens": 39, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 47895.93, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6257.4, "current_rss_mib": 5669.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -4.3, "page_cache_delta_since_start_mib": -2240.2, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round2", "source_file": "hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 53.386, "ttft_s": 3.155, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 36.017, "total_time_s": 39.172, "decode_tok_per_s": 1.305, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.225, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 13, "drafted_tokens": 59, "accepted_tokens": 32, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 69183.0, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6257.4, "current_rss_mib": 5669.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -14.6, "page_cache_delta_since_start_mib": -2254.7, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round2", "source_file": "hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 75.668, "ttft_s": 5.074, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 63.663, "total_time_s": 68.736, "decode_tok_per_s": 0.487, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.466, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 25, "drafted_tokens": 26, "accepted_tokens": 7, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 125190.63, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5982.3, "current_rss_mib": 5152.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 57.9, "page_cache_delta_since_start_mib": -181.0, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" }, { "round": "round2", "source_file": "hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 75.668, "ttft_s": 3.087, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 23.822, "total_time_s": 26.909, "decode_tok_per_s": 1.973, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.784, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 10, "drafted_tokens": 45, "accepted_tokens": 38, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 50076.25, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5982.3, "current_rss_mib": 5153.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 3.7, "page_cache_delta_since_start_mib": -177.3, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round2", "source_file": "hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 75.668, "ttft_s": 2.75, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 79.476, "total_time_s": 82.225, "decode_tok_per_s": 0.591, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.584, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 31, "drafted_tokens": 25, "accepted_tokens": 17, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 155236.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5982.3, "current_rss_mib": 5155.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 2.4, "page_cache_delta_since_start_mib": -174.9, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round2", "source_file": "stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 2.4, "ttft_s": 9.753, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 247.344, "total_time_s": 257.097, "decode_tok_per_s": 0.125, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.124, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 241263.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3249.2, "current_rss_mib": 3249.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 2.0, "page_cache_delta_since_start_mib": 2.3, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" }, { "round": "round2", "source_file": "stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 2.4, "ttft_s": 8.395, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 380.266, "total_time_s": 388.66, "decode_tok_per_s": 0.124, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.124, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 48, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 361895.06, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3249.2, "current_rss_mib": 3249.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 2.1, "page_cache_delta_since_start_mib": 4.4, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round2", "source_file": "stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 2.4, "ttft_s": 8.088, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 385.094, "total_time_s": 393.183, "decode_tok_per_s": 0.122, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.122, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 48, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 361895.06, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3249.2, "current_rss_mib": 3249.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 9.7, "page_cache_delta_since_start_mib": 14.1, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round2", "source_file": "stream_r0_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 2.102, "ttft_s": 10.312, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 204.874, "total_time_s": 215.185, "decode_tok_per_s": 0.151, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.149, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 25, "drafted_tokens": 26, "accepted_tokens": 7, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 188487.01, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3253.6, "current_rss_mib": 3253.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 34.6, "page_cache_delta_since_start_mib": 46.1, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for compute, keeping essential layers in memory, and asynchronously strea" }, { "round": "round2", "source_file": "stream_r0_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 2.102, "ttft_s": 8.983, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 76.835, "total_time_s": 85.818, "decode_tok_per_s": 0.612, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.559, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 10, "drafted_tokens": 45, "accepted_tokens": 38, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 75394.8, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3253.6, "current_rss_mib": 3253.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 3.6, "page_cache_delta_since_start_mib": 49.7, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round2", "source_file": "stream_r0_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "stream_r0_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 2.102, "ttft_s": 8.666, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 256.328, "total_time_s": 264.994, "decode_tok_per_s": 0.183, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.181, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 31, "drafted_tokens": 25, "accepted_tokens": 17, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 233723.89, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3255.7, "current_rss_mib": 3255.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 39.6, "page_cache_delta_since_start_mib": 89.3, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round3", "source_file": "r3_hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r3_hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.886, "ttft_s": 5.916, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 69.787, "total_time_s": 75.704, "decode_tok_per_s": 0.444, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.423, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 62, "accepted_tokens": 16, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 79826.54, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6079.0, "current_rss_mib": 6079.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -57.5, "page_cache_delta_since_start_mib": -1016.1, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round3", "source_file": "r3_hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r3_hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.886, "ttft_s": 3.49, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 18.261, "total_time_s": 21.75, "decode_tok_per_s": 2.574, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.207, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 31930.62, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6083.3, "current_rss_mib": 6083.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.3, "page_cache_delta_since_start_mib": -1015.8, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round3", "source_file": "r3_hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r3_hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.886, "ttft_s": 3.289, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 31.885, "total_time_s": 35.174, "decode_tok_per_s": 1.474, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.365, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 64, "accepted_tokens": 34, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 58539.46, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6083.6, "current_rss_mib": 6083.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 1.2, "page_cache_delta_since_start_mib": -1014.6, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round3", "source_file": "r3_stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r3_stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 1.859, "ttft_s": 7.419, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 260.939, "total_time_s": 268.358, "decode_tok_per_s": 0.119, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.119, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 241263.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3168.0, "current_rss_mib": 3168.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.6, "page_cache_delta_since_start_mib": 1.2, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round4", "source_file": "r4_auto_budget_5800_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_auto_budget_5800_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 8.611, "ttft_s": 4.823, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 38.613, "total_time_s": 43.437, "decode_tok_per_s": 0.803, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.737, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 62, "accepted_tokens": 16, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 68186.31, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6482.0, "current_rss_mib": 5978.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 7.5, "page_cache_delta_since_start_mib": -178.0, "idle_watts": null, "avg_watts": 12.65, "net_watts": null, "joules_per_tok": 17.17, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round4", "source_file": "r4_auto_budget_5800_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_auto_budget_5800_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 8.611, "ttft_s": 3.137, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 13.633, "total_time_s": 16.77, "decode_tok_per_s": 3.447, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.862, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 27274.52, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6482.0, "current_rss_mib": 5971.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -1.5, "page_cache_delta_since_start_mib": -179.5, "idle_watts": null, "avg_watts": 12.59, "net_watts": null, "joules_per_tok": 4.4, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round4", "source_file": "r4_auto_budget_5800_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_auto_budget_5800_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 8.611, "ttft_s": 2.907, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 27.967, "total_time_s": 30.874, "decode_tok_per_s": 1.681, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.555, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 81, "accepted_tokens": 34, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 50003.29, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6482.0, "current_rss_mib": 5939.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -23.4, "page_cache_delta_since_start_mib": -202.8, "idle_watts": null, "avg_watts": 12.2, "net_watts": null, "joules_per_tok": 7.85, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round4", "source_file": "r4_hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 7.8, "ttft_s": 5.093, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 39.028, "total_time_s": 44.12, "decode_tok_per_s": 0.794, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.725, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 62, "accepted_tokens": 16, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 79992.96, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5708.1, "current_rss_mib": 5348.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -87.8, "page_cache_delta_since_start_mib": -690.5, "idle_watts": null, "avg_watts": 12.85, "net_watts": null, "joules_per_tok": 17.72, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round4", "source_file": "r4_hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 7.8, "ttft_s": 3.155, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 13.958, "total_time_s": 17.113, "decode_tok_per_s": 3.367, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.805, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 31930.62, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5708.1, "current_rss_mib": 5352.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -14.7, "page_cache_delta_since_start_mib": -705.2, "idle_watts": null, "avg_watts": 12.36, "net_watts": null, "joules_per_tok": 4.41, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round4", "source_file": "r4_hybrid_r14_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r14_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 7.8, "ttft_s": 2.936, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 28.241, "total_time_s": 31.177, "decode_tok_per_s": 1.664, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.54, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 81, "accepted_tokens": 34, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 58539.46, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5708.1, "current_rss_mib": 5353.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 5.8, "page_cache_delta_since_start_mib": -699.4, "idle_watts": null, "avg_watts": 12.27, "net_watts": null, "joules_per_tok": 7.97, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 5.469, "ttft_s": 4.708, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 38.462, "total_time_s": 43.17, "decode_tok_per_s": 0.806, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.741, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 62, "accepted_tokens": 16, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 75114.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6201.6, "current_rss_mib": 6201.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -54.8, "page_cache_delta_since_start_mib": -451.7, "idle_watts": null, "avg_watts": 12.67, "net_watts": null, "joules_per_tok": 17.09, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 5.469, "ttft_s": 3.142, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 13.715, "total_time_s": 16.857, "decode_tok_per_s": 3.427, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.848, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 30193.48, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.8, "current_rss_mib": 6198.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -11.1, "page_cache_delta_since_start_mib": -462.8, "idle_watts": null, "avg_watts": 12.09, "net_watts": null, "joules_per_tok": 4.25, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 5.469, "ttft_s": 2.91, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 27.794, "total_time_s": 30.704, "decode_tok_per_s": 1.691, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.563, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 81, "accepted_tokens": 34, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55083.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.8, "current_rss_mib": 6179.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 7.0, "page_cache_delta_since_start_mib": -455.8, "idle_watts": null, "avg_watts": 12.86, "net_watts": null, "joules_per_tok": 8.23, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b_t2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b_t2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 5.178, "ttft_s": 5.791, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 53.552, "total_time_s": 59.343, "decode_tok_per_s": 0.579, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.539, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 62, "accepted_tokens": 16, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 75262.1, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6199.6, "current_rss_mib": 6199.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -53.2, "page_cache_delta_since_start_mib": 4.1, "idle_watts": null, "avg_watts": 11.14, "net_watts": null, "joules_per_tok": 20.65, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b_t2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b_t2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 5.178, "ttft_s": 3.989, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 18.402, "total_time_s": 22.391, "decode_tok_per_s": 2.554, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.144, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 29898.02, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.1, "current_rss_mib": 6203.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -18.3, "page_cache_delta_since_start_mib": -14.1, "idle_watts": null, "avg_watts": 11.43, "net_watts": null, "joules_per_tok": 5.33, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b_t2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b_t2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 5.178, "ttft_s": 3.805, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 38.511, "total_time_s": 42.316, "decode_tok_per_s": 1.22, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.134, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 81, "accepted_tokens": 34, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55083.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.1, "current_rss_mib": 6203.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -9.2, "page_cache_delta_since_start_mib": -23.4, "idle_watts": null, "avg_watts": 11.34, "net_watts": null, "joules_per_tok": 10.0, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b_t4.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b_t4", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 5.561, "ttft_s": 5.592, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 51.504, "total_time_s": 57.095, "decode_tok_per_s": 0.602, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.56, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 15, "drafted_tokens": 62, "accepted_tokens": 16, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 75262.1, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.0, "current_rss_mib": 6186.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -245.6, "page_cache_delta_since_start_mib": -166.0, "idle_watts": null, "avg_watts": 11.47, "net_watts": null, "joules_per_tok": 20.47, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b_t4.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b_t4", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 5.561, "ttft_s": 6.269, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 22.093, "total_time_s": 28.362, "decode_tok_per_s": 2.127, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.692, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 30045.75, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.0, "current_rss_mib": 6174.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 3.3, "page_cache_delta_since_start_mib": -162.6, "idle_watts": null, "avg_watts": 10.5, "net_watts": null, "joules_per_tok": 6.21, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round4", "source_file": "r4_hybrid_r16_draft05b_t4.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_draft05b_t4", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 5.561, "ttft_s": 3.665, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 65.359, "total_time_s": 69.025, "decode_tok_per_s": 0.719, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.695, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 81, "accepted_tokens": 34, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55083.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6203.0, "current_rss_mib": 6172.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 3.4, "page_cache_delta_since_start_mib": -159.2, "idle_watts": null, "avg_watts": 9.77, "net_watts": null, "joules_per_tok": 14.04, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round4", "source_file": "r4_hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 4.735, "ttft_s": 7.575, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 147.639, "total_time_s": 155.213, "decode_tok_per_s": 0.21, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.206, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 25, "drafted_tokens": 29, "accepted_tokens": 7, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 125190.63, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5596.0, "current_rss_mib": 5596.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 58.6, "page_cache_delta_since_start_mib": 73.6, "idle_watts": null, "avg_watts": 9.86, "net_watts": null, "joules_per_tok": 47.84, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping critical layers in memory, and asynchrono" }, { "round": "round4", "source_file": "r4_hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 4.735, "ttft_s": 6.145, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 28.747, "total_time_s": 34.892, "decode_tok_per_s": 1.635, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.376, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 43, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 30045.75, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5599.5, "current_rss_mib": 5599.5, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 12.2, "page_cache_delta_since_start_mib": 85.7, "idle_watts": null, "avg_watts": 10.26, "net_watts": null, "joules_per_tok": 7.45, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round4", "source_file": "r4_hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 4.735, "ttft_s": 5.783, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 196.684, "total_time_s": 202.467, "decode_tok_per_s": 0.239, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.237, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 30, "drafted_tokens": 32, "accepted_tokens": 18, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 150228.75, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5601.2, "current_rss_mib": 5164.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 166.6, "page_cache_delta_since_start_mib": 252.4, "idle_watts": null, "avg_watts": 9.69, "net_watts": null, "joules_per_tok": 40.88, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round4", "source_file": "r4_stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r4_stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 2.595, "ttft_s": 10.378, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 265.468, "total_time_s": 275.846, "decode_tok_per_s": 0.117, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.116, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 241429.79, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 3073.8, "current_rss_mib": 3073.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 21.8, "page_cache_delta_since_start_mib": 35.4, "idle_watts": null, "avg_watts": 9.86, "net_watts": null, "joules_per_tok": 85.0, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_auto_budget_5800_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_auto_budget_5800_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.523, "ttft_s": 5.366, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 51.725, "total_time_s": 57.091, "decode_tok_per_s": 0.599, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.561, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 80269.73, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5580.8, "current_rss_mib": 5580.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -58.6, "page_cache_delta_since_start_mib": 369.9, "idle_watts": null, "avg_watts": 11.19, "net_watts": null, "joules_per_tok": 19.97, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_auto_budget_5800_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_auto_budget_5800_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.523, "ttft_s": 3.662, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 13.455, "total_time_s": 17.117, "decode_tok_per_s": 3.493, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.804, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 24890.4, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5584.6, "current_rss_mib": 5584.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -12.0, "page_cache_delta_since_start_mib": 357.9, "idle_watts": null, "avg_watts": 11.84, "net_watts": null, "joules_per_tok": 4.22, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_auto_budget_5800_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_auto_budget_5800_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.523, "ttft_s": 3.534, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 33.908, "total_time_s": 37.442, "decode_tok_per_s": 1.386, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.282, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 115, "accepted_tokens": 34, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55231.6, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5584.6, "current_rss_mib": 5584.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -12.8, "page_cache_delta_since_start_mib": 345.1, "idle_watts": null, "avg_watts": 11.27, "net_watts": null, "joules_per_tok": 8.79, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round5", "source_file": "r5_zc_hybrid_r14_draft15b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r14_draft15b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 7.065, "ttft_s": 7.763, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 83.656, "total_time_s": 91.419, "decode_tok_per_s": 0.371, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.35, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 103, "accepted_tokens": 16, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 85314.73, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5865.6, "current_rss_mib": 5865.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -41.6, "page_cache_delta_since_start_mib": -289.5, "idle_watts": null, "avg_watts": 10.33, "net_watts": null, "joules_per_tok": 29.51, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_hybrid_r14_draft15b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r14_draft15b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 7.065, "ttft_s": 5.808, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 22.176, "total_time_s": 27.984, "decode_tok_per_s": 2.119, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.715, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 26608.85, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5870.3, "current_rss_mib": 5870.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 4.3, "page_cache_delta_since_start_mib": -285.2, "idle_watts": null, "avg_watts": 9.81, "net_watts": null, "joules_per_tok": 5.72, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_hybrid_r14_draft15b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r14_draft15b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 7.065, "ttft_s": 5.726, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 50.56, "total_time_s": 56.285, "decode_tok_per_s": 0.93, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.853, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 10, "drafted_tokens": 94, "accepted_tokens": 35, "resident_layers": 14, "streamed_layers": 34, "n_layers": 48, "resident_mib": 3244.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 53051.28, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5870.3, "current_rss_mib": 5870.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -14.2, "page_cache_delta_since_start_mib": -299.4, "idle_watts": null, "avg_watts": 10.43, "net_watts": null, "joules_per_tok": 12.23, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.533, "ttft_s": 5.432, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 50.131, "total_time_s": 55.563, "decode_tok_per_s": 0.618, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.576, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 80269.73, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5542.5, "current_rss_mib": 5542.5, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -51.2, "page_cache_delta_since_start_mib": -1137.8, "idle_watts": null, "avg_watts": 11.45, "net_watts": null, "joules_per_tok": 19.88, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.533, "ttft_s": 3.664, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 13.386, "total_time_s": 17.05, "decode_tok_per_s": 3.511, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.815, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 25038.13, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5547.2, "current_rss_mib": 5547.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -13.0, "page_cache_delta_since_start_mib": -1150.8, "idle_watts": null, "avg_watts": 10.73, "net_watts": null, "joules_per_tok": 3.81, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.533, "ttft_s": 3.452, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 33.828, "total_time_s": 37.281, "decode_tok_per_s": 1.389, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.288, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 115, "accepted_tokens": 34, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55083.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5547.2, "current_rss_mib": 5547.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 6.8, "page_cache_delta_since_start_mib": -1144.0, "idle_watts": null, "avg_watts": 10.97, "net_watts": null, "joules_per_tok": 8.52, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.341, "ttft_s": 7.225, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 52.134, "total_time_s": 59.359, "decode_tok_per_s": 0.595, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.539, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 75, "accepted_tokens": 21, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55083.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5575.7, "current_rss_mib": 5575.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -24.5, "page_cache_delta_since_start_mib": 257.5, "idle_watts": null, "avg_watts": 10.25, "net_watts": null, "joules_per_tok": 19.02, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described in the document is layer-streaming inference, which manages to fit large language models into limited memory by " }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.341, "ttft_s": 5.607, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 20.974, "total_time_s": 26.581, "decode_tok_per_s": 2.241, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.806, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 25038.13, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5580.9, "current_rss_mib": 5580.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.2, "page_cache_delta_since_start_mib": 257.7, "idle_watts": null, "avg_watts": 10.46, "net_watts": null, "joules_per_tok": 5.79, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.341, "ttft_s": 5.345, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 52.259, "total_time_s": 57.603, "decode_tok_per_s": 0.899, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.833, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 107, "accepted_tokens": 32, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 55083.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5580.9, "current_rss_mib": 5580.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.2, "page_cache_delta_since_start_mib": 257.9, "idle_watts": null, "avg_watts": 9.78, "net_watts": null, "joules_per_tok": 11.73, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers in the context of streaming transformer layers on an 8 GB edge " }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 5.227, "ttft_s": 6.954, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 125.207, "total_time_s": 132.161, "decode_tok_per_s": 0.248, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.242, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 25, "drafted_tokens": 37, "accepted_tokens": 7, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 125338.35, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 4992.8, "current_rss_mib": 4992.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.9, "page_cache_delta_since_start_mib": -11.4, "idle_watts": null, "avg_watts": 9.98, "net_watts": null, "joules_per_tok": 41.23, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 5.227, "ttft_s": 5.484, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 26.205, "total_time_s": 31.689, "decode_tok_per_s": 1.794, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.515, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 31, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 29898.02, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 4995.1, "current_rss_mib": 4995.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.7, "page_cache_delta_since_start_mib": -10.7, "idle_watts": null, "avg_watts": 9.8, "net_watts": null, "joules_per_tok": 6.47, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_hybrid_r16_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r16_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 5.227, "ttft_s": 5.319, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 145.983, "total_time_s": 151.302, "decode_tok_per_s": 0.322, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.317, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 29, "drafted_tokens": 30, "accepted_tokens": 19, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 145368.85, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 4997.1, "current_rss_mib": 4997.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 3.4, "page_cache_delta_since_start_mib": -7.2, "idle_watts": null, "avg_watts": 9.8, "net_watts": null, "joules_per_tok": 30.88, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round5", "source_file": "r5_zc_hybrid_r19_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r19_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 7.737, "ttft_s": 4.043, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 31.922, "total_time_s": 35.965, "decode_tok_per_s": 0.971, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.89, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 72732.06, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5890.0, "current_rss_mib": 5890.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -77.4, "page_cache_delta_since_start_mib": -448.5, "idle_watts": null, "avg_watts": 12.96, "net_watts": null, "joules_per_tok": 14.57, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_hybrid_r19_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r19_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 7.737, "ttft_s": 2.45, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 8.27, "total_time_s": 10.72, "decode_tok_per_s": 5.684, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 4.478, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 22728.77, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5894.9, "current_rss_mib": 5894.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.4, "page_cache_delta_since_start_mib": -448.1, "idle_watts": null, "avg_watts": 12.7, "net_watts": null, "joules_per_tok": 2.84, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_hybrid_r19_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_hybrid_r19_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 7.737, "ttft_s": 2.262, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 21.636, "total_time_s": 23.898, "decode_tok_per_s": 2.172, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.009, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 115, "accepted_tokens": 34, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 50003.29, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5895.1, "current_rss_mib": 5895.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -33.9, "page_cache_delta_since_start_mib": -482.0, "idle_watts": null, "avg_watts": 12.48, "net_watts": null, "joules_per_tok": 6.21, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round5", "source_file": "r5_zc_multiturn_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_multiturn_r16_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 5.897, "ttft_s": 7.188, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 78.222, "total_time_s": 85.41, "decode_tok_per_s": 0.396, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.375, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 16, "streamed_layers": 32, "n_layers": 48, "resident_mib": 3558.5, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 80122.0, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5574.5, "current_rss_mib": 5574.5, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -74.1, "page_cache_delta_since_start_mib": 16.3, "idle_watts": null, "avg_watts": 10.19, "net_watts": null, "joules_per_tok": 27.19, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_multiturn_r16_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_multiturn_r16_draft05b_multiturn", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "multi_turn_prefix_reuse", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": null, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": null, "total_time_s": null, "decode_tok_per_s": null, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": null, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": null, "current_rss_mib": null, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": null, "page_cache_delta_since_start_mib": null, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": 340, "turn1_ttft_s": 5.174, "turn2_prompt_tokens": 377, "turn2_ttft_s": 5.4, "ttft_speedup_x": 0.96, "notes": null, "output_preview": " O_DIRECT prevents page-cache thrashing by bypassing the operating system's page cache" }, { "round": "round5", "source_file": "r5_zc_stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 1.99, "ttft_s": 6.96, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 163.279, "total_time_s": 170.239, "decode_tok_per_s": 0.19, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.188, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 241429.79, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2492.4, "current_rss_mib": 2492.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.8, "page_cache_delta_since_start_mib": 1.3, "idle_watts": null, "avg_watts": 10.71, "net_watts": null, "joules_per_tok": 56.97, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 1.99, "ttft_s": 5.618, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 247.709, "total_time_s": 253.327, "decode_tok_per_s": 0.19, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.189, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 48, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 361728.64, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2492.4, "current_rss_mib": 2492.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 36.1, "page_cache_delta_since_start_mib": 37.4, "idle_watts": null, "avg_watts": 10.67, "net_watts": null, "joules_per_tok": 56.33, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_stream_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 1.99, "ttft_s": 5.368, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 247.761, "total_time_s": 253.129, "decode_tok_per_s": 0.19, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.19, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 48, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 362061.48, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2492.4, "current_rss_mib": 2492.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 26.8, "page_cache_delta_since_start_mib": 64.2, "idle_watts": null, "avg_watts": 10.7, "net_watts": null, "joules_per_tok": 56.41, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round5", "source_file": "r5_zc_stream_r0_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_stream_r0_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 2.011, "ttft_s": 6.246, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 109.781, "total_time_s": 116.027, "decode_tok_per_s": 0.282, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.276, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 25, "drafted_tokens": 37, "accepted_tokens": 7, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 188653.43, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2500.8, "current_rss_mib": 2500.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.9, "page_cache_delta_since_start_mib": 1.3, "idle_watts": null, "avg_watts": 11.16, "net_watts": null, "joules_per_tok": 40.47, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round5", "source_file": "r5_zc_stream_r0_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_stream_r0_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 2.011, "ttft_s": 4.906, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 22.969, "total_time_s": 27.875, "decode_tok_per_s": 2.046, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.722, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 31, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 45236.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2503.1, "current_rss_mib": 2503.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.2, "page_cache_delta_since_start_mib": 1.5, "idle_watts": null, "avg_watts": 11.03, "net_watts": null, "joules_per_tok": 6.41, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round5", "source_file": "r5_zc_stream_r0_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r5_zc_stream_r0_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 2.011, "ttft_s": 4.612, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 128.056, "total_time_s": 132.668, "decode_tok_per_s": 0.367, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.362, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 29, "drafted_tokens": 30, "accepted_tokens": 19, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 218644.93, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2505.1, "current_rss_mib": 2505.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 1.5, "page_cache_delta_since_start_mib": 3.0, "idle_watts": null, "avg_watts": 10.97, "net_watts": null, "joules_per_tok": 30.31, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round6", "source_file": "r6_auto_budget_6000_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_auto_budget_6000_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.615, "ttft_s": 7.057, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 77.232, "total_time_s": 84.289, "decode_tok_per_s": 0.401, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.38, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 17, "streamed_layers": 31, "n_layers": 48, "resident_mib": 3706.3, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 77758.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5715.0, "current_rss_mib": 5715.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -91.1, "page_cache_delta_since_start_mib": 151.9, "idle_watts": null, "avg_watts": 10.11, "net_watts": null, "joules_per_tok": 26.62, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round6", "source_file": "r6_auto_budget_6000_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_auto_budget_6000_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.615, "ttft_s": 5.474, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 20.38, "total_time_s": 25.854, "decode_tok_per_s": 2.306, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.857, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 17, "streamed_layers": 31, "n_layers": 48, "resident_mib": 3706.3, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 24299.49, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5719.7, "current_rss_mib": 5719.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.4, "page_cache_delta_since_start_mib": 152.4, "idle_watts": null, "avg_watts": 10.14, "net_watts": null, "joules_per_tok": 5.46, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round6", "source_file": "r6_auto_budget_6000_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_auto_budget_6000_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.615, "ttft_s": 5.291, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 51.813, "total_time_s": 57.105, "decode_tok_per_s": 0.907, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.841, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 115, "accepted_tokens": 34, "resident_layers": 17, "streamed_layers": 31, "n_layers": 48, "resident_mib": 3706.3, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 53458.88, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5719.7, "current_rss_mib": 5719.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.1, "page_cache_delta_since_start_mib": 152.5, "idle_watts": null, "avg_watts": 9.78, "net_watts": null, "joules_per_tok": 11.64, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round6", "source_file": "r6_hybrid_r12_draft15b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r12_draft15b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.343, "ttft_s": 8.156, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 88.085, "total_time_s": 96.241, "decode_tok_per_s": 0.352, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.332, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 103, "accepted_tokens": 16, "resident_layers": 12, "streamed_layers": 36, "n_layers": 48, "resident_mib": 2949.0, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 90023.29, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5580.4, "current_rss_mib": 5580.4, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -82.5, "page_cache_delta_since_start_mib": 391.5, "idle_watts": null, "avg_watts": 10.05, "net_watts": null, "joules_per_tok": 30.22, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round6", "source_file": "r6_hybrid_r12_draft15b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r12_draft15b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.343, "ttft_s": 6.099, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 23.446, "total_time_s": 29.545, "decode_tok_per_s": 2.005, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.625, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 12, "streamed_layers": 36, "n_layers": 48, "resident_mib": 2949.0, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 28086.11, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5585.0, "current_rss_mib": 5585.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -69.6, "page_cache_delta_since_start_mib": 321.9, "idle_watts": null, "avg_watts": 9.65, "net_watts": null, "joules_per_tok": 5.94, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round6", "source_file": "r6_hybrid_r12_draft15b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r12_draft15b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-1.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.343, "ttft_s": 6.014, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 63.741, "total_time_s": 69.755, "decode_tok_per_s": 0.737, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.688, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 10, "drafted_tokens": 94, "accepted_tokens": 35, "resident_layers": 12, "streamed_layers": 36, "n_layers": 48, "resident_mib": 2949.0, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 56172.23, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5585.0, "current_rss_mib": 5585.0, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 2.4, "page_cache_delta_since_start_mib": 324.3, "idle_watts": null, "avg_watts": 9.78, "net_watts": null, "joules_per_tok": 14.22, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round6", "source_file": "r6_hybrid_r19_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 7.145, "ttft_s": 3.976, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 20.457, "total_time_s": 24.432, "decode_tok_per_s": 1.515, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.31, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 75, "accepted_tokens": 21, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 50003.29, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6037.1, "current_rss_mib": 6037.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -33.1, "page_cache_delta_since_start_mib": -382.9, "idle_watts": null, "avg_watts": 13.05, "net_watts": null, "joules_per_tok": 9.97, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described in the document is layer-streaming inference, which manages to fit large language models into limited memory by " }, { "round": "round6", "source_file": "r6_hybrid_r19_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 7.145, "ttft_s": 2.358, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 7.928, "total_time_s": 10.286, "decode_tok_per_s": 5.928, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 4.666, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 22728.77, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6042.3, "current_rss_mib": 6042.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -35.3, "page_cache_delta_since_start_mib": -418.2, "idle_watts": null, "avg_watts": 13.81, "net_watts": null, "joules_per_tok": 2.96, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round6", "source_file": "r6_hybrid_r19_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 7.145, "ttft_s": 2.168, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 20.461, "total_time_s": 22.629, "decode_tok_per_s": 2.297, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.121, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 107, "accepted_tokens": 32, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 50003.29, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6042.3, "current_rss_mib": 6042.3, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 4.3, "page_cache_delta_since_start_mib": -413.9, "idle_watts": null, "avg_watts": 13.11, "net_watts": null, "joules_per_tok": 6.18, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers in the context of streaming transformer layers on an 8 GB edge " }, { "round": "round6", "source_file": "r6_hybrid_r19_draft05b_th2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_draft05b_th2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 7.943, "ttft_s": 3.957, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 31.355, "total_time_s": 35.312, "decode_tok_per_s": 0.989, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.906, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 72732.06, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5927.6, "current_rss_mib": 5927.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -76.5, "page_cache_delta_since_start_mib": -829.4, "idle_watts": null, "avg_watts": 12.61, "net_watts": null, "joules_per_tok": 13.92, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round6", "source_file": "r6_hybrid_r19_draft05b_th2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_draft05b_th2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 7.943, "ttft_s": 2.355, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 7.874, "total_time_s": 10.23, "decode_tok_per_s": 5.969, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 4.692, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 22728.77, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5932.5, "current_rss_mib": 5932.5, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -11.2, "page_cache_delta_since_start_mib": -840.6, "idle_watts": null, "avg_watts": 13.02, "net_watts": null, "joules_per_tok": 2.78, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round6", "source_file": "r6_hybrid_r19_draft05b_th2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_draft05b_th2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 7.943, "ttft_s": 2.189, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 32.682, "total_time_s": 34.871, "decode_tok_per_s": 1.438, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.377, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 115, "accepted_tokens": 34, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 50003.29, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5932.7, "current_rss_mib": 5932.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -2.4, "page_cache_delta_since_start_mib": -843.0, "idle_watts": null, "avg_watts": 11.23, "net_watts": null, "joules_per_tok": 8.16, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round6", "source_file": "r6_hybrid_r19_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.029, "ttft_s": 4.353, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 64.64, "total_time_s": 68.993, "decode_tok_per_s": 0.48, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.464, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 25, "drafted_tokens": 37, "accepted_tokens": 7, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 113643.85, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5458.6, "current_rss_mib": 5458.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 3.9, "page_cache_delta_since_start_mib": -45.7, "idle_watts": null, "avg_watts": 11.94, "net_watts": null, "joules_per_tok": 25.74, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round6", "source_file": "r6_hybrid_r19_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.029, "ttft_s": 3.152, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 13.547, "total_time_s": 16.699, "decode_tok_per_s": 3.469, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 2.874, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 31, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 27274.52, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5460.9, "current_rss_mib": 5460.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.7, "page_cache_delta_since_start_mib": -45.0, "idle_watts": null, "avg_watts": 11.95, "net_watts": null, "joules_per_tok": 4.16, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round6", "source_file": "r6_hybrid_r19_ngram.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r19_ngram", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.029, "ttft_s": 2.87, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 75.455, "total_time_s": 78.325, "decode_tok_per_s": 0.623, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.613, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 29, "drafted_tokens": 30, "accepted_tokens": 19, "resident_layers": 19, "streamed_layers": 29, "n_layers": 48, "resident_mib": 4020.4, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 131826.86, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5463.0, "current_rss_mib": 5462.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 22.1, "page_cache_delta_since_start_mib": -23.0, "idle_watts": null, "avg_watts": 11.6, "net_watts": null, "joules_per_tok": 18.93, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round6", "source_file": "r6_hybrid_r20_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r20_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.749, "ttft_s": 4.078, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 30.016, "total_time_s": 34.094, "decode_tok_per_s": 1.033, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.939, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 16, "drafted_tokens": 107, "accepted_tokens": 16, "resident_layers": 20, "streamed_layers": 28, "n_layers": 48, "resident_mib": 4168.1, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 70534.86, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6152.5, "current_rss_mib": 6069.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -74.6, "page_cache_delta_since_start_mib": -171.5, "idle_watts": null, "avg_watts": 12.89, "net_watts": null, "joules_per_tok": 13.73, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round6", "source_file": "r6_hybrid_r20_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r20_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.749, "ttft_s": 2.305, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 7.868, "total_time_s": 10.173, "decode_tok_per_s": 5.974, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 4.718, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 36, "resident_layers": 20, "streamed_layers": 28, "n_layers": 48, "resident_mib": 4168.1, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 21990.14, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6152.5, "current_rss_mib": 6074.6, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.7, "page_cache_delta_since_start_mib": -170.8, "idle_watts": null, "avg_watts": 12.89, "net_watts": null, "joules_per_tok": 2.73, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round6", "source_file": "r6_hybrid_r20_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_hybrid_r20_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.749, "ttft_s": 2.124, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 22.288, "total_time_s": 24.412, "decode_tok_per_s": 2.109, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 1.966, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 115, "accepted_tokens": 34, "resident_layers": 20, "streamed_layers": 28, "n_layers": 48, "resident_mib": 4168.1, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 48378.3, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 6152.5, "current_rss_mib": 6074.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -2.6, "page_cache_delta_since_start_mib": -173.4, "idle_watts": null, "avg_watts": 12.84, "net_watts": null, "joules_per_tok": 6.53, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round6", "source_file": "r6_multiturn_long_r19_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_multiturn_long_r19_draft05b_multiturn", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "multi_turn_prefix_reuse", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": null, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": null, "total_time_s": null, "decode_tok_per_s": null, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": null, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": null, "current_rss_mib": null, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": null, "page_cache_delta_since_start_mib": null, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": 1377, "turn1_ttft_s": 16.705, "turn2_prompt_tokens": 1414, "turn2_ttft_s": 3.416, "ttft_speedup_x": 4.89, "notes": null, "output_preview": " O_DIRECT prevents page-cache thrashing by bypassing the operating system's page cache" }, { "round": "round6", "source_file": "r6_stream_r0_th2.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_stream_r0_th2", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 1.913, "ttft_s": 7.374, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 174.278, "total_time_s": 181.652, "decode_tok_per_s": 0.178, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.176, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": null, "io_mib_read": 241429.79, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2484.7, "current_rss_mib": 2484.7, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 19.2, "page_cache_delta_since_start_mib": 25.6, "idle_watts": null, "avg_watts": 10.58, "net_watts": null, "joules_per_tok": 60.05, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of buffers for active computation, keeping only essential layers in memory, and asyn" }, { "round": "round6_32b", "source_file": "r6_32b_auto5600_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_auto5600_draft05b", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 6.838, "ttft_s": 11.69, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 92.127, "total_time_s": 103.817, "decode_tok_per_s": 0.336, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.308, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 13, "drafted_tokens": 87, "accepted_tokens": 19, "resident_layers": 6, "streamed_layers": 58, "n_layers": 64, "resident_mib": 2813.1, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 209768.71, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5289.1, "current_rss_mib": 5289.1, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -59.0, "page_cache_delta_since_start_mib": -4749.6, "idle_watts": null, "avg_watts": 12.24, "net_watts": null, "joules_per_tok": 39.7, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described involves layer-streaming inference, which uses a small fixed pool of reusable compute buffers and asynchronously" }, { "round": "round6_32b", "source_file": "r6_32b_auto5600_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_auto5600_draft05b", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 6.838, "ttft_s": 8.416, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 49.8, "total_time_s": 58.216, "decode_tok_per_s": 0.944, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.825, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 7, "drafted_tokens": 66, "accepted_tokens": 41, "resident_layers": 6, "streamed_layers": 58, "n_layers": 64, "resident_mib": 2813.1, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 112605.38, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5293.8, "current_rss_mib": 5293.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -16.0, "page_cache_delta_since_start_mib": -4765.5, "idle_watts": null, "avg_watts": 11.79, "net_watts": null, "joules_per_tok": 14.3, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\n pub model_path: PathBuf,\n pub n_ctx: u32,\n pub n_batch: u32,\n pub resident_layers: Option<u32>,\n " }, { "round": "round6_32b", "source_file": "r6_32b_auto5600_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_auto5600_draft05b", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 6.838, "ttft_s": 7.916, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 92.905, "total_time_s": 100.821, "decode_tok_per_s": 0.506, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.476, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 13, "drafted_tokens": 127, "accepted_tokens": 35, "resident_layers": 6, "streamed_layers": 58, "n_layers": 64, "resident_mib": 2813.1, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 209657.66, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5293.8, "current_rss_mib": 5293.8, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -0.1, "page_cache_delta_since_start_mib": -4765.6, "idle_watts": null, "avg_watts": 11.96, "net_watts": null, "joules_per_tok": 25.12, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers can b" }, { "round": "round6_32b", "source_file": "r6_32b_auto5600_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_auto5600_draft05b_temp07", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 5.459, "ttft_s": 16.181, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 146.116, "total_time_s": 162.297, "decode_tok_per_s": 0.212, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.197, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 13, "drafted_tokens": 89, "accepted_tokens": 19, "resident_layers": 6, "streamed_layers": 58, "n_layers": 64, "resident_mib": 2813.1, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 209582.03, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5293.9, "current_rss_mib": 5293.9, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -95.5, "page_cache_delta_since_start_mib": 5.7, "idle_watts": null, "avg_watts": 10.75, "net_watts": null, "joules_per_tok": 54.52, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described involves layer-streaming inference, where a fixed pool of reusable compute buffers is allocated for transformer " }, { "round": "round6_32b", "source_file": "r6_32b_auto5600_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_auto5600_draft05b_temp07", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 5.459, "ttft_s": 12.943, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 73.28, "total_time_s": 86.223, "decode_tok_per_s": 0.641, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.557, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 7, "drafted_tokens": 66, "accepted_tokens": 41, "resident_layers": 6, "streamed_layers": 58, "n_layers": 64, "resident_mib": 2813.1, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 112792.07, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5299.2, "current_rss_mib": 5299.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.0, "page_cache_delta_since_start_mib": 5.7, "idle_watts": null, "avg_watts": 10.84, "net_watts": null, "joules_per_tok": 19.47, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\n pub model_path: PathBuf,\n pub n_ctx: u32,\n pub n_batch: u32,\n pub resident_layers: Option<u32>,\n " }, { "round": "round6_32b", "source_file": "r6_32b_auto5600_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_auto5600_draft05b_temp07", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 5.459, "ttft_s": 12.177, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 121.648, "total_time_s": 133.825, "decode_tok_per_s": 0.386, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.359, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 11, "drafted_tokens": 98, "accepted_tokens": 37, "resident_layers": 6, "streamed_layers": 58, "n_layers": 64, "resident_mib": 2813.1, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 177244.68, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 5299.2, "current_rss_mib": 5299.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -16.2, "page_cache_delta_since_start_mib": -10.5, "idle_watts": null, "avg_watts": 10.53, "net_watts": null, "joules_per_tok": 29.35, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " In the context of streaming transformer layers on an 8 GB edge device, understanding how page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffer" }, { "round": "round6_32b", "source_file": "r6_32b_stream_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r6_32b_stream_r0", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 2.137, "ttft_s": 14.479, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": 330.254, "total_time_s": 344.733, "decode_tok_per_s": 0.094, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": 0.093, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 64, "n_layers": 64, "resident_mib": 1026.8, "slot_mib": 297.7, "expert_slot_mib": null, "io_mib_read": 573082.1, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": false, "is_unified_memory": null, "direct_io_active": true, "peak_rss_mib": 2957.2, "current_rss_mib": 2957.2, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 6.9, "page_cache_delta_since_start_mib": 8.2, "idle_watts": null, "avg_watts": 11.06, "net_watts": null, "joules_per_tok": 119.1, "net_joules_per_tok": null, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described involves layer-streaming inference, which uses a small fixed pool of reusable compute buffers and asynchronously" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": true, "vram_budget_mb": 0, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 4.936, "ttft_s": 4.467, "ttft_s_min": 4.464, "ttft_s_max": 4.504, "decode_time_s": 40.794, "total_time_s": 45.286, "decode_tok_per_s": 0.76, "decode_tok_per_s_min": 0.755, "decode_tok_per_s_max": 0.76, "total_tok_per_s": 0.707, "total_tok_per_s_min": 0.703, "total_tok_per_s_max": 0.707, "sweeps": 15, "drafted_tokens": 77, "accepted_tokens": 18, "resident_layers": 4, "streamed_layers": 44, "n_layers": 48, "resident_mib": 1692.4, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 103107.13, "pread_mib_s": 2796.4, "effective_io_mib_s": 2276.8, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3709.2, "current_rss_mib": 3709.2, "mem_avail_before_mib": 2397.2, "mem_avail_after_mib": 2403.7, "swap_delta_mib": -0.3, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -0.4, "page_cache_delta_since_start_mib": -1657.4, "idle_watts": 7.77, "avg_watts": 13.21, "net_watts": 5.43, "joules_per_tok": 18.69, "net_joules_per_tok": 7.69, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": true, "vram_budget_mb": 0, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 4.936, "ttft_s": 3.258, "ttft_s_min": 3.236, "ttft_s_max": 3.281, "decode_time_s": 14.884, "total_time_s": 18.158, "decode_tok_per_s": 3.158, "decode_tok_per_s_min": 2.244, "decode_tok_per_s_max": 3.159, "total_tok_per_s": 2.643, "total_tok_per_s_min": 1.985, "total_tok_per_s_max": 2.646, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 43, "resident_layers": 4, "streamed_layers": 44, "n_layers": 48, "resident_mib": 1692.4, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 41242.85, "pread_mib_s": 2799.5, "effective_io_mib_s": 2273.3, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3713.9, "current_rss_mib": 3713.9, "mem_avail_before_mib": 2382.9, "mem_avail_after_mib": 2385.3, "swap_delta_mib": -0.3, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.2, "page_cache_delta_since_start_mib": -1654.7, "idle_watts": 7.77, "avg_watts": 12.88, "net_watts": 5.11, "joules_per_tok": 4.87, "net_joules_per_tok": 1.93, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": true, "vram_budget_mb": 0, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 4.936, "ttft_s": 2.952, "ttft_s_min": 2.921, "ttft_s_max": 2.995, "decode_time_s": 29.778, "total_time_s": 32.699, "decode_tok_per_s": 1.578, "decode_tok_per_s_min": 1.564, "decode_tok_per_s_max": 1.582, "total_tok_per_s": 1.468, "total_tok_per_s_min": 1.452, "total_tok_per_s_max": 1.469, "sweeps": 11, "drafted_tokens": 97, "accepted_tokens": 38, "resident_layers": 4, "streamed_layers": 44, "n_layers": 48, "resident_mib": 1692.4, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 75611.89, "pread_mib_s": 2794.3, "effective_io_mib_s": 2312.4, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3715.9, "current_rss_mib": 3715.9, "mem_avail_before_mib": 2387.5, "mem_avail_after_mib": 2388.8, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.8, "page_cache_delta_since_start_mib": -1654.0, "idle_watts": 7.77, "avg_watts": 12.84, "net_watts": 5.06, "joules_per_tok": 8.75, "net_joules_per_tok": 3.45, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b_multiturn", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "multi_turn_prefix_reuse", "chat_template_applied": null, "warmup": null, "reps": null, "temperature": null, "seed": null, "threads": null, "kv_cache_type": null, "vram_budget_auto": null, "vram_budget_mb": null, "prompt_tokens": null, "completion_tokens": null, "load_time_s": null, "ttft_s": null, "ttft_s_min": null, "ttft_s_max": null, "decode_time_s": null, "total_time_s": null, "decode_tok_per_s": null, "decode_tok_per_s_min": null, "decode_tok_per_s_max": null, "total_tok_per_s": null, "total_tok_per_s_min": null, "total_tok_per_s_max": null, "sweeps": null, "drafted_tokens": null, "accepted_tokens": null, "resident_layers": null, "streamed_layers": null, "n_layers": null, "resident_mib": null, "slot_mib": null, "expert_slot_mib": null, "io_mib_read": null, "pread_mib_s": null, "effective_io_mib_s": null, "expert_cache_hits": null, "expert_cache_misses": null, "expert_cache_hit_rate": null, "expert_cache_mib_saved": null, "moe_layers": null, "moe_active_streaming": null, "uma_zero_copy": null, "is_unified_memory": null, "direct_io_active": null, "peak_rss_mib": null, "current_rss_mib": null, "mem_avail_before_mib": null, "mem_avail_after_mib": null, "swap_delta_mib": null, "cuda_max_allocated_mib": null, "page_cache_delta_mib": null, "page_cache_delta_since_start_mib": null, "idle_watts": null, "avg_watts": null, "net_watts": null, "joules_per_tok": null, "net_joules_per_tok": null, "turn1_prompt_tokens": 1377, "turn1_ttft_s": 14.877, "turn2_prompt_tokens": 1414, "turn2_ttft_s": 2.993, "ttft_speedup_x": 4.97, "notes": null, "output_preview": " O_DIRECT prevents page-cache thrashing by bypassing the operating system's page cache" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": false, "warmup": false, "reps": 3, "temperature": 0.7, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": true, "vram_budget_mb": 0, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 3.799, "ttft_s": 8.945, "ttft_s_min": 8.935, "ttft_s_max": 9.013, "decode_time_s": 103.065, "total_time_s": 112.011, "decode_tok_per_s": 0.301, "decode_tok_per_s_min": 0.263, "decode_tok_per_s_max": 0.42, "total_tok_per_s": 0.286, "total_tok_per_s_min": 0.252, "total_tok_per_s_max": 0.386, "sweeps": 15, "drafted_tokens": 72, "accepted_tokens": 18, "resident_layers": 3, "streamed_layers": 45, "n_layers": 48, "resident_mib": 1526.0, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 105603.4, "pread_mib_s": 1031.5, "effective_io_mib_s": 942.8, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3542.2, "current_rss_mib": 3542.2, "mem_avail_before_mib": 2538.7, "mem_avail_after_mib": 2522.2, "swap_delta_mib": -0.5, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 1.1, "page_cache_delta_since_start_mib": 11.2, "idle_watts": 7.84, "avg_watts": 9.95, "net_watts": 2.11, "joules_per_tok": 34.83, "net_joules_per_tok": 7.37, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of compute buffers and asynchronously streaming transformer blocks from NVMe storage" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": false, "warmup": false, "reps": 3, "temperature": 0.7, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": true, "vram_budget_mb": 0, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 3.799, "ttft_s": 7.714, "ttft_s_min": 7.712, "ttft_s_max": 7.719, "decode_time_s": 37.27, "total_time_s": 44.989, "decode_tok_per_s": 1.261, "decode_tok_per_s_min": 1.261, "decode_tok_per_s_max": 1.262, "total_tok_per_s": 1.067, "total_tok_per_s_min": 1.067, "total_tok_per_s_max": 1.068, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 43, "resident_layers": 3, "streamed_layers": 45, "n_layers": 48, "resident_mib": 1526.0, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 42241.36, "pread_mib_s": 1031.2, "effective_io_mib_s": 938.9, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3546.9, "current_rss_mib": 3546.9, "mem_avail_before_mib": 2540.0, "mem_avail_after_mib": 2537.3, "swap_delta_mib": -0.5, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.1, "page_cache_delta_since_start_mib": 11.9, "idle_watts": 7.84, "avg_watts": 9.98, "net_watts": 2.14, "joules_per_tok": 9.35, "net_joules_per_tok": 2.0, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round8_v010", "source_file": "r8_14b_auto_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_auto_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": false, "warmup": false, "reps": 3, "temperature": 0.7, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": true, "vram_budget_mb": 0, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 3.799, "ttft_s": 7.421, "ttft_s_min": 7.406, "ttft_s_max": 7.424, "decode_time_s": 111.118, "total_time_s": 118.524, "decode_tok_per_s": 0.423, "decode_tok_per_s_min": 0.373, "decode_tok_per_s_max": 0.454, "total_tok_per_s": 0.405, "total_tok_per_s_min": 0.36, "total_tok_per_s_max": 0.432, "sweeps": 15, "drafted_tokens": 112, "accepted_tokens": 34, "resident_layers": 3, "streamed_layers": 45, "n_layers": 48, "resident_mib": 1526.0, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 105603.4, "pread_mib_s": 1031.8, "effective_io_mib_s": 950.4, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3548.9, "current_rss_mib": 3548.9, "mem_avail_before_mib": 2527.6, "mem_avail_after_mib": 2521.9, "swap_delta_mib": -0.5, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.5, "page_cache_delta_since_start_mib": 34.2, "idle_watts": 7.84, "avg_watts": 9.97, "net_watts": 2.12, "joules_per_tok": 24.61, "net_joules_per_tok": 5.24, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " In the context of running transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU bu" }, { "round": "round8_v010", "source_file": "r8_14b_b5400_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_b5400_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 4.581, "ttft_s": 4.238, "ttft_s_min": 4.23, "ttft_s_max": 7.883, "decode_time_s": 37.618, "total_time_s": 42.173, "decode_tok_per_s": 0.824, "decode_tok_per_s_min": 0.817, "decode_tok_per_s_max": 0.827, "total_tok_per_s": 0.759, "total_tok_per_s_min": 0.706, "total_tok_per_s_max": 0.765, "sweeps": 15, "drafted_tokens": 77, "accepted_tokens": 18, "resident_layers": 7, "streamed_layers": 41, "n_layers": 48, "resident_mib": 2155.6, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 96159.67, "pread_mib_s": 2923.7, "effective_io_mib_s": 2280.1, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4171.4, "current_rss_mib": 4171.1, "mem_avail_before_mib": 1915.6, "mem_avail_after_mib": 1914.0, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -0.8, "page_cache_delta_since_start_mib": -59.6, "idle_watts": 7.7, "avg_watts": 13.29, "net_watts": 5.59, "joules_per_tok": 17.51, "net_joules_per_tok": 7.37, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of reusable buffers for compute, keeping essential layers in memory, and asynchronou" }, { "round": "round8_v010", "source_file": "r8_14b_b5400_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_b5400_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 4.581, "ttft_s": 3.006, "ttft_s_min": 3.006, "ttft_s_max": 5.221, "decode_time_s": 13.803, "total_time_s": 16.809, "decode_tok_per_s": 3.405, "decode_tok_per_s_min": 3.284, "decode_tok_per_s_max": 3.414, "total_tok_per_s": 2.856, "total_tok_per_s_min": 2.457, "total_tok_per_s_max": 2.862, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 43, "resident_layers": 7, "streamed_layers": 41, "n_layers": 48, "resident_mib": 2155.6, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 38463.87, "pread_mib_s": 2961.1, "effective_io_mib_s": 2288.3, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4175.9, "current_rss_mib": 4175.8, "mem_avail_before_mib": 1909.0, "mem_avail_after_mib": 1907.4, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.2, "page_cache_delta_since_start_mib": -66.9, "idle_watts": 7.7, "avg_watts": 13.16, "net_watts": 5.47, "joules_per_tok": 4.61, "net_joules_per_tok": 1.91, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round8_v010", "source_file": "r8_14b_b5400_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_b5400_draft05b", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 4.581, "ttft_s": 2.736, "ttft_s_min": 2.709, "ttft_s_max": 2.765, "decode_time_s": 27.66, "total_time_s": 30.396, "decode_tok_per_s": 1.699, "decode_tok_per_s_min": 1.685, "decode_tok_per_s_max": 1.713, "total_tok_per_s": 1.579, "total_tok_per_s_min": 1.568, "total_tok_per_s_max": 1.589, "sweeps": 11, "drafted_tokens": 97, "accepted_tokens": 38, "resident_layers": 7, "streamed_layers": 41, "n_layers": 48, "resident_mib": 2155.6, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 70517.09, "pread_mib_s": 2931.5, "effective_io_mib_s": 2319.9, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4177.9, "current_rss_mib": 4177.8, "mem_avail_before_mib": 1906.0, "mem_avail_after_mib": 1904.2, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -0.3, "page_cache_delta_since_start_mib": -68.6, "idle_watts": 7.7, "avg_watts": 13.29, "net_watts": 5.59, "joules_per_tok": 8.42, "net_joules_per_tok": 3.54, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers is cr" }, { "round": "round8_v010", "source_file": "r8_14b_b5400_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_b5400_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": false, "warmup": false, "reps": 3, "temperature": 0.7, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 4.333, "ttft_s": 8.337, "ttft_s_min": 8.329, "ttft_s_max": 8.556, "decode_time_s": 95.405, "total_time_s": 103.742, "decode_tok_per_s": 0.325, "decode_tok_per_s_min": 0.284, "decode_tok_per_s_max": 0.454, "total_tok_per_s": 0.308, "total_tok_per_s_min": 0.273, "total_tok_per_s_max": 0.417, "sweeps": 15, "drafted_tokens": 72, "accepted_tokens": 18, "resident_layers": 6, "streamed_layers": 42, "n_layers": 48, "resident_mib": 2006.6, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 98543.92, "pread_mib_s": 1026.8, "effective_io_mib_s": 948.4, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4023.3, "current_rss_mib": 4023.3, "mem_avail_before_mib": 2060.9, "mem_avail_after_mib": 2060.2, "swap_delta_mib": -0.3, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.3, "page_cache_delta_since_start_mib": 22.2, "idle_watts": 7.72, "avg_watts": 10.03, "net_watts": 2.31, "joules_per_tok": 32.52, "net_joules_per_tok": 7.47, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " Layer-streaming inference manages memory by allocating a small, fixed pool of compute buffers and asynchronously streaming transformer blocks from NVMe storage" }, { "round": "round8_v010", "source_file": "r8_14b_b5400_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_b5400_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": false, "warmup": false, "reps": 3, "temperature": 0.7, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 4.333, "ttft_s": 7.122, "ttft_s_min": 7.121, "ttft_s_max": 7.142, "decode_time_s": 34.313, "total_time_s": 41.434, "decode_tok_per_s": 1.37, "decode_tok_per_s_min": 1.369, "decode_tok_per_s_max": 1.372, "total_tok_per_s": 1.158, "total_tok_per_s_min": 1.158, "total_tok_per_s_max": 1.16, "sweeps": 6, "drafted_tokens": 50, "accepted_tokens": 43, "resident_layers": 6, "streamed_layers": 42, "n_layers": 48, "resident_mib": 2006.6, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 39208.95, "pread_mib_s": 1025.5, "effective_io_mib_s": 946.3, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4028.0, "current_rss_mib": 4028.0, "mem_avail_before_mib": 2052.5, "mem_avail_after_mib": 2056.1, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.1, "page_cache_delta_since_start_mib": 22.9, "idle_watts": 7.72, "avg_watts": 9.92, "net_watts": 2.2, "joules_per_tok": 8.56, "net_joules_per_tok": 1.9, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers: Option<u32>,\npub vram_budget_b" }, { "round": "round8_v010", "source_file": "r8_14b_b5400_draft05b_temp07.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_b5400_draft05b_temp07", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": false, "warmup": false, "reps": 3, "temperature": 0.7, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 4.333, "ttft_s": 6.816, "ttft_s_min": 6.803, "ttft_s_max": 6.841, "decode_time_s": 102.588, "total_time_s": 109.39, "decode_tok_per_s": 0.458, "decode_tok_per_s_min": 0.405, "decode_tok_per_s_max": 0.491, "total_tok_per_s": 0.439, "total_tok_per_s_min": 0.391, "total_tok_per_s_max": 0.468, "sweeps": 15, "drafted_tokens": 112, "accepted_tokens": 34, "resident_layers": 6, "streamed_layers": 42, "n_layers": 48, "resident_mib": 2006.6, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 98543.92, "pread_mib_s": 1027.3, "effective_io_mib_s": 961.5, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4030.0, "current_rss_mib": 4030.0, "mem_avail_before_mib": 2047.3, "mem_avail_after_mib": 2046.9, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 1.2, "page_cache_delta_since_start_mib": 25.8, "idle_watts": 7.72, "avg_watts": 9.77, "net_watts": 2.05, "joules_per_tok": 22.27, "net_joules_per_tok": 4.67, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " In the context of running transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU bu" }, { "round": "round8_v010", "source_file": "r8_14b_r0.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_14b_r0", "status": "ok", "model": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": false, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 31, "completion_tokens": 16, "load_time_s": 2.993, "ttft_s": 5.966, "ttft_s_min": 5.783, "ttft_s_max": 6.149, "decode_time_s": 86.265, "total_time_s": 92.231, "decode_tok_per_s": 0.174, "decode_tok_per_s_min": 0.174, "decode_tok_per_s_max": 0.174, "total_tok_per_s": 0.173, "total_tok_per_s_min": 0.173, "total_tok_per_s_max": 0.174, "sweeps": 16, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 1026.8, "slot_mib": 166.4, "expert_slot_mib": 0.0, "io_mib_read": 120631.69, "pread_mib_s": 1426.9, "effective_io_mib_s": 1308.8, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 2458.0, "current_rss_mib": 2457.9, "mem_avail_before_mib": 3656.6, "mem_avail_after_mib": 3632.7, "swap_delta_mib": -3.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 11.8, "page_cache_delta_since_start_mib": -677.4, "idle_watts": 7.71, "avg_watts": 10.37, "net_watts": 2.66, "joules_per_tok": 59.75, "net_joules_per_tok": 15.33, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page" }, { "round": "round8_v010", "source_file": "r8_32b_b5400_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_32b_b5400_draft05b", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "prompt_heavy", "chat_template_applied": false, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 340, "completion_tokens": 32, "load_time_s": 3.764, "ttft_s": 18.01, "ttft_s_min": 17.468, "ttft_s_max": 18.552, "decode_time_s": 183.023, "total_time_s": 201.033, "decode_tok_per_s": 0.171, "decode_tok_per_s_min": 0.156, "decode_tok_per_s_max": 0.185, "total_tok_per_s": 0.16, "total_tok_per_s_min": 0.148, "total_tok_per_s_max": 0.172, "sweeps": 13, "drafted_tokens": 67, "accepted_tokens": 19, "resident_layers": 2, "streamed_layers": 62, "n_layers": 64, "resident_mib": 1622.2, "slot_mib": 297.7, "expert_slot_mib": 0.0, "io_mib_read": 224952.76, "pread_mib_s": 1229.4, "effective_io_mib_s": 1125.0, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4108.7, "current_rss_mib": 4108.6, "mem_avail_before_mib": 1961.5, "mem_avail_after_mib": 1964.1, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -10.4, "page_cache_delta_since_start_mib": -21.0, "idle_watts": 7.94, "avg_watts": 10.21, "net_watts": 2.26, "joules_per_tok": 64.13, "net_joules_per_tok": 14.21, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " The core memory-management mechanism described involves layer-streaming inference, which uses a small fixed pool of reusable compute buffers and asynchronously" }, { "round": "round8_v010", "source_file": "r8_32b_b5400_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_32b_b5400_draft05b", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "code_edit", "chat_template_applied": false, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 106, "completion_tokens": 48, "load_time_s": 3.764, "ttft_s": 14.704, "ttft_s_min": 14.671, "ttft_s_max": 14.738, "decode_time_s": 55.967, "total_time_s": 70.672, "decode_tok_per_s": 0.84, "decode_tok_per_s_min": 0.838, "decode_tok_per_s_max": 0.841, "total_tok_per_s": 0.679, "total_tok_per_s_min": 0.678, "total_tok_per_s_max": 0.681, "sweeps": 5, "drafted_tokens": 44, "accepted_tokens": 44, "resident_layers": 2, "streamed_layers": 62, "n_layers": 64, "resident_mib": 1622.2, "slot_mib": 297.7, "expert_slot_mib": 0.0, "io_mib_read": 86818.02, "pread_mib_s": 1337.6, "effective_io_mib_s": 1224.3, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4113.2, "current_rss_mib": 4113.2, "mem_avail_before_mib": 1958.5, "mem_avail_after_mib": 1951.6, "swap_delta_mib": -0.5, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -1.9, "page_cache_delta_since_start_mib": -24.5, "idle_watts": 7.94, "avg_watts": 10.54, "net_watts": 2.59, "joules_per_tok": 15.51, "net_joules_per_tok": 3.82, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\n pub model_path: PathBuf,\n pub n_ctx: u32,\n pub n_batch: u32,\n pub resident_layers: Option<u32>,\n " }, { "round": "round8_v010", "source_file": "r8_32b_b5400_draft05b.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_32b_b5400_draft05b", "status": "ok", "model": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", "draft_model": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "workload": "open_ended", "chat_template_applied": false, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 5400, "prompt_tokens": 31, "completion_tokens": 48, "load_time_s": 3.764, "ttft_s": 16.558, "ttft_s_min": 14.005, "ttft_s_max": 19.111, "decode_time_s": 154.474, "total_time_s": 171.031, "decode_tok_per_s": 0.304, "decode_tok_per_s_min": 0.304, "decode_tok_per_s_max": 0.304, "total_tok_per_s": 0.281, "total_tok_per_s_min": 0.277, "total_tok_per_s_max": 0.285, "sweeps": 12, "drafted_tokens": 104, "accepted_tokens": 37, "resident_layers": 2, "streamed_layers": 62, "n_layers": 64, "resident_mib": 1622.2, "slot_mib": 297.7, "expert_slot_mib": 0.0, "io_mib_read": 207648.7, "pread_mib_s": 1311.4, "effective_io_mib_s": 1213.5, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 4115.2, "current_rss_mib": 4115.2, "mem_avail_before_mib": 1930.7, "mem_avail_after_mib": 1955.5, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -12.6, "page_cache_delta_since_start_mib": -37.9, "idle_watts": 7.94, "avg_watts": 10.27, "net_watts": 2.33, "joules_per_tok": 36.61, "net_joules_per_tok": 8.3, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": " When streaming transformer layers on an 8 GB edge device, the interaction between page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers can b" }, { "round": "round8_v010", "source_file": "r8_olmoe_r0_cached.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_olmoe_r0_sparse_cached", "status": "ok", "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", "draft_model": null, "workload": "prompt_heavy", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 367, "completion_tokens": 32, "load_time_s": 0.681, "ttft_s": 2.078, "ttft_s_min": 2.078, "ttft_s_max": 2.08, "decode_time_s": 2.268, "total_time_s": 4.346, "decode_tok_per_s": 13.667, "decode_tok_per_s_min": 13.652, "decode_tok_per_s_max": 13.922, "total_tok_per_s": 7.362, "total_tok_per_s_min": 7.356, "total_tok_per_s_max": 7.433, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 16, "n_layers": 16, "resident_mib": 218.3, "slot_mib": 0.0, "expert_slot_mib": 139.0, "io_mib_read": 2085.12, "pread_mib_s": 1404.5, "effective_io_mib_s": 479.7, "expert_cache_hits": 11928, "expert_cache_misses": 0, "expert_cache_hit_rate": 1.0, "expert_cache_mib_saved": 8635.38, "moe_layers": 16, "moe_active_streaming": true, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 2921.4, "current_rss_mib": 2921.4, "mem_avail_before_mib": 2467.0, "mem_avail_after_mib": 2465.2, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.4, "page_cache_delta_since_start_mib": -90.3, "idle_watts": 8.13, "avg_watts": 12.32, "net_watts": 4.2, "joules_per_tok": 1.67, "net_joules_per_tok": 0.57, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "\nThe core memory-management mechanism involves a combination of zero-copy staging from NVMe with page-aligned DMA, layer-streaming inference with reusable" }, { "round": "round8_v010", "source_file": "r8_olmoe_r0_cached.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_olmoe_r0_sparse_cached", "status": "ok", "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", "draft_model": null, "workload": "code_edit", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 133, "completion_tokens": 48, "load_time_s": 0.681, "ttft_s": 1.865, "ttft_s_min": 1.863, "ttft_s_max": 1.868, "decode_time_s": 3.601, "total_time_s": 5.465, "decode_tok_per_s": 13.054, "decode_tok_per_s_min": 12.451, "decode_tok_per_s_max": 13.24, "total_tok_per_s": 8.783, "total_tok_per_s_min": 8.513, "total_tok_per_s_max": 8.86, "sweeps": 48, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 16, "n_layers": 16, "resident_mib": 218.3, "slot_mib": 0.0, "expert_slot_mib": 139.0, "io_mib_read": 2180.74, "pread_mib_s": 1269.1, "effective_io_mib_s": 409.4, "expert_cache_hits": 17941, "expert_cache_misses": 131, "expert_cache_hit_rate": 0.993, "expert_cache_mib_saved": 12988.27, "moe_layers": 16, "moe_active_streaming": true, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 2953.3, "current_rss_mib": 2953.3, "mem_avail_before_mib": 2438.6, "mem_avail_after_mib": 2437.3, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.1, "page_cache_delta_since_start_mib": -103.2, "idle_watts": 8.13, "avg_watts": 12.61, "net_watts": 4.49, "joules_per_tok": 1.44, "net_joules_per_tok": 0.51, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "#[derive(Debug, Clone)]\npub struct StreamConfig {\npub model_path: PathBuf,\npub n_ctx: u32,\npub n_batch: u32,\npub resident_layers" }, { "round": "round8_v010", "source_file": "r8_olmoe_r0_cached.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_olmoe_r0_sparse_cached", "status": "ok", "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": false, "warmup": true, "reps": 3, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 33, "completion_tokens": 48, "load_time_s": 0.681, "ttft_s": 1.72, "ttft_s_min": 1.718, "ttft_s_max": 1.724, "decode_time_s": 3.901, "total_time_s": 5.622, "decode_tok_per_s": 12.047, "decode_tok_per_s_min": 11.45, "decode_tok_per_s_max": 12.505, "total_tok_per_s": 8.538, "total_tok_per_s_min": 8.235, "total_tok_per_s_max": 8.764, "sweeps": 48, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 16, "n_layers": 16, "resident_mib": 218.3, "slot_mib": 0.0, "expert_slot_mib": 139.0, "io_mib_read": 2353.59, "pread_mib_s": 1167.9, "effective_io_mib_s": 436.5, "expert_cache_hits": 17705, "expert_cache_misses": 367, "expert_cache_hit_rate": 0.98, "expert_cache_mib_saved": 12816.33, "moe_layers": 16, "moe_active_streaming": true, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 2970.7, "current_rss_mib": 2970.7, "mem_avail_before_mib": 2415.1, "mem_avail_after_mib": 2413.6, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 0.0, "page_cache_delta_since_start_mib": -114.5, "idle_watts": 8.13, "avg_watts": 11.98, "net_watts": 3.86, "joules_per_tok": 1.4, "net_joules_per_tok": 0.45, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "\n\n1. **Page-Aligned O_DIRECT NVMe Reads:**\n - **Explanation:** When performing an O_DIRECT read on an NVMe storage device, the read operation is aligned with " }, { "round": "round8_v010", "source_file": "r8_olmoe_r0_full.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_olmoe_r0_full", "status": "ok", "model": "OLMoE-1B-7B-0125-Instruct-Q2_K.gguf", "draft_model": null, "workload": "open_ended", "chat_template_applied": false, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 33, "completion_tokens": 32, "load_time_s": 0.507, "ttft_s": 1.448, "ttft_s_min": 1.355, "ttft_s_max": 1.54, "decode_time_s": 40.615, "total_time_s": 42.062, "decode_tok_per_s": 0.763, "decode_tok_per_s_min": 0.763, "decode_tok_per_s_max": 0.763, "total_tok_per_s": 0.761, "total_tok_per_s_min": 0.759, "total_tok_per_s_max": 0.762, "sweeps": 32, "drafted_tokens": 0, "accepted_tokens": 0, "resident_layers": 0, "streamed_layers": 16, "n_layers": 16, "resident_mib": 112.8, "slot_mib": 145.6, "expert_slot_mib": 0.0, "io_mib_read": 74545.94, "pread_mib_s": 1942.9, "effective_io_mib_s": 1774.0, "expert_cache_hits": 0, "expert_cache_misses": 0, "expert_cache_hit_rate": null, "expert_cache_mib_saved": 0.0, "moe_layers": 0, "moe_active_streaming": false, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 1312.2, "current_rss_mib": 1312.2, "mem_avail_before_mib": 4053.9, "mem_avail_after_mib": 4054.1, "swap_delta_mib": -2.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": 1.0, "page_cache_delta_since_start_mib": 1.0, "idle_watts": 7.86, "avg_watts": 10.91, "net_watts": 3.05, "joules_per_tok": 14.34, "net_joules_per_tok": 4.01, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "\n\n1. **Page-Aligned O_DIRECT NVMe Reads:**\n - **Explanation:** When performing an O_DIRECT read on" }, { "round": "round8_v010", "source_file": "r8_qwen3_30b_a3b_draft06b_code.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_qwen3_30b_a3b_r0_draft06b", "status": "ok", "model": "Qwen3-30B-A3B-Q4_K_M.gguf", "draft_model": "Qwen3-0.6B-Q4_K_M.gguf", "workload": "code_edit", "chat_template_applied": true, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 114, "completion_tokens": 32, "load_time_s": 33.063, "ttft_s": 13.596, "ttft_s_min": 13.512, "ttft_s_max": 13.681, "decode_time_s": 331.875, "total_time_s": 345.471, "decode_tok_per_s": 0.113, "decode_tok_per_s_min": 0.066, "decode_tok_per_s_max": 0.159, "total_tok_per_s": 0.11, "total_tok_per_s_min": 0.066, "total_tok_per_s_max": 0.154, "sweeps": 11, "drafted_tokens": 62, "accepted_tokens": 22, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 951.3, "slot_mib": 0.0, "expert_slot_mib": 373.5, "io_mib_read": 74126.45, "pread_mib_s": 267.3, "effective_io_mib_s": 257.1, "expert_cache_hits": 4218, "expert_cache_misses": 22119, "expert_cache_hit_rate": 0.16, "expert_cache_mib_saved": 3767.38, "moe_layers": 48, "moe_active_streaming": true, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3974.3, "current_rss_mib": 3973.1, "mem_avail_before_mib": 1549.3, "mem_avail_after_mib": 1539.5, "swap_delta_mib": -2.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -667.3, "page_cache_delta_since_start_mib": -1066.2, "idle_watts": 7.85, "avg_watts": 8.33, "net_watts": 0.48, "joules_per_tok": 89.98, "net_joules_per_tok": 5.21, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "<think>\nOkay, let me try to figure out how to do this. The user wants me to take the given Rust struct and add a new field called `" }, { "round": "round8_v010", "source_file": "r8_qwen3_30b_a3b_draft06b_open.json", "hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)", "hardware_id": "orin-nano-8gb-cuda", "engine": "layerstream-plugin", "engine_version": "0.1.0", "git_rev": null, "mode": "r8_qwen3_30b_a3b_r0_draft06b", "status": "ok", "model": "Qwen3-30B-A3B-Q4_K_M.gguf", "draft_model": "Qwen3-0.6B-Q4_K_M.gguf", "workload": "open_ended", "chat_template_applied": true, "warmup": false, "reps": 2, "temperature": 0.0, "seed": 42, "threads": 4, "kv_cache_type": "q8_0", "vram_budget_auto": false, "vram_budget_mb": 0, "prompt_tokens": 39, "completion_tokens": 32, "load_time_s": 3.263, "ttft_s": 11.238, "ttft_s_min": 11.08, "ttft_s_max": 11.395, "decode_time_s": 105.204, "total_time_s": 116.442, "decode_tok_per_s": 0.317, "decode_tok_per_s_min": 0.233, "decode_tok_per_s_max": 0.4, "total_tok_per_s": 0.291, "total_tok_per_s_min": 0.222, "total_tok_per_s_max": 0.361, "sweeps": 6, "drafted_tokens": 48, "accepted_tokens": 27, "resident_layers": 0, "streamed_layers": 48, "n_layers": 48, "resident_mib": 951.3, "slot_mib": 0.0, "expert_slot_mib": 373.5, "io_mib_read": 73026.22, "pread_mib_s": 708.0, "effective_io_mib_s": 667.7, "expert_cache_hits": 1826, "expert_cache_misses": 6019, "expert_cache_hit_rate": 0.233, "expert_cache_mib_saved": 1555.77, "moe_layers": 48, "moe_active_streaming": true, "uma_zero_copy": false, "is_unified_memory": true, "direct_io_active": true, "peak_rss_mib": 3969.2, "current_rss_mib": 3969.2, "mem_avail_before_mib": 1661.1, "mem_avail_after_mib": 1629.2, "swap_delta_mib": 0.0, "cuda_max_allocated_mib": null, "page_cache_delta_mib": -24.4, "page_cache_delta_since_start_mib": -236.8, "idle_watts": 8.18, "avg_watts": 9.06, "net_watts": 0.88, "joules_per_tok": 32.98, "net_joules_per_tok": 3.21, "turn1_prompt_tokens": null, "turn1_ttft_s": null, "turn2_prompt_tokens": null, "turn2_ttft_s": null, "ttft_speedup_x": null, "notes": null, "output_preview": "<think>\nOkay, so I need to explain how page-aligned O_DIRECT NVMe reads and asynchronous ping-pong GPU buffers interact when streaming transformer layers on an " } ]
[
{
"round": "airllm",
"source_file": "airllm_14b_4bit_code_edit_warm.json",
"hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)",
"hardware_id": "orin-nano-8gb-cuda",
"engine": "airllm",
"engine_version": "4.0.0",
"git_rev": null,
"mode": "airllm_4bit_prefetch",
"status": "ok",
"model": "Qwen/Qwen2.5-14B-Instruct",
"draft_model": null,
"workload": "code_edit",
"chat_template_applied": null,
"warmup": null,
"reps": null,
"temperature": null,
"seed": null,
"threads": null,
"kv_cache_type": null,
"vram_budget_auto": null,
"vram_budget_mb": null,
"prompt_tokens": 113,
"completion_tokens": 8,
"load_time_s": 2.55,
"ttft_s": 33.371,
"ttft_s_min": null,
"ttft_s_max": null,
"decode_time_s": 214.529,
"total_time_s": 247.9,
"decode_tok_per_s": 0.033,
"decode_tok_per_s_min": null,
"decode_tok_per_s_max": null,
"total_tok_per_s": 0.032,
"total_tok_per_s_min": null,
"total_tok_per_s_max": null,
"sweeps": null,
"drafted_tokens": null,
"accepted_tokens": null,
"resident_layers": null,
"streamed_layers": null,
"n_layers": null,
"resident_mib": null,
"slot_mib": null,
"expert_slot_mib": null,
"io_mib_read": null,
"pread_mib_s": null,
"effective_io_mib_s": null,
"expert_cache_hits": null,
"expert_cache_misses": null,
"expert_cache_hit_rate": null,
"expert_cache_mib_saved": null,
"moe_layers": null,
"moe_active_streaming": null,
"uma_zero_copy": null,
"is_unified_memory": null,
"direct_io_active": null,
"peak_rss_mib": 3536.4,
"current_rss_mib": 626.8,
"mem_avail_before_mib": null,
"mem_avail_after_mib": null,
"swap_delta_mib": null,
"cuda_max_allocated_mib": 1936.2,
"page_cache_delta_mib": -503.0,
"page_cache_delta_since_start_mib": -458.6,
"idle_watts": null,
"avg_watts": 9.3,
"net_watts": null,
"joules_per_tok": 288.21,
"net_joules_per_tok": null,
"turn1_prompt_tokens": null,
"turn1_ttft_s": null,
"turn2_prompt_tokens": null,
"turn2_ttft_s": null,
"ttft_speedup_x": null,
"notes": null,
"output_preview": "#[derive(Debug, Clone)]\npub struct"
},
{
"round": "airllm",
"source_file": "airllm_14b_4bit_open_ended_warm.json",
"hardware": "NVIDIA Jetson Orin Nano (8 GiB unified LPDDR5, CUDA sm_87, dm-crypt NVMe)",
"hardware_id": "orin-nano-8gb-cuda",
"engine": "airllm",
"engine_version": "4.0.0",
"git_rev": null,
"mode": "airllm_4bit_prefetch",
"status": "ok",
"model": "Qwen/Qwen2.5-14B-Instruct",
"draft_model": null,
"workload": "open_ended",
"chat_template_applied": null,
"warmup": null,
"reps": null,
"temperature": null,
"seed": null,
"threads": null,
"kv_cache_type": null,
"vram_budget_auto": null,