Spaces:
Running
Running
Publish interactive LayerStream benchmark explorer & hardware estimator Space
Browse files- README.md +11 -5
- benchmarks.json +0 -0
- index.html +928 -17
README.md
CHANGED
|
@@ -1,10 +1,16 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: LayerStream Edge LLM Benchmarks
|
| 3 |
+
emoji: ⚡
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: blue
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
| 8 |
+
license: apache-2.0
|
| 9 |
+
datasets:
|
| 10 |
+
- kleinbem/layerstream-orin-nano-benchmarks
|
| 11 |
+
short_description: 4-engine NVMe LLM benchmark explorer & hardware estimator
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# LayerStream Edge LLM Benchmarks
|
| 15 |
+
|
| 16 |
+
Interactive dashboard for **[`layerstream-plugin`](https://github.com/kleinbem/layerstream-plugin)** backed by the bare-metal NVIDIA Jetson Orin Nano dataset **[`kleinbem/layerstream-orin-nano-benchmarks`](https://huggingface.co/datasets/kleinbem/layerstream-orin-nano-benchmarks)** (**130 runs × 75 columns** across `layerstream-plugin`, stock `llama.cpp`, Hugging Face `accelerate`, and `AirLLM 4.0.0`).
|
benchmarks.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
index.html
CHANGED
|
@@ -1,19 +1,930 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
</html>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>layerstream-plugin — Out-of-Core Edge LLM Benchmark Explorer</title>
|
| 7 |
+
<style>
|
| 8 |
+
:root {
|
| 9 |
+
color-scheme: dark;
|
| 10 |
+
--bg: #0b1020;
|
| 11 |
+
--surface: #121a30;
|
| 12 |
+
--surface-2: #182341;
|
| 13 |
+
--border: #26355d;
|
| 14 |
+
--text: #e6edf7;
|
| 15 |
+
--muted: #98a8c7;
|
| 16 |
+
--accent: #38bdf8;
|
| 17 |
+
--accent-2: #34d399;
|
| 18 |
+
--warn: #fbbf24;
|
| 19 |
+
--danger: #f87171;
|
| 20 |
+
--purple: #a78bfa;
|
| 21 |
+
--radius: 12px;
|
| 22 |
+
--mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
|
| 23 |
+
--sans: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
* { box-sizing: border-box; }
|
| 27 |
+
|
| 28 |
+
body {
|
| 29 |
+
margin: 0;
|
| 30 |
+
font-family: var(--sans);
|
| 31 |
+
background: radial-gradient(1200px 600px at 20% -10%, #1a2750 0%, var(--bg) 60%);
|
| 32 |
+
color: var(--text);
|
| 33 |
+
line-height: 1.5;
|
| 34 |
+
accent-color: var(--accent);
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
@supports not (accent-color: #38bdf8) {
|
| 38 |
+
input[type="range"] {
|
| 39 |
+
-webkit-appearance: none;
|
| 40 |
+
appearance: none;
|
| 41 |
+
background: transparent;
|
| 42 |
+
}
|
| 43 |
+
input[type="range"]::-webkit-slider-runnable-track {
|
| 44 |
+
height: 6px;
|
| 45 |
+
border-radius: 3px;
|
| 46 |
+
background: var(--border);
|
| 47 |
+
}
|
| 48 |
+
input[type="range"]::-webkit-slider-thumb {
|
| 49 |
+
-webkit-appearance: none;
|
| 50 |
+
height: 16px;
|
| 51 |
+
width: 16px;
|
| 52 |
+
border-radius: 50%;
|
| 53 |
+
background: var(--accent);
|
| 54 |
+
margin-top: -5px;
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
a { color: var(--accent); text-decoration: none; }
|
| 59 |
+
a:hover { text-decoration: underline; }
|
| 60 |
+
:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
| 61 |
+
|
| 62 |
+
.container {
|
| 63 |
+
max-width: 1180px;
|
| 64 |
+
margin: 0 auto;
|
| 65 |
+
padding: 1.5rem clamp(1rem, 3vw, 2rem) 3rem;
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
header.hero {
|
| 69 |
+
display: flex;
|
| 70 |
+
flex-wrap: wrap;
|
| 71 |
+
justify-content: space-between;
|
| 72 |
+
align-items: flex-start;
|
| 73 |
+
gap: 1rem;
|
| 74 |
+
margin-bottom: 1.5rem;
|
| 75 |
+
padding-bottom: 1.25rem;
|
| 76 |
+
border-bottom: 1px solid var(--border);
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
.hero h1 {
|
| 80 |
+
margin: 0 0 0.35rem;
|
| 81 |
+
font-size: clamp(1.5rem, 3vw, 2.15rem);
|
| 82 |
+
letter-spacing: -0.02em;
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
.hero p {
|
| 86 |
+
margin: 0;
|
| 87 |
+
color: var(--muted);
|
| 88 |
+
max-width: 68ch;
|
| 89 |
+
font-size: 0.95rem;
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
.pill-links {
|
| 93 |
+
display: flex;
|
| 94 |
+
flex-wrap: wrap;
|
| 95 |
+
gap: 0.5rem;
|
| 96 |
+
align-items: center;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
.pill {
|
| 100 |
+
display: inline-flex;
|
| 101 |
+
align-items: center;
|
| 102 |
+
gap: 0.4rem;
|
| 103 |
+
padding: 0.45rem 0.85rem;
|
| 104 |
+
border-radius: 999px;
|
| 105 |
+
border: 1px solid var(--border);
|
| 106 |
+
background: var(--surface);
|
| 107 |
+
color: var(--text);
|
| 108 |
+
font-size: 0.85rem;
|
| 109 |
+
font-weight: 500;
|
| 110 |
+
}
|
| 111 |
+
.pill:hover {
|
| 112 |
+
border-color: var(--accent);
|
| 113 |
+
text-decoration: none;
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
.kpi-grid {
|
| 117 |
+
display: grid;
|
| 118 |
+
grid-template-columns: repeat(auto-fit, minmax(210px, 1fr));
|
| 119 |
+
gap: 0.85rem;
|
| 120 |
+
margin-bottom: 1.5rem;
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
.kpi {
|
| 124 |
+
background: var(--surface);
|
| 125 |
+
border: 1px solid var(--border);
|
| 126 |
+
border-radius: var(--radius);
|
| 127 |
+
padding: 0.95rem 1.1rem;
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
.kpi .label {
|
| 131 |
+
font-size: 0.78rem;
|
| 132 |
+
text-transform: uppercase;
|
| 133 |
+
letter-spacing: 0.05em;
|
| 134 |
+
color: var(--muted);
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
.kpi .val {
|
| 138 |
+
font-size: 1.55rem;
|
| 139 |
+
font-weight: 700;
|
| 140 |
+
font-variant-numeric: tabular-nums;
|
| 141 |
+
margin: 0.2rem 0;
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
.kpi .sub {
|
| 145 |
+
font-size: 0.8rem;
|
| 146 |
+
color: var(--muted);
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
.tabs {
|
| 150 |
+
display: flex;
|
| 151 |
+
flex-wrap: wrap;
|
| 152 |
+
gap: 0.45rem;
|
| 153 |
+
margin-bottom: 1.25rem;
|
| 154 |
+
border-bottom: 1px solid var(--border);
|
| 155 |
+
padding-bottom: 0.65rem;
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
.tab-btn {
|
| 159 |
+
background: transparent;
|
| 160 |
+
border: 1px solid transparent;
|
| 161 |
+
color: var(--muted);
|
| 162 |
+
padding: 0.55rem 0.95rem;
|
| 163 |
+
border-radius: 8px;
|
| 164 |
+
cursor: pointer;
|
| 165 |
+
font-size: 0.92rem;
|
| 166 |
+
font-weight: 600;
|
| 167 |
+
font-family: inherit;
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
.tab-btn:hover { color: var(--text); background: var(--surface); }
|
| 171 |
+
.tab-btn[aria-selected="true"] {
|
| 172 |
+
background: var(--surface-2);
|
| 173 |
+
border-color: var(--accent);
|
| 174 |
+
color: var(--text);
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
.panel { display: none; }
|
| 178 |
+
.panel.active { display: block; }
|
| 179 |
+
|
| 180 |
+
.card {
|
| 181 |
+
background: var(--surface);
|
| 182 |
+
border: 1px solid var(--border);
|
| 183 |
+
border-radius: var(--radius);
|
| 184 |
+
padding: 1.15rem 1.25rem;
|
| 185 |
+
margin-bottom: 1rem;
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
.card h2 {
|
| 189 |
+
margin: 0 0 0.35rem;
|
| 190 |
+
font-size: 1.15rem;
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
.card .desc {
|
| 194 |
+
margin: 0 0 1rem;
|
| 195 |
+
color: var(--muted);
|
| 196 |
+
font-size: 0.88rem;
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
.controls {
|
| 200 |
+
display: flex;
|
| 201 |
+
flex-wrap: wrap;
|
| 202 |
+
gap: 0.85rem;
|
| 203 |
+
align-items: flex-end;
|
| 204 |
+
margin-bottom: 1rem;
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
.ctrl {
|
| 208 |
+
display: flex;
|
| 209 |
+
flex-direction: column;
|
| 210 |
+
gap: 0.3rem;
|
| 211 |
+
min-width: 170px;
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
.ctrl label {
|
| 215 |
+
font-size: 0.78rem;
|
| 216 |
+
font-weight: 600;
|
| 217 |
+
color: var(--muted);
|
| 218 |
+
text-transform: uppercase;
|
| 219 |
+
letter-spacing: 0.04em;
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
select, input[type="number"] {
|
| 223 |
+
background: var(--surface-2);
|
| 224 |
+
color: var(--text);
|
| 225 |
+
border: 1px solid var(--border);
|
| 226 |
+
border-radius: 8px;
|
| 227 |
+
padding: 0.48rem 0.65rem;
|
| 228 |
+
font-family: inherit;
|
| 229 |
+
font-size: 0.9rem;
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
.bar-list {
|
| 233 |
+
display: flex;
|
| 234 |
+
flex-direction: column;
|
| 235 |
+
gap: 0.7rem;
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
.bar-row {
|
| 239 |
+
display: grid;
|
| 240 |
+
grid-template-columns: minmax(210px, 300px) 1fr minmax(120px, 165px);
|
| 241 |
+
gap: 0.75rem;
|
| 242 |
+
align-items: center;
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
@media (max-width: 720px) {
|
| 246 |
+
.bar-row { grid-template-columns: 1fr; gap: 0.25rem; }
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
.bar-label {
|
| 250 |
+
font-size: 0.86rem;
|
| 251 |
+
font-weight: 600;
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
.bar-sub {
|
| 255 |
+
display: block;
|
| 256 |
+
font-size: 0.75rem;
|
| 257 |
+
font-weight: 400;
|
| 258 |
+
color: var(--muted);
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
.bar-track {
|
| 262 |
+
height: 26px;
|
| 263 |
+
background: #0d1428;
|
| 264 |
+
border: 1px solid var(--border);
|
| 265 |
+
border-radius: 6px;
|
| 266 |
+
overflow: hidden;
|
| 267 |
+
position: relative;
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
.bar-fill {
|
| 271 |
+
height: 100%;
|
| 272 |
+
border-radius: 5px;
|
| 273 |
+
transition: width 220ms ease;
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
.bar-value {
|
| 277 |
+
font-family: var(--mono);
|
| 278 |
+
font-size: 0.84rem;
|
| 279 |
+
text-align: right;
|
| 280 |
+
font-variant-numeric: tabular-nums;
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
.badge {
|
| 284 |
+
display: inline-block;
|
| 285 |
+
padding: 0.15rem 0.5rem;
|
| 286 |
+
border-radius: 6px;
|
| 287 |
+
font-size: 0.74rem;
|
| 288 |
+
font-weight: 700;
|
| 289 |
+
font-family: var(--mono);
|
| 290 |
+
letter-spacing: 0.02em;
|
| 291 |
+
}
|
| 292 |
+
|
| 293 |
+
.b-ok { background: rgba(52, 211, 153, 0.16); color: var(--accent-2); border: 1px solid rgba(52, 211, 153, 0.35); }
|
| 294 |
+
.b-warn { background: rgba(251, 191, 36, 0.16); color: var(--warn); border: 1px solid rgba(251, 191, 36, 0.35); }
|
| 295 |
+
.b-err { background: rgba(248, 113, 113, 0.16); color: var(--danger); border: 1px solid rgba(248, 113, 113, 0.35); }
|
| 296 |
+
.b-info { background: rgba(56, 189, 248, 0.16); color: var(--accent); border: 1px solid rgba(56, 189, 248, 0.35); }
|
| 297 |
+
|
| 298 |
+
.callout {
|
| 299 |
+
border-left: 3px solid var(--warn);
|
| 300 |
+
background: rgba(251, 191, 36, 0.07);
|
| 301 |
+
padding: 0.75rem 1rem;
|
| 302 |
+
border-radius: 0 8px 8px 0;
|
| 303 |
+
font-size: 0.86rem;
|
| 304 |
+
color: var(--text);
|
| 305 |
+
margin-top: 1rem;
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
.callout.good {
|
| 309 |
+
border-left-color: var(--accent-2);
|
| 310 |
+
background: rgba(52, 211, 153, 0.07);
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
.preset-row {
|
| 314 |
+
display: flex;
|
| 315 |
+
flex-wrap: wrap;
|
| 316 |
+
gap: 0.5rem;
|
| 317 |
+
margin-bottom: 1rem;
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
.preset-btn {
|
| 321 |
+
background: var(--surface-2);
|
| 322 |
+
border: 1px solid var(--border);
|
| 323 |
+
color: var(--text);
|
| 324 |
+
padding: 0.4rem 0.75rem;
|
| 325 |
+
border-radius: 8px;
|
| 326 |
+
font-size: 0.82rem;
|
| 327 |
+
cursor: pointer;
|
| 328 |
+
font-family: inherit;
|
| 329 |
+
}
|
| 330 |
+
.preset-btn:hover { border-color: var(--accent); }
|
| 331 |
+
|
| 332 |
+
.est-grid {
|
| 333 |
+
display: grid;
|
| 334 |
+
grid-template-columns: repeat(auto-fit, minmax(230px, 1fr));
|
| 335 |
+
gap: 1rem;
|
| 336 |
+
margin-bottom: 1.1rem;
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
.slider-wrap {
|
| 340 |
+
display: flex;
|
| 341 |
+
flex-direction: column;
|
| 342 |
+
gap: 0.3rem;
|
| 343 |
+
}
|
| 344 |
+
|
| 345 |
+
.slider-head {
|
| 346 |
+
display: flex;
|
| 347 |
+
justify-content: space-between;
|
| 348 |
+
font-size: 0.84rem;
|
| 349 |
+
}
|
| 350 |
+
|
| 351 |
+
.slider-head strong {
|
| 352 |
+
font-family: var(--mono);
|
| 353 |
+
color: var(--accent);
|
| 354 |
+
font-variant-numeric: tabular-nums;
|
| 355 |
+
}
|
| 356 |
+
|
| 357 |
+
.table-wrapper {
|
| 358 |
+
overflow: auto;
|
| 359 |
+
max-inline-size: 100%;
|
| 360 |
+
max-block-size: min(560px, 75vh);
|
| 361 |
+
border: 1px solid var(--border);
|
| 362 |
+
border-radius: 8px;
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
+
table {
|
| 366 |
+
width: 100%;
|
| 367 |
+
border-collapse: separate;
|
| 368 |
+
border-spacing: 0;
|
| 369 |
+
font-size: 0.82rem;
|
| 370 |
+
font-variant-numeric: tabular-nums;
|
| 371 |
+
}
|
| 372 |
+
|
| 373 |
+
thead {
|
| 374 |
+
position: sticky;
|
| 375 |
+
inset-block-start: 0;
|
| 376 |
+
z-index: 2;
|
| 377 |
+
background: var(--surface-2);
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
th, td {
|
| 381 |
+
padding: 0.5rem 0.65rem;
|
| 382 |
+
border-bottom: 1px solid var(--border);
|
| 383 |
+
text-align: left;
|
| 384 |
+
white-space: nowrap;
|
| 385 |
+
}
|
| 386 |
+
|
| 387 |
+
th { font-weight: 600; color: var(--muted); font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.03em; }
|
| 388 |
+
td.num, th.num { text-align: right; font-family: var(--mono); }
|
| 389 |
+
tbody tr:hover { background: rgba(56, 189, 248, 0.06); }
|
| 390 |
+
|
| 391 |
+
.two-col {
|
| 392 |
+
display: grid;
|
| 393 |
+
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
|
| 394 |
+
gap: 1rem;
|
| 395 |
+
}
|
| 396 |
+
|
| 397 |
+
footer {
|
| 398 |
+
margin-top: 2rem;
|
| 399 |
+
padding-top: 1rem;
|
| 400 |
+
border-top: 1px solid var(--border);
|
| 401 |
+
color: var(--muted);
|
| 402 |
+
font-size: 0.82rem;
|
| 403 |
+
display: flex;
|
| 404 |
+
flex-wrap: wrap;
|
| 405 |
+
justify-content: space-between;
|
| 406 |
+
gap: 0.5rem;
|
| 407 |
+
}
|
| 408 |
+
</style>
|
| 409 |
+
</head>
|
| 410 |
+
<body>
|
| 411 |
+
<main class="container">
|
| 412 |
+
<header class="hero">
|
| 413 |
+
<div>
|
| 414 |
+
<h1>⚡ layerstream-plugin — Edge Out-of-Core LLM Benchmarks</h1>
|
| 415 |
+
<p>
|
| 416 |
+
Bare-metal 4-engine comparison on an <strong>8 GB NVIDIA Jetson Orin Nano</strong> (7.3 GiB RAM, CUDA <code>sm_87</code>, LUKS2 NVMe, INA3221 hardware power rail):
|
| 417 |
+
<strong>layerstream-plugin v0.1.0</strong> vs. <strong>stock llama.cpp</strong> (Ollama/LM Studio runtime), <strong>Hugging Face accelerate</strong>, and <strong>AirLLM 4.0.0</strong>.
|
| 418 |
+
</p>
|
| 419 |
+
</div>
|
| 420 |
+
<nav class="pill-links" aria-label="Project links">
|
| 421 |
+
<a class="pill" href="https://github.com/kleinbem/layerstream-plugin" target="_blank" rel="noopener">GitHub Repo</a>
|
| 422 |
+
<a class="pill" href="https://github.com/kleinbem/layerstream-plugin/releases/tag/v0.1.0" target="_blank" rel="noopener">v0.1.0 Release</a>
|
| 423 |
+
<a class="pill" href="https://huggingface.co/datasets/kleinbem/layerstream-orin-nano-benchmarks" target="_blank" rel="noopener">HF Dataset (130 runs)</a>
|
| 424 |
+
</nav>
|
| 425 |
+
</header>
|
| 426 |
+
|
| 427 |
+
<section class="kpi-grid" aria-label="Headline benchmark metrics">
|
| 428 |
+
<div class="kpi">
|
| 429 |
+
<div class="label">Qwen2.5-14B (8.37 GiB)</div>
|
| 430 |
+
<div class="val" style="color:var(--accent)">3.41 – 5.97 tok/s</div>
|
| 431 |
+
<div class="sub">0 MiB swap · 2,961 MiB/s O_DIRECT · 1.91 J/tok net</div>
|
| 432 |
+
</div>
|
| 433 |
+
<div class="kpi">
|
| 434 |
+
<div class="label">OLMoE-1B-7B (Sparse + TinyLFU)</div>
|
| 435 |
+
<div class="val" style="color:var(--accent-2)">12.0 – 13.7 tok/s</div>
|
| 436 |
+
<div class="sub">15.8×–17.9× vs full-layer R=0 · 98–100% cache hit</div>
|
| 437 |
+
</div>
|
| 438 |
+
<div class="kpi">
|
| 439 |
+
<div class="label">Qwen2.5-32B (2.53× RAM)</div>
|
| 440 |
+
<div class="val" style="color:var(--purple)">0.84 – 0.94 tok/s</div>
|
| 441 |
+
<div class="sub">18.5 GiB on 7.3 GiB RAM · Stock llama.cpp +17.2 GiB swap</div>
|
| 442 |
+
</div>
|
| 443 |
+
<div class="kpi">
|
| 444 |
+
<div class="label">Multi-Turn KV Prefix Reuse</div>
|
| 445 |
+
<div class="val" style="color:var(--warn)">4.97× faster TTFT</div>
|
| 446 |
+
<div class="sub">14.88 s → 2.99 s on 1,377-token conversation</div>
|
| 447 |
+
</div>
|
| 448 |
+
</section>
|
| 449 |
+
|
| 450 |
+
<div class="tabs" role="tablist" aria-label="Dashboard sections">
|
| 451 |
+
<button class="tab-btn" role="tab" id="tab-compare" aria-controls="panel-compare" aria-selected="true">1. 4-Engine Comparison</button>
|
| 452 |
+
<button class="tab-btn" role="tab" id="tab-tuning" aria-controls="panel-tuning" aria-selected="false">2. Tuning Story (R2 → R8)</button>
|
| 453 |
+
<button class="tab-btn" role="tab" id="tab-estimator" aria-controls="panel-estimator" aria-selected="false">3. Hardware Estimator</button>
|
| 454 |
+
<button class="tab-btn" role="tab" id="tab-raw" aria-controls="panel-raw" aria-selected="false">4. Raw Dataset (130 Runs)</button>
|
| 455 |
+
</div>
|
| 456 |
+
|
| 457 |
+
<!-- PANEL 1: 4-ENGINE COMPARISON -->
|
| 458 |
+
<section id="panel-compare" class="panel active" role="tabpanel" aria-labelledby="tab-compare">
|
| 459 |
+
<div class="card">
|
| 460 |
+
<h2>Head-to-Head Engine Comparison (Orin Nano Bare Metal)</h2>
|
| 461 |
+
<p class="desc">Compare measured throughput, latency, energy, and memory pressure across all 4 engines. Failed runs (OOM / 300 s timeout) are shown honestly alongside successful runs.</p>
|
| 462 |
+
<div class="controls">
|
| 463 |
+
<div class="ctrl">
|
| 464 |
+
<label for="cmp-model">Model</label>
|
| 465 |
+
<select id="cmp-model">
|
| 466 |
+
<option value="14b">Qwen2.5-14B-Instruct (8.37 GiB Dense)</option>
|
| 467 |
+
<option value="32b">Qwen2.5-32B-Instruct (18.49 GiB Dense — 2.53× RAM)</option>
|
| 468 |
+
<option value="olmoe">OLMoE-1B-7B (2.5 GiB, 64-Expert MoE)</option>
|
| 469 |
+
<option value="qwen3_30b">Qwen3-30B-A3B (17.3 GiB, 128-Expert MoE)</option>
|
| 470 |
+
</select>
|
| 471 |
+
</div>
|
| 472 |
+
<div class="ctrl">
|
| 473 |
+
<label for="cmp-workload">Workload</label>
|
| 474 |
+
<select id="cmp-workload">
|
| 475 |
+
<option value="code_edit">code_edit (48 tok, high structural overlap)</option>
|
| 476 |
+
<option value="open_ended" selected>open_ended (48 tok, unconstrained prose)</option>
|
| 477 |
+
<option value="prompt_heavy">prompt_heavy (340 prompt → 32 tok)</option>
|
| 478 |
+
</select>
|
| 479 |
+
</div>
|
| 480 |
+
<div class="ctrl">
|
| 481 |
+
<label for="cmp-metric">Metric</label>
|
| 482 |
+
<select id="cmp-metric">
|
| 483 |
+
<option value="decode_tok_per_s">Decode Speed (tok/s — higher is better)</option>
|
| 484 |
+
<option value="ttft_s">TTFT (seconds — lower is better)</option>
|
| 485 |
+
<option value="joules_per_tok">Gross Energy (J/tok — lower is better)</option>
|
| 486 |
+
<option value="peak_rss_mib">Peak RSS (MiB — lower is better)</option>
|
| 487 |
+
</select>
|
| 488 |
+
</div>
|
| 489 |
+
</div>
|
| 490 |
+
<div id="cmp-chart" class="bar-list" aria-live="polite"></div>
|
| 491 |
+
<div id="cmp-note" class="callout"></div>
|
| 492 |
+
</div>
|
| 493 |
+
</section>
|
| 494 |
+
|
| 495 |
+
<!-- PANEL 2: TUNING STORY -->
|
| 496 |
+
<section id="panel-tuning" class="panel" role="tabpanel" aria-labelledby="tab-tuning">
|
| 497 |
+
<div class="two-col">
|
| 498 |
+
<div class="card">
|
| 499 |
+
<h2>Qwen2.5-14B Iterative Optimization (R2 → R8)</h2>
|
| 500 |
+
<p class="desc">Every engineering change profiled with microsecond I/O + DMA timers on bare-metal Jetson Orin Nano (`code_edit` decode tok/s).</p>
|
| 501 |
+
<div id="tuning-14b-chart" class="bar-list"></div>
|
| 502 |
+
<div class="callout good">
|
| 503 |
+
<strong>Takeaway:</strong> Zero-copy <code>O_DIRECT → cudaHostAlloc</code> + per-tensor <code>cudaMemcpyAsync</code> + <code>io_threads=1</code> + adaptive speculation lifted 14B from <strong>0.19 tok/s</strong> (pure stream) to <strong>3.41–5.97 tok/s</strong> with <strong>2,961 MiB/s</strong> sustained through LUKS2 <code>dm-crypt</code>.
|
| 504 |
+
</div>
|
| 505 |
+
</div>
|
| 506 |
+
<div class="card">
|
| 507 |
+
<h2>Sparse MoE + Negative Ablations</h2>
|
| 508 |
+
<p class="desc">Honest negative ablations matter just as much as wins — here is what actually hurt vs. helped on real hardware.</p>
|
| 509 |
+
<div id="tuning-ablation-chart" class="bar-list"></div>
|
| 510 |
+
<div class="callout">
|
| 511 |
+
<strong>Key findings:</strong> (1) Multi-threaded <code>O_DIRECT</code> (<code>io_threads=4</code>) <em>regressed -57%</em> on Linux LUKS2 <code>dm-crypt</code>. (2) Spending 580 MiB on a <code>1.5B</code> draft instead of extra resident layers lost 2.5×–3.0× to <code>0.5B</code>. (3) TinyLFU hot-expert caching jumped <code>OLMoE</code> <strong>15.8×–17.9×</strong> (`0.76 → 13.67 tok/s`).
|
| 512 |
+
</div>
|
| 513 |
+
</div>
|
| 514 |
+
</div>
|
| 515 |
+
</section>
|
| 516 |
+
|
| 517 |
+
<!-- PANEL 3: HARDWARE ESTIMATOR -->
|
| 518 |
+
<section id="panel-estimator" class="panel" role="tabpanel" aria-labelledby="tab-estimator">
|
| 519 |
+
<div class="card">
|
| 520 |
+
<h2>Interactive Hardware Estimator (Mirrors <code>layerstream-plugin check</code>)</h2>
|
| 521 |
+
<p class="desc">Estimate resident layer pinning, cold load time, and expected decode speed on your own machine (Jetson, laptop, or AMD 8700G 64 GB APU).</p>
|
| 522 |
+
<div class="preset-row">
|
| 523 |
+
<button class="preset-btn" type="button" data-preset="orin14">8 GB Jetson Orin Nano · 14B Dense</button>
|
| 524 |
+
<button class="preset-btn" type="button" data-preset="orin32">8 GB Jetson Orin Nano · 32B Dense</button>
|
| 525 |
+
<button class="preset-btn" type="button" data-preset="apu123">64 GB AMD 8700G · 123B Dense</button>
|
| 526 |
+
<button class="preset-btn" type="button" data-preset="apu235">64 GB AMD 8700G · 235B MoE</button>
|
| 527 |
+
<button class="preset-btn" type="button" data-preset="laptop70">16 GB Laptop · 70B Dense</button>
|
| 528 |
+
</div>
|
| 529 |
+
|
| 530 |
+
<div class="est-grid">
|
| 531 |
+
<div class="slider-wrap">
|
| 532 |
+
<div class="slider-head"><label for="est-ram">Free GPU / Unified RAM</label><strong id="est-ram-val">5.2 GiB</strong></div>
|
| 533 |
+
<input id="est-ram" type="range" min="3" max="56" step="0.5" value="5.2">
|
| 534 |
+
</div>
|
| 535 |
+
<div class="slider-wrap">
|
| 536 |
+
<div class="slider-head"><label for="est-nvme">Sustained NVMe O_DIRECT BW</label><strong id="est-nvme-val">2,800 MiB/s</strong></div>
|
| 537 |
+
<input id="est-nvme" type="range" min="100" max="7000" step="100" value="2800">
|
| 538 |
+
</div>
|
| 539 |
+
<div class="slider-wrap">
|
| 540 |
+
<div class="slider-head"><label for="est-model">Model GGUF Size</label><strong id="est-model-val">8.4 GiB</strong></div>
|
| 541 |
+
<input id="est-model" type="range" min="2" max="200" step="0.5" value="8.4">
|
| 542 |
+
</div>
|
| 543 |
+
<div class="ctrl">
|
| 544 |
+
<label for="est-arch">Architecture & Speculation</label>
|
| 545 |
+
<select id="est-arch">
|
| 546 |
+
<option value="dense_draft" selected>Dense + Small Draft Model (2.5×–5× spec)</option>
|
| 547 |
+
<option value="dense_none">Dense, No Speculation (1 tok/sweep floor)</option>
|
| 548 |
+
<option value="moe_cached">Sparse MoE + TinyLFU Hot-Expert Cache</option>
|
| 549 |
+
</select>
|
| 550 |
+
</div>
|
| 551 |
+
</div>
|
| 552 |
+
|
| 553 |
+
<div id="est-output" class="kpi-grid" aria-live="polite"></div>
|
| 554 |
+
<div id="est-verdict" class="callout good"></div>
|
| 555 |
+
</div>
|
| 556 |
+
</section>
|
| 557 |
+
|
| 558 |
+
<!-- PANEL 4: RAW DATASET -->
|
| 559 |
+
<section id="panel-raw" class="panel" role="tabpanel" aria-labelledby="tab-raw">
|
| 560 |
+
<div class="card">
|
| 561 |
+
<h2>Normalized Benchmark Records (<span id="raw-count">0</span> / 130 rows)</h2>
|
| 562 |
+
<p class="desc">Loaded directly from <code>kleinbem/layerstream-orin-nano-benchmarks</code> (Parquet / JSON). Use keyboard or horizontal scroll to inspect all telemetry columns.</p>
|
| 563 |
+
<div class="controls">
|
| 564 |
+
<div class="ctrl">
|
| 565 |
+
<label for="raw-round">Round</label>
|
| 566 |
+
<select id="raw-round"><option value="all">All Rounds</option></select>
|
| 567 |
+
</div>
|
| 568 |
+
<div class="ctrl">
|
| 569 |
+
<label for="raw-engine">Engine</label>
|
| 570 |
+
<select id="raw-engine"><option value="all">All Engines</option></select>
|
| 571 |
+
</div>
|
| 572 |
+
<div class="ctrl">
|
| 573 |
+
<label for="raw-workload">Workload</label>
|
| 574 |
+
<select id="raw-workload"><option value="all">All Workloads</option></select>
|
| 575 |
+
</div>
|
| 576 |
+
</div>
|
| 577 |
+
<div tabindex="0" role="region" aria-labelledby="raw-table-cap" class="table-wrapper">
|
| 578 |
+
<table>
|
| 579 |
+
<caption id="raw-table-cap" style="position:absolute;width:1px;height:1px;overflow:hidden;clip-path:inset(50%)">Normalized Orin Nano benchmark runs</caption>
|
| 580 |
+
<thead>
|
| 581 |
+
<tr>
|
| 582 |
+
<th scope="col">Round</th>
|
| 583 |
+
<th scope="col">Engine</th>
|
| 584 |
+
<th scope="col">Mode</th>
|
| 585 |
+
<th scope="col">Model</th>
|
| 586 |
+
<th scope="col">Workload</th>
|
| 587 |
+
<th scope="col">Status</th>
|
| 588 |
+
<th scope="col" class="num">R</th>
|
| 589 |
+
<th scope="col" class="num">Load (s)</th>
|
| 590 |
+
<th scope="col" class="num">TTFT (s)</th>
|
| 591 |
+
<th scope="col" class="num">Decode tok/s</th>
|
| 592 |
+
<th scope="col" class="num">pread MiB/s</th>
|
| 593 |
+
<th scope="col" class="num">Peak RSS</th>
|
| 594 |
+
<th scope="col" class="num">Swap Δ</th>
|
| 595 |
+
<th scope="col" class="num">J/tok (Net)</th>
|
| 596 |
+
</tr>
|
| 597 |
+
</thead>
|
| 598 |
+
<tbody id="raw-tbody"></tbody>
|
| 599 |
+
</table>
|
| 600 |
+
</div>
|
| 601 |
+
</div>
|
| 602 |
+
</section>
|
| 603 |
+
|
| 604 |
+
<footer>
|
| 605 |
+
<span>Data source: <a href="https://huggingface.co/datasets/kleinbem/layerstream-orin-nano-benchmarks" target="_blank" rel="noopener">kleinbem/layerstream-orin-nano-benchmarks</a> (Apache-2.0)</span>
|
| 606 |
+
<span id="data-status">Loading dataset…</span>
|
| 607 |
+
</footer>
|
| 608 |
+
</main>
|
| 609 |
+
|
| 610 |
+
<script>
|
| 611 |
+
const HF_JSON_URL = "https://huggingface.co/datasets/kleinbem/layerstream-orin-nano-benchmarks/resolve/main/data/benchmarks.json";
|
| 612 |
+
let ALL_ROWS = [];
|
| 613 |
+
|
| 614 |
+
const fmt = (v, d = 2) => (v === null || v === undefined || Number.isNaN(v)) ? "—" : Number(v).toFixed(d);
|
| 615 |
+
|
| 616 |
+
// Tab switching + arrow-key navigation
|
| 617 |
+
const tabs = Array.from(document.querySelectorAll('[role="tab"]'));
|
| 618 |
+
const panels = Array.from(document.querySelectorAll('[role="tabpanel"]'));
|
| 619 |
+
function activateTab(btn) {
|
| 620 |
+
tabs.forEach(t => t.setAttribute("aria-selected", String(t === btn)));
|
| 621 |
+
panels.forEach(p => p.classList.toggle("active", p.id === btn.getAttribute("aria-controls")));
|
| 622 |
+
}
|
| 623 |
+
tabs.forEach((btn, idx) => {
|
| 624 |
+
btn.addEventListener("click", () => activateTab(btn));
|
| 625 |
+
btn.addEventListener("keydown", (e) => {
|
| 626 |
+
if (e.key === "ArrowRight") { const n = tabs[(idx + 1) % tabs.length]; n.focus(); activateTab(n); }
|
| 627 |
+
if (e.key === "ArrowLeft") { const p = tabs[(idx - 1 + tabs.length) % tabs.length]; p.focus(); activateTab(p); }
|
| 628 |
+
});
|
| 629 |
+
});
|
| 630 |
+
|
| 631 |
+
function engineColor(engine, status) {
|
| 632 |
+
if (status && status !== "ok") return "var(--danger)";
|
| 633 |
+
if (engine === "layerstream-plugin") return "linear-gradient(90deg, #38bdf8, #34d399)";
|
| 634 |
+
if (engine === "llama.cpp") return "var(--purple)";
|
| 635 |
+
if (engine === "airllm") return "var(--warn)";
|
| 636 |
+
return "#64748b";
|
| 637 |
+
}
|
| 638 |
+
|
| 639 |
+
function pickComparisonRows(modelKey, workload) {
|
| 640 |
+
const findOne = (pred) => ALL_ROWS.find(pred);
|
| 641 |
+
if (modelKey === "14b") {
|
| 642 |
+
return [
|
| 643 |
+
{ label: "layerstream-plugin (R6 max R=20 + 0.5B)", sub: "6.15 GiB RSS · clean RAM", row: findOne(r => r.round === "round6" && r.mode === "r6_hybrid_r20_draft05b" && r.workload === workload) },
|
| 644 |
+
{ label: "layerstream-plugin (v0.1.0 b5400 + 0.5B)", sub: "R=7 · reps=3 · 0 MiB swap · 2,961 MiB/s", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_14b_b5400_draft05b" && r.workload === workload) },
|
| 645 |
+
{ label: "layerstream-plugin (v0.1.0 auto + 0.5B)", sub: "R=4 · reps=3 · 3.7 GiB RSS · 0 MiB swap", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_14b_auto_draft05b" && r.workload === workload) },
|
| 646 |
+
{ label: "layerstream-plugin (v0.1.0 R=0 pure stream)", sub: "No draft · 2.46 GiB RSS · 0 MiB swap", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_14b_r0") || findOne(r => r.round === "round5" && r.mode === "r5_zc_stream_r0" && r.workload === workload) },
|
| 647 |
+
{ label: "AirLLM 4.0.0 (4-bit NF4)", sub: workload === "prompt_heavy" ? "Cold shards (365s load, 32 tok)" : "Warm shards (8 tok only · 36.8 GiB disk)", row: findOne(r => r.round === "airllm" && r.mode === "airllm_4bit_prefetch" && r.workload === workload) },
|
| 648 |
+
{ label: "HF accelerate (4-bit NF4 disk offload)", sub: "144 s load → 300 s timeout (0 tok)", row: findOne(r => r.round === "competitors" && r.engine === "hf-accelerate") },
|
| 649 |
+
{ label: "Stock llama.cpp (-ngl 12, -b 512)", sub: "OOM rc=1 · +3.7 GiB swap (Ollama/LM Studio runtime)", row: findOne(r => r.round === "competitors" && r.mode === "llama_cpp_14b_ngl12_b512" && r.workload === workload) },
|
| 650 |
+
{ label: "Stock llama.cpp CPU mmap (-ngl 0 -dev none)", sub: "300 s timeout · 6.1 GiB RSS · +363 MiB swap", row: findOne(r => r.round === "competitors" && r.mode === "llama_cpp_14b_cpu_mmap") },
|
| 651 |
+
].filter(x => x.row);
|
| 652 |
+
}
|
| 653 |
+
if (modelKey === "32b") {
|
| 654 |
+
return [
|
| 655 |
+
{ label: "layerstream-plugin (R6 b5600 + 0.5B)", sub: "R=6/64 · 5.29 GiB RSS · clean RAM", row: findOne(r => r.round === "round6_32b" && r.mode === "r6_32b_auto5600_draft05b" && r.workload === workload) },
|
| 656 |
+
{ label: "layerstream-plugin (v0.1.0 b5400 + 0.5B)", sub: "R=2/64 · reps=2 · 4.1 GiB RSS · 0 MiB swap", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_32b_b5400_draft05b" && r.workload === workload) },
|
| 657 |
+
{ label: "layerstream-plugin (R6 R=0 pure stream)", sub: "0 draft · 2.96 GiB RSS · 559.7 GiB read", row: findOne(r => r.round === "round6_32b" && r.mode === "r6_32b_stream_r0") },
|
| 658 |
+
{ label: "Stock llama.cpp (-ngl 10)", sub: "300 s timeout · +17.24 GiB swap thrashing!", row: findOne(r => r.round === "competitors" && r.mode === "llama_cpp_32b_ngl10") },
|
| 659 |
+
].filter(x => x.row);
|
| 660 |
+
}
|
| 661 |
+
if (modelKey === "olmoe") {
|
| 662 |
+
return [
|
| 663 |
+
{ label: "layerstream-plugin R=0 Sparse + TinyLFU", sub: "98–100% cache hit · reps=3 · 0 MiB swap", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_olmoe_r0_sparse_cached" && r.workload === workload) },
|
| 664 |
+
{ label: "layerstream-plugin R=0 Full-Layer Stream", sub: "Baseline without sparse MoE / TinyLFU", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_olmoe_r0_full") },
|
| 665 |
+
].filter(x => x.row);
|
| 666 |
+
}
|
| 667 |
+
return [
|
| 668 |
+
{ label: "Stock llama.cpp CPU mmap (-ngl 0 -dev none -b 512)", sub: "Wins on Orin! Linux page cache avoids nvme1 IRQ stalls (+248 MiB swap)", row: findOne(r => r.round === "competitors" && r.mode === "llama_cpp_qwen3_30b_cpu_mmap_b512") },
|
| 669 |
+
{ label: "layerstream-plugin R=0 Sparse + TinyLFU + 0.6B (open_ended)", sub: "3.97 GiB RSS · 0 MiB swap · 11.2 s TTFT", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_qwen3_30b_a3b_r0_draft06b" && r.workload === "open_ended") },
|
| 670 |
+
{ label: "layerstream-plugin R=0 Sparse + TinyLFU + 0.6B (code_edit)", sub: "Hit 30 s Orin Tegra nvme1 IRQ stall · 0 MiB swap", row: findOne(r => r.round === "round8_v010" && r.mode === "r8_qwen3_30b_a3b_r0_draft06b" && r.workload === "code_edit") },
|
| 671 |
+
{ label: "Stock llama.cpp GPU MoE offload (-ot exps=CPU)", sub: "OOM rc=1 · +12.34 GiB swap", row: findOne(r => r.round === "competitors" && r.mode === "llama_cpp_qwen3_30b_gpu_moe_offload") },
|
| 672 |
+
].filter(x => x.row);
|
| 673 |
+
}
|
| 674 |
+
|
| 675 |
+
function renderComparison() {
|
| 676 |
+
const modelKey = document.getElementById("cmp-model").value;
|
| 677 |
+
const workload = document.getElementById("cmp-workload").value;
|
| 678 |
+
const metric = document.getElementById("cmp-metric").value;
|
| 679 |
+
const items = pickComparisonRows(modelKey, workload);
|
| 680 |
+
const vals = items.map(x => {
|
| 681 |
+
const r = x.row;
|
| 682 |
+
if (r.status && r.status !== "ok" && metric !== "peak_rss_mib") return null;
|
| 683 |
+
return r[metric];
|
| 684 |
+
});
|
| 685 |
+
const maxVal = Math.max(...vals.filter(v => typeof v === "number" && v > 0), 1e-6);
|
| 686 |
+
const unit = metric === "decode_tok_per_s" ? "tok/s" : metric === "ttft_s" ? "s" : metric === "joules_per_tok" ? "J/tok" : "MiB";
|
| 687 |
+
|
| 688 |
+
const container = document.getElementById("cmp-chart");
|
| 689 |
+
container.innerHTML = items.map((item, i) => {
|
| 690 |
+
const r = item.row;
|
| 691 |
+
const failed = r.status && r.status !== "ok";
|
| 692 |
+
const v = vals[i];
|
| 693 |
+
const pct = (typeof v === "number" && v > 0) ? Math.max(2, (v / maxVal) * 100) : 0;
|
| 694 |
+
const valText = failed && metric !== "peak_rss_mib"
|
| 695 |
+
? `<span class="badge b-err">${r.status.toUpperCase()}</span>`
|
| 696 |
+
: (typeof v === "number" ? `${fmt(v, metric === "peak_rss_mib" ? 0 : 2)} ${unit}` : `<span class="badge b-warn">N/A</span>`);
|
| 697 |
+
return `
|
| 698 |
+
<div class="bar-row">
|
| 699 |
+
<div class="bar-label">${item.label}<span class="bar-sub">${item.sub}</span></div>
|
| 700 |
+
<div class="bar-track">
|
| 701 |
+
<div class="bar-fill" style="width:${failed && metric !== "peak_rss_mib" ? 100 : pct}%;background:${engineColor(r.engine, r.status)};opacity:${failed && metric !== "peak_rss_mib" ? 0.22 : 1}"></div>
|
| 702 |
+
</div>
|
| 703 |
+
<div class="bar-value">${valText}</div>
|
| 704 |
+
</div>`;
|
| 705 |
+
}).join("");
|
| 706 |
+
|
| 707 |
+
const note = document.getElementById("cmp-note");
|
| 708 |
+
if (modelKey === "qwen3_30b") {
|
| 709 |
+
note.className = "callout";
|
| 710 |
+
note.innerHTML = "<strong>Honest loss on 128-expert MoE on Orin Nano:</strong> Stock <code>llama.cpp</code> CPU <code>mmap</code> (<code>1.27 tok/s</code>) beats <code>layerstream-plugin</code> (<code>0.32 tok/s</code>) here because Linux page cache absorbs small random expert slices in 6 GiB RAM, while tiny random <code>O_DIRECT</code> bursts trigger Orin's Tegra <code>nvme1</code> 30 s IRQ polling timeouts. On <code>OLMoE</code> (64 experts), TinyLFU hits 98–100% and delivers <strong>12.0–13.7 tok/s</strong>.";
|
| 711 |
+
} else if (modelKey === "14b") {
|
| 712 |
+
note.className = "callout good";
|
| 713 |
+
note.innerHTML = "<strong>Why layerstream wins on oversized dense models:</strong> Stock <code>llama.cpp</code> (the runtime behind Ollama & LM Studio) OOMs or times out, HF <code>accelerate</code> spends 144 s loading then times out, and AirLLM runs <code>0.033 tok/s</code> (8-token warm shards, 36.8 GiB disk). <code>layerstream-plugin</code> streams straight from a single 8.37 GiB GGUF at <strong>3.16–5.97 tok/s</strong> with <strong>0 MiB swap</strong>.";
|
| 714 |
+
} else if (modelKey === "32b") {
|
| 715 |
+
note.className = "callout good";
|
| 716 |
+
note.innerHTML = "<strong>2.53× larger than total system RAM:</strong> <code>Qwen2.5-32B</code> (18.49 GiB) runs cleanly on a 7.3 GiB Orin Nano with <strong>0 MiB swap</strong> at <strong>0.84–0.94 tok/s</strong> (<code>code_edit</code>), whereas stock <code>llama.cpp -ngl 10</code> thrashed <strong>+17.24 GiB</strong> of swap and produced 0 tokens in 300 s.";
|
| 717 |
+
} else {
|
| 718 |
+
note.className = "callout good";
|
| 719 |
+
note.innerHTML = "<strong>15.8×–17.9× MoE speedup at R=0:</strong> Combining active-expert router inspection with our scan-resistant TinyLFU hot-expert RAM cache lifts <code>OLMoE-1B-7B</code> from <code>0.76 tok/s</code> (full-layer) to <strong>12.05–13.67 tok/s</strong> and cuts energy <strong>10×</strong> (`14.34 → 1.40 J/tok`, net `0.45 J/tok`).";
|
| 720 |
+
}
|
| 721 |
+
}
|
| 722 |
+
|
| 723 |
+
function renderTuningCharts() {
|
| 724 |
+
const steps14b = [
|
| 725 |
+
{ label: "R2: Initial O_DIRECT + K=4 (R=14)", sub: "Load 53.4 s · 6.26 GiB RSS", val: 1.957, color: "#64748b" },
|
| 726 |
+
{ label: "R3: Chunked Load + Q8_0 KV + Adaptive K", sub: "Load 6.89 s (-7.7×)", val: 2.574, color: "#60a5fa" },
|
| 727 |
+
{ label: "R4: Single pin_buf + io_threads=1", sub: "5.71 GiB RSS · 1,834 MiB/s", val: 3.427, color: "#38bdf8" },
|
| 728 |
+
{ label: "R5: Zero-Copy cudaHostAlloc + cudaMemcpyAsync", sub: "R=19 · -4.5 s memcpy · 2,660 MiB/s", val: 5.684, color: "#2dd4bf" },
|
| 729 |
+
{ label: "R6: Final Tuned High-Residency (R=20)", sub: "6.15 GiB RSS · 2.73 J/tok gross", val: 5.974, color: "#34d399" },
|
| 730 |
+
{ label: "R8 v0.1.0: Safe Auto-Clamped (b5400 → R=7)", sub: "4.18 GiB RSS · 0 MiB swap · 2,961 MiB/s", val: 3.405, color: "#a78bfa" },
|
| 731 |
+
];
|
| 732 |
+
const max14 = Math.max(...steps14b.map(s => s.val));
|
| 733 |
+
document.getElementById("tuning-14b-chart").innerHTML = steps14b.map(s => `
|
| 734 |
+
<div class="bar-row">
|
| 735 |
+
<div class="bar-label">${s.label}<span class="bar-sub">${s.sub}</span></div>
|
| 736 |
+
<div class="bar-track"><div class="bar-fill" style="width:${(s.val / max14) * 100}%;background:${s.color}"></div></div>
|
| 737 |
+
<div class="bar-value">${s.val.toFixed(3)} tok/s</div>
|
| 738 |
+
</div>`).join("");
|
| 739 |
+
|
| 740 |
+
const ablations = [
|
| 741 |
+
{ label: "OLMoE R=0 Sparse + TinyLFU (prompt_heavy)", sub: "100% cache hit · 17.9× vs full-layer", val: 13.667, color: "#34d399" },
|
| 742 |
+
{ label: "OLMoE R=0 Full-Layer Stream", sub: "Reads all 64 experts every token", val: 0.763, color: "#64748b" },
|
| 743 |
+
{ label: "14B R=16 · io_threads=1 (optimal LUKS2)", sub: "Sequential BIOs across 6 kcryptd workers", val: 3.427, color: "#38bdf8" },
|
| 744 |
+
{ label: "14B R=16 · io_threads=4 (regressed!)", sub: "-38% code_edit / -57% open_ended on dm-crypt", val: 2.127, color: "#f87171" },
|
| 745 |
+
{ label: "14B + 0.5B Draft (R=19, 469 MiB draft)", sub: "Pareto winner: +7 resident target layers", val: 5.684, color: "#2dd4bf" },
|
| 746 |
+
{ label: "14B + 1.5B Draft (R=12, 1.04 GiB draft)", sub: "2.8× slower due to lost target residency", val: 2.005, color: "#fbbf24" },
|
| 747 |
+
];
|
| 748 |
+
const maxAb = Math.max(...ablations.map(s => s.val));
|
| 749 |
+
document.getElementById("tuning-ablation-chart").innerHTML = ablations.map(s => `
|
| 750 |
+
<div class="bar-row">
|
| 751 |
+
<div class="bar-label">${s.label}<span class="bar-sub">${s.sub}</span></div>
|
| 752 |
+
<div class="bar-track"><div class="bar-fill" style="width:${(s.val / maxAb) * 100}%;background:${s.color}"></div></div>
|
| 753 |
+
<div class="bar-value">${s.val.toFixed(3)} tok/s</div>
|
| 754 |
+
</div>`).join("");
|
| 755 |
+
}
|
| 756 |
+
|
| 757 |
+
// Interactive Hardware Estimator
|
| 758 |
+
const PRESETS = {
|
| 759 |
+
orin14: { ram: 5.2, nvme: 2800, model: 8.4, arch: "dense_draft" },
|
| 760 |
+
orin32: { ram: 5.2, nvme: 2400, model: 18.5, arch: "dense_draft" },
|
| 761 |
+
apu123: { ram: 48.0, nvme: 6000, model: 70.0, arch: "dense_draft" },
|
| 762 |
+
apu235: { ram: 48.0, nvme: 6000, model: 135.0, arch: "moe_cached" },
|
| 763 |
+
laptop70: { ram: 12.0, nvme: 4500, model: 40.0, arch: "dense_draft" },
|
| 764 |
+
};
|
| 765 |
+
|
| 766 |
+
function updateEstimator() {
|
| 767 |
+
const ram = parseFloat(document.getElementById("est-ram").value);
|
| 768 |
+
const nvme = parseFloat(document.getElementById("est-nvme").value);
|
| 769 |
+
const model = parseFloat(document.getElementById("est-model").value);
|
| 770 |
+
const arch = document.getElementById("est-arch").value;
|
| 771 |
+
|
| 772 |
+
document.getElementById("est-ram-val").textContent = `${ram.toFixed(1)} GiB`;
|
| 773 |
+
document.getElementById("est-nvme-val").textContent = `${nvme.toLocaleString()} MiB/s`;
|
| 774 |
+
document.getElementById("est-model-val").textContent = `${model.toFixed(1)} GiB`;
|
| 775 |
+
|
| 776 |
+
const nvmeGibS = nvme / 1024;
|
| 777 |
+
const draftReserve = arch === "dense_draft" ? 0.65 : 0;
|
| 778 |
+
const headroom = Math.max(1.1, ram * 0.14);
|
| 779 |
+
const kvAndSlots = Math.min(2.2, Math.max(0.9, model * 0.08));
|
| 780 |
+
const usableForWeights = Math.max(0, ram - headroom - kvAndSlots - draftReserve);
|
| 781 |
+
const fitsResident = model <= usableForWeights;
|
| 782 |
+
|
| 783 |
+
let pinnedFrac = fitsResident ? 1.0 : Math.max(0, Math.min(0.85, usableForWeights / model));
|
| 784 |
+
const streamedGib = Math.max(0, model * (1 - pinnedFrac));
|
| 785 |
+
const loadS = (Math.min(model, usableForWeights + 0.8) / Math.max(nvmeGibS, 0.1));
|
| 786 |
+
|
| 787 |
+
let lowTps, highTps, verdictBadge, verdictMsg, calloutClass = "callout good";
|
| 788 |
+
if (fitsResident) {
|
| 789 |
+
lowTps = 12; highTps = 45;
|
| 790 |
+
verdictBadge = `<span class="badge b-info">FITS RESIDENT</span>`;
|
| 791 |
+
verdictMsg = "<strong>Use stock llama.cpp / Mesh-LLM Skippy!</strong> This model fits completely in RAM/VRAM — running resident will be 10×–50× faster than NVMe streaming.";
|
| 792 |
+
} else if (nvme < 150) {
|
| 793 |
+
lowTps = 0.01; highTps = 0.05;
|
| 794 |
+
verdictBadge = `<span class="badge b-err">BLOCKED (<150 MiB/s)</span>`;
|
| 795 |
+
calloutClass = "callout";
|
| 796 |
+
verdictMsg = "<strong>Too slow for out-of-core streaming:</strong> <code>layerstream-plugin check</code> fails closed below 150 MiB/s (SD/HDD/USB) to avoid 0.01 tok/s misery.";
|
| 797 |
+
} else if (arch === "moe_cached") {
|
| 798 |
+
// Sparse MoE active fraction ~8-12% + TinyLFU cache hit rate scales with RAM/model
|
| 799 |
+
const activeFrac = 0.10;
|
| 800 |
+
const cacheHit = Math.min(0.95, Math.max(0.20, (usableForWeights / model) * 1.45));
|
| 801 |
+
const effReadGib = model * activeFrac * (1 - cacheHit * 0.85);
|
| 802 |
+
const sweepS = Math.max(0.06, effReadGib / nvmeGibS + 0.04);
|
| 803 |
+
lowTps = 1 / (sweepS * 1.4);
|
| 804 |
+
highTps = 1 / (sweepS * 0.75);
|
| 805 |
+
verdictBadge = lowTps >= 2 ? `<span class="badge b-ok">STRONG (MoE + TinyLFU)</span>` : `<span class="badge b-warn">USABLE (ASYNC / MoE)</span>`;
|
| 806 |
+
verdictMsg = `<strong>Sparse MoE estimate:</strong> ~${Math.round(cacheHit * 100)}% projected TinyLFU hot-expert cache hit rate in ~${usableForWeights.toFixed(1)} GiB RAM; remaining cold experts stream at ${nvme} MiB/s. Great fit for on-demand Mesh-LLM escalation.`;
|
| 807 |
+
} else {
|
| 808 |
+
const sweepS = Math.max(0.15, (streamedGib / nvmeGibS) * 1.15);
|
| 809 |
+
const multLow = arch === "dense_draft" ? 2.4 : 1.0;
|
| 810 |
+
const multHigh = arch === "dense_draft" ? 5.2 : 1.0;
|
| 811 |
+
lowTps = multLow / sweepS;
|
| 812 |
+
highTps = multHigh / sweepS;
|
| 813 |
+
if (lowTps >= 1.0) {
|
| 814 |
+
verdictBadge = `<span class="badge b-ok">GOOD</span>`;
|
| 815 |
+
verdictMsg = `<strong>Interactive / code-edit ready:</strong> Pins ~${Math.round(pinnedFrac * 100)}% of layers in RAM/VRAM and streams ${streamedGib.toFixed(1)} GiB/sweep with draft speculation.`;
|
| 816 |
+
} else if (highTps >= 0.5) {
|
| 817 |
+
verdictBadge = `<span class="badge b-warn">SLOW (BATCH / ASYNC)</span>`;
|
| 818 |
+
calloutClass = "callout";
|
| 819 |
+
verdictMsg = `<strong>Best for async tasks / code edits:</strong> Streams ${streamedGib.toFixed(1)} GiB/sweep (${sweepS.toFixed(1)} s/sweep). Pairing a small 0.5B–1.5B draft model is essential here.`;
|
| 820 |
+
} else {
|
| 821 |
+
verdictBadge = `<span class="badge b-err">VERY SLOW</span>`;
|
| 822 |
+
calloutClass = "callout";
|
| 823 |
+
verdictMsg = `<strong>Bandwidth-limited:</strong> Streaming ${streamedGib.toFixed(1)} GiB per sweep at ${nvme} MiB/s without speculation takes ${sweepS.toFixed(1)} s/token. Add a draft model, use a smaller quant, or choose a sparse MoE.`;
|
| 824 |
+
}
|
| 825 |
+
}
|
| 826 |
+
|
| 827 |
+
document.getElementById("est-output").innerHTML = `
|
| 828 |
+
<div class="kpi"><div class="label">Preflight Verdict</div><div class="val" style="font-size:1.1rem">${verdictBadge}</div><div class="sub">Mirrors <code>layerstream-plugin check</code></div></div>
|
| 829 |
+
<div class="kpi"><div class="label">Pinned Layers (R)</div><div class="val">${Math.round(pinnedFrac * 100)}%</div><div class="sub">${streamedGib.toFixed(1)} GiB streamed per sweep</div></div>
|
| 830 |
+
<div class="kpi"><div class="label">Est. Cold Load</div><div class="val">${loadS.toFixed(1)} s</div><div class="sub">0 MB weights while idle</div></div>
|
| 831 |
+
<div class="kpi"><div class="label">Est. Decode Speed</div><div class="val" style="color:var(--accent-2)">${fitsResident ? "12–45+" : `${lowTps.toFixed(2)}–${highTps.toFixed(2)}`} tok/s</div><div class="sub">${arch === "dense_draft" ? "prose → code_edit range" : arch === "moe_cached" ? "warm TinyLFU cache range" : "1 tok/sweep floor"}</div></div>
|
| 832 |
+
`;
|
| 833 |
+
const vEl = document.getElementById("est-verdict");
|
| 834 |
+
vEl.className = calloutClass;
|
| 835 |
+
vEl.innerHTML = verdictMsg;
|
| 836 |
+
}
|
| 837 |
+
|
| 838 |
+
document.querySelectorAll("[data-preset]").forEach(btn => {
|
| 839 |
+
btn.addEventListener("click", () => {
|
| 840 |
+
const p = PRESETS[btn.dataset.preset];
|
| 841 |
+
document.getElementById("est-ram").value = p.ram;
|
| 842 |
+
document.getElementById("est-nvme").value = p.nvme;
|
| 843 |
+
document.getElementById("est-model").value = p.model;
|
| 844 |
+
document.getElementById("est-arch").value = p.arch;
|
| 845 |
+
updateEstimator();
|
| 846 |
+
});
|
| 847 |
+
});
|
| 848 |
+
["est-ram", "est-nvme", "est-model", "est-arch"].forEach(id => {
|
| 849 |
+
document.getElementById(id).addEventListener("input", updateEstimator);
|
| 850 |
+
});
|
| 851 |
+
|
| 852 |
+
// Raw table filter
|
| 853 |
+
function populateRawFilters() {
|
| 854 |
+
const fill = (id, vals) => {
|
| 855 |
+
const sel = document.getElementById(id);
|
| 856 |
+
Array.from(new Set(vals.filter(Boolean))).sort().forEach(v => {
|
| 857 |
+
const opt = document.createElement("option");
|
| 858 |
+
opt.value = v; opt.textContent = v;
|
| 859 |
+
sel.appendChild(opt);
|
| 860 |
+
});
|
| 861 |
+
};
|
| 862 |
+
fill("raw-round", ALL_ROWS.map(r => r.round));
|
| 863 |
+
fill("raw-engine", ALL_ROWS.map(r => r.engine));
|
| 864 |
+
fill("raw-workload", ALL_ROWS.map(r => r.workload));
|
| 865 |
+
}
|
| 866 |
+
|
| 867 |
+
function renderRawTable() {
|
| 868 |
+
const rf = document.getElementById("raw-round").value;
|
| 869 |
+
const ef = document.getElementById("raw-engine").value;
|
| 870 |
+
const wf = document.getElementById("raw-workload").value;
|
| 871 |
+
const filtered = ALL_ROWS.filter(r =>
|
| 872 |
+
(rf === "all" || r.round === rf) &&
|
| 873 |
+
(ef === "all" || r.engine === ef) &&
|
| 874 |
+
(wf === "all" || r.workload === wf)
|
| 875 |
+
);
|
| 876 |
+
document.getElementById("raw-count").textContent = filtered.length;
|
| 877 |
+
document.getElementById("raw-tbody").innerHTML = filtered.map(r => {
|
| 878 |
+
const st = r.status || "ok";
|
| 879 |
+
const stBadge = st === "ok" ? `<span class="badge b-ok">ok</span>` : `<span class="badge b-err">${st}</span>`;
|
| 880 |
+
const dec = r.workload === "multi_turn_prefix_reuse"
|
| 881 |
+
? `${fmt(r.ttft_speedup_x, 2)}× TTFT`
|
| 882 |
+
: (r.decode_tok_per_s_min != null && r.decode_tok_per_s_min !== r.decode_tok_per_s_max
|
| 883 |
+
? `${fmt(r.decode_tok_per_s, 3)} (${fmt(r.decode_tok_per_s_min, 2)}–${fmt(r.decode_tok_per_s_max, 2)})`
|
| 884 |
+
: fmt(r.decode_tok_per_s, 3));
|
| 885 |
+
const jStr = r.joules_per_tok != null
|
| 886 |
+
? (r.net_joules_per_tok != null ? `${fmt(r.joules_per_tok, 1)} (${fmt(r.net_joules_per_tok, 1)})` : fmt(r.joules_per_tok, 1))
|
| 887 |
+
: "—";
|
| 888 |
+
const rStr = r.resident_layers != null ? `${r.resident_layers}/${r.n_layers || "?"}` : "—";
|
| 889 |
+
return `<tr>
|
| 890 |
+
<td>${r.round || "—"}</td>
|
| 891 |
+
<td>${r.engine || "—"}</td>
|
| 892 |
+
<td><code>${r.mode || "—"}</code></td>
|
| 893 |
+
<td>${r.model || "—"}</td>
|
| 894 |
+
<td>${r.workload || "—"}</td>
|
| 895 |
+
<td>${stBadge}</td>
|
| 896 |
+
<td class="num">${rStr}</td>
|
| 897 |
+
<td class="num">${fmt(r.load_time_s, 2)}</td>
|
| 898 |
+
<td class="num">${r.workload === "multi_turn_prefix_reuse" ? `${fmt(r.turn1_ttft_s, 1)}→${fmt(r.turn2_ttft_s, 1)}` : fmt(r.ttft_s, 2)}</td>
|
| 899 |
+
<td class="num">${dec}</td>
|
| 900 |
+
<td class="num">${fmt(r.pread_mib_s, 0)}</td>
|
| 901 |
+
<td class="num">${fmt(r.peak_rss_mib, 0)}</td>
|
| 902 |
+
<td class="num">${fmt(r.swap_delta_mib, 1)}</td>
|
| 903 |
+
<td class="num">${jStr}</td>
|
| 904 |
+
</tr>`;
|
| 905 |
+
}).join("");
|
| 906 |
+
}
|
| 907 |
+
|
| 908 |
+
["cmp-model", "cmp-workload", "cmp-metric"].forEach(id => document.getElementById(id).addEventListener("change", renderComparison));
|
| 909 |
+
["raw-round", "raw-engine", "raw-workload"].forEach(id => document.getElementById(id).addEventListener("change", renderRawTable));
|
| 910 |
+
|
| 911 |
+
async function init() {
|
| 912 |
+
updateEstimator();
|
| 913 |
+
renderTuningCharts();
|
| 914 |
+
try {
|
| 915 |
+
let resp = await fetch("./benchmarks.json");
|
| 916 |
+
if (!resp.ok) resp = await fetch(HF_JSON_URL);
|
| 917 |
+
ALL_ROWS = await resp.json();
|
| 918 |
+
document.getElementById("data-status").textContent = `Loaded ${ALL_ROWS.length} verified benchmark records`;
|
| 919 |
+
} catch (err) {
|
| 920 |
+
document.getElementById("data-status").textContent = `Failed to load dataset: ${err}`;
|
| 921 |
+
return;
|
| 922 |
+
}
|
| 923 |
+
populateRawFilters();
|
| 924 |
+
renderComparison();
|
| 925 |
+
renderRawTable();
|
| 926 |
+
}
|
| 927 |
+
init();
|
| 928 |
+
</script>
|
| 929 |
+
</body>
|
| 930 |
</html>
|