Spaces:
Running
Running
Commit ·
44745f2
1
Parent(s): 20fb354
Commiting v0.3
Browse files- CHANGELOG.md +33 -2
- README.md +158 -97
- app.js +613 -59
- docs/architecture.md +34 -13
- docs/methodology.md +75 -31
- docs/research.md +43 -13
- docs/validation.md +34 -18
- index.html +256 -39
- py/inferscale/__init__.py +4 -2
- py/inferscale/api.py +10 -4
- py/inferscale/diagnostics.py +35 -10
- py/inferscale/disaggregated.py +362 -0
- py/inferscale/kv_cache.py +9 -7
- py/inferscale/latency.py +1 -1
- py/inferscale/models.py +32 -0
- py/inferscale/optimizer.py +142 -0
- py/inferscale/simulator.py +22 -7
- py/inferscale/workloads.py +7 -4
- pyproject.toml +1 -1
- scripts/release_check.py +54 -10
- src/inferscale/__init__.py +4 -2
- src/inferscale/api.py +10 -4
- src/inferscale/diagnostics.py +35 -10
- src/inferscale/disaggregated.py +362 -0
- src/inferscale/kv_cache.py +9 -7
- src/inferscale/latency.py +1 -1
- src/inferscale/models.py +32 -0
- src/inferscale/optimizer.py +142 -0
- src/inferscale/simulator.py +22 -7
- src/inferscale/workloads.py +7 -4
- styles.css +288 -12
- tests/test_design_space.py +41 -0
- tests/test_disaggregated.py +39 -0
- tests/test_prefix_cache.py +28 -0
- worker.mjs +1 -0
CHANGELOG.md
CHANGED
|
@@ -1,9 +1,41 @@
|
|
| 1 |
# Changelog
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
## v0.2.0
|
| 4 |
|
| 5 |
- auto-hide runtime header after Pyodide becomes ready; hover/focus at the top edge reveals it
|
| 6 |
-
- redesigned the interface around a flatter
|
| 7 |
- replaced decorative/non-ASCII ranking glyphs with explicit text labels
|
| 8 |
- added chart expansion with Escape-to-close support
|
| 9 |
- added copy-to-clipboard and CSV export for Scheduler Arena and Capacity Planner tables
|
|
@@ -14,7 +46,6 @@
|
|
| 14 |
- made capacity feasibility conservative across repetitions: every repetition must meet the target and fully drain
|
| 15 |
- added minimum/mean/maximum repetition evidence to capacity traces
|
| 16 |
- changed the capacity plot to a true linear request-rate x-axis and added worst-repetition evidence
|
| 17 |
-
- strengthened release checks for browser-module synchronization, package version, provenance, and ASCII-safe UI labels
|
| 18 |
|
| 19 |
## v0.1.0
|
| 20 |
|
|
|
|
| 1 |
# Changelog
|
| 2 |
|
| 3 |
+
## v0.3.0
|
| 4 |
+
|
| 5 |
+
### Research / simulator
|
| 6 |
+
|
| 7 |
+
- added prefill/decode disaggregation with independent accelerator profiles and worker counts
|
| 8 |
+
- added a genuine event-driven P/D pipeline with overlapping prefill, transfer, and decode activity
|
| 9 |
+
- added explicit serialized KV-transfer modeling with bandwidth/base-latency controls
|
| 10 |
+
- added P/D telemetry: role utilization, transfer volume, mean/p95 transfer latency, and transfer utilization
|
| 11 |
+
- added P/D-specific bottleneck diagnoses for prefill-pool, decode-pool, and KV-transfer pressure
|
| 12 |
+
- added deterministic shared-prefix reuse with independent cache RNG so enabling cache preserves the workload trace
|
| 13 |
+
- added shared-prefix KV accounting and saved-prefill-token telemetry
|
| 14 |
+
- added four-way colocated/P-D x cache-off/on comparison
|
| 15 |
+
- added bounded design-space exploration and goodput/p95-TTFT Pareto-frontier marking
|
| 16 |
+
|
| 17 |
+
### UI / usability
|
| 18 |
+
|
| 19 |
+
- removed the product-style footer
|
| 20 |
+
- simplified top branding toward a research-prototype presentation
|
| 21 |
+
- added explicit Worst repetition and Target columns to Capacity Planner tables/exports
|
| 22 |
+
- removed area fill from the capacity line chart
|
| 23 |
+
- changed line-chart legends to line swatches rather than rectangle blocks
|
| 24 |
+
- added PNG download controls to every chart
|
| 25 |
+
- added meaningful experiment-aware PNG filenames instead of generic chart names
|
| 26 |
+
- retained full-screen chart expansion and Escape-to-close behavior
|
| 27 |
+
- added Modern Serving and Design Explorer sections
|
| 28 |
+
- added live topology, shared-prefix, P/D worker, and interconnect controls to Serving Lab
|
| 29 |
+
|
| 30 |
+
### Validation
|
| 31 |
+
|
| 32 |
+
- expanded tests to 17 deterministic cases
|
| 33 |
+
- release check now validates P/D transfer telemetry, design-space output, PNG controls, planner evidence columns, UI ASCII safety, and absence of the old footer
|
| 34 |
+
|
| 35 |
## v0.2.0
|
| 36 |
|
| 37 |
- auto-hide runtime header after Pyodide becomes ready; hover/focus at the top edge reveals it
|
| 38 |
+
- redesigned the interface around a flatter engineering-dashboard visual system
|
| 39 |
- replaced decorative/non-ASCII ranking glyphs with explicit text labels
|
| 40 |
- added chart expansion with Escape-to-close support
|
| 41 |
- added copy-to-clipboard and CSV export for Scheduler Arena and Capacity Planner tables
|
|
|
|
| 46 |
- made capacity feasibility conservative across repetitions: every repetition must meet the target and fully drain
|
| 47 |
- added minimum/mean/maximum repetition evidence to capacity traces
|
| 48 |
- changed the capacity plot to a true linear request-rate x-axis and added worst-repetition evidence
|
|
|
|
| 49 |
|
| 50 |
## v0.1.0
|
| 51 |
|
README.md
CHANGED
|
@@ -10,87 +10,137 @@ license: mit
|
|
| 10 |
short_description: Interactive LLM serving simulator and SLO planner
|
| 11 |
---
|
| 12 |
|
| 13 |
-
# InferScale-Sim v0.
|
| 14 |
|
| 15 |
-
**Interactive LLM serving simulator and SLO-aware
|
| 16 |
|
| 17 |
-
InferScale-Sim explores a
|
| 18 |
|
| 19 |
-
> How do
|
| 20 |
|
| 21 |
-
The public Hugging Face Space uses **no server CPU, no GPU, no API key, and no paid inference provider**. Hugging Face serves static files; Pyodide executes the same Python
|
| 22 |
|
| 23 |
> [!IMPORTANT]
|
| 24 |
-
> v0.
|
| 25 |
|
| 26 |
## Why simulation?
|
| 27 |
|
| 28 |
-
Exhaustively testing
|
| 29 |
|
| 30 |
-
|
| 31 |
|
| 32 |
-
InferScale-Sim is
|
| 33 |
|
| 34 |
-
## v0.
|
| 35 |
|
| 36 |
-
|
|
|
|
|
|
|
| 37 |
- log-normal prompt/output-length distributions
|
| 38 |
-
-
|
| 39 |
-
-
|
| 40 |
- shortest-job-first scheduling
|
| 41 |
-
-
|
| 42 |
-
-
|
| 43 |
- paged KV-cache accounting and VRAM admission control
|
| 44 |
- analytical roofline-style prefill/decode latency proxy
|
| 45 |
- FP16 / INT8 / INT4 weight-footprint scenarios
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
-
|
| 50 |
-
-
|
| 51 |
-
-
|
| 52 |
-
-
|
| 53 |
-
-
|
| 54 |
-
-
|
| 55 |
-
-
|
| 56 |
-
|
| 57 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
|
| 59 |
## Architecture
|
| 60 |
|
| 61 |
```text
|
| 62 |
Hugging Face Static Space
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
|
|
|
|
|
|
| 89 |
```
|
| 90 |
|
|
|
|
|
|
|
| 91 |
## Run locally
|
| 92 |
|
| 93 |
-
The simulator
|
| 94 |
|
| 95 |
```bash
|
| 96 |
python -m venv .venv
|
|
@@ -102,23 +152,23 @@ ruff check src tests scripts
|
|
| 102 |
python scripts/release_check.py
|
| 103 |
```
|
| 104 |
|
| 105 |
-
Run one simulation
|
| 106 |
|
| 107 |
```bash
|
| 108 |
python scripts/run_simulation.py
|
| 109 |
```
|
| 110 |
|
| 111 |
-
Serve the browser app
|
| 112 |
|
| 113 |
```bash
|
| 114 |
python -m http.server 8000
|
| 115 |
```
|
| 116 |
|
| 117 |
-
Open `http://localhost:8000`. The first page load downloads
|
| 118 |
|
| 119 |
## Deploy to Hugging Face
|
| 120 |
|
| 121 |
-
Create a **Static Space**
|
| 122 |
|
| 123 |
```yaml
|
| 124 |
sdk: static
|
|
@@ -127,80 +177,91 @@ app_file: index.html
|
|
| 127 |
|
| 128 |
No Hugging Face secret is required.
|
| 129 |
|
| 130 |
-
Before
|
| 131 |
|
| 132 |
```bash
|
| 133 |
python scripts/sync_web_python.py
|
| 134 |
-
python scripts/release_check.py
|
| 135 |
pytest -q
|
|
|
|
| 136 |
```
|
| 137 |
|
| 138 |
-
`sync_web_python.py` mirrors the canonical `src/inferscale/` package into `py/inferscale/`, which
|
| 139 |
|
| 140 |
-
## Core
|
| 141 |
|
| 142 |
### Goodput
|
| 143 |
|
| 144 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
|
| 146 |
```text
|
| 147 |
-
|
| 148 |
```
|
| 149 |
|
| 150 |
-
|
| 151 |
|
| 152 |
-
|
| 153 |
|
| 154 |
-
|
| 155 |
|
| 156 |
-
|
| 157 |
|
| 158 |
-
|
|
|
|
|
|
|
| 159 |
|
| 160 |
-
|
| 161 |
|
| 162 |
-
|
| 163 |
|
| 164 |
-
|
|
|
|
|
|
|
| 165 |
|
| 166 |
## Research lineage
|
| 167 |
|
| 168 |
-
- **Vidur: A Large-Scale Simulation Framework for LLM Inference** (2024)
|
| 169 |
-
- **
|
| 170 |
-
- **
|
| 171 |
-
- **
|
| 172 |
-
- **
|
|
|
|
|
|
|
| 173 |
|
| 174 |
-
See
|
| 175 |
|
| 176 |
## Repository
|
| 177 |
|
| 178 |
```text
|
| 179 |
.
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
```
|
| 190 |
|
| 191 |
## Roadmap
|
| 192 |
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
Potential v0.3 extensions, only after v0.2 validation:
|
| 196 |
|
| 197 |
-
- empirical
|
| 198 |
-
-
|
| 199 |
-
-
|
| 200 |
-
- multi-replica routing
|
| 201 |
-
-
|
| 202 |
-
-
|
| 203 |
-
- calibration-vs-held-out validation report
|
| 204 |
|
| 205 |
## License
|
| 206 |
|
|
|
|
| 10 |
short_description: Interactive LLM serving simulator and SLO planner
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# InferScale-Sim v0.3.0
|
| 14 |
|
| 15 |
+
**Interactive LLM serving simulator and SLO-aware design laboratory - written in Python and executed entirely in the browser.**
|
| 16 |
|
| 17 |
+
InferScale-Sim explores a systems question:
|
| 18 |
|
| 19 |
+
> How do workload shape, batching, scheduling, KV-cache pressure, reusable prefixes, and prefill/decode disaggregation change tail latency and sustainable LLM-serving capacity?
|
| 20 |
|
| 21 |
+
The public Hugging Face Space uses **no server CPU, no GPU, no API key, and no paid inference provider**. Hugging Face serves static files; Pyodide executes the same Python package used by the local test suite inside a Web Worker on the visitor's ordinary CPU.
|
| 22 |
|
| 23 |
> [!IMPORTANT]
|
| 24 |
+
> v0.3 ships with **analytical reference latency profiles**, not measured GPU calibration data. It is designed for systems behavior, what-if studies, and comparative exploration. Absolute milliseconds must not be presented as empirical L4/A10G/A100 benchmark results.
|
| 25 |
|
| 26 |
## Why simulation?
|
| 27 |
|
| 28 |
+
Exhaustively testing serving configurations on real GPU clusters is expensive. **Vidur** reported finding a LLaMA2-70B deployment configuration in roughly one CPU-hour while estimating that deployment-based exploration would require about **42,000 GPU-hours (~$218K)**. Its evaluated latency predictions were within 9% error.
|
| 29 |
|
| 30 |
+
That research direction has continued. **Revati (2026)** explores GPU-free time-warp emulation of real serving control paths; **LLMServingSim 2.0 (2026)** models heterogeneous and disaggregated serving; **Frontier (May 2026)** models P/D disaggregation, communication, runtime optimizations, stateful workloads, and SLA-dependent Pareto exploration; and **HeteroPanacea (Aug 2026)** studies heterogeneous specialization across prefill/decode/attention/FFN serving stages.
|
| 31 |
|
| 32 |
+
InferScale-Sim is deliberately smaller: a dependency-light, inspectable Python implementation intended to make the core serving trade-offs interactive and understandable.
|
| 33 |
|
| 34 |
+
## v0.3 capabilities
|
| 35 |
|
| 36 |
+
### Workload and serving loop
|
| 37 |
+
|
| 38 |
+
- deterministic constant, Poisson, and bursty arrival processes
|
| 39 |
- log-normal prompt/output-length distributions
|
| 40 |
+
- static batching baseline
|
| 41 |
+
- continuous batching with FCFS
|
| 42 |
- shortest-job-first scheduling
|
| 43 |
+
- least-slack/SLO-aware scheduling
|
| 44 |
+
- chunked prefill
|
| 45 |
- paged KV-cache accounting and VRAM admission control
|
| 46 |
- analytical roofline-style prefill/decode latency proxy
|
| 47 |
- FP16 / INT8 / INT4 weight-footprint scenarios
|
| 48 |
+
|
| 49 |
+
### Metrics and SLOs
|
| 50 |
+
|
| 51 |
+
- TTFT, TPOT, E2E and queue-latency percentiles
|
| 52 |
+
- request and output-token throughput
|
| 53 |
+
- **goodput**: completed requests that satisfy both configured SLOs per simulated second
|
| 54 |
+
- component TTFT/E2E SLO attainment
|
| 55 |
+
- virtual resource utilization
|
| 56 |
+
- peak KV usage
|
| 57 |
+
- heuristic bottleneck diagnosis with explicit simulator provenance
|
| 58 |
+
|
| 59 |
+
### Prefix reuse - new in v0.3
|
| 60 |
+
|
| 61 |
+
v0.3 adds a controlled shared-prefix scenario:
|
| 62 |
+
|
| 63 |
+
- configurable shared-prefix length
|
| 64 |
+
- configurable request reuse fraction
|
| 65 |
+
- deterministic cache-hit assignment independent of the generated workload trace
|
| 66 |
+
- cached prefill tokens skipped on a hit
|
| 67 |
+
- one persistent shared KV allocation instead of per-request duplication
|
| 68 |
+
- cache-hit rate and saved-prefill-token telemetry
|
| 69 |
+
|
| 70 |
+
This is intentionally **not** a complete RadixAttention implementation. It isolates the compute/memory effect of exact reusable prefixes while keeping the simulator small enough to inspect.
|
| 71 |
+
|
| 72 |
+
### Prefill/decode disaggregation - new in v0.3
|
| 73 |
+
|
| 74 |
+
The simulator now supports a separate P/D topology with:
|
| 75 |
+
|
| 76 |
+
- independent prefill/decode accelerator profiles
|
| 77 |
+
- configurable prefill and decode worker counts
|
| 78 |
+
- independent role utilization
|
| 79 |
+
- continuous decode admission between iterations
|
| 80 |
+
- explicit KV transfer after prefill
|
| 81 |
+
- serialized analytical interconnect model
|
| 82 |
+
- configurable interconnect GB/s and base transfer latency
|
| 83 |
+
- p95 KV-transfer latency, total transfer volume, and link utilization
|
| 84 |
+
- P/D-specific bottleneck diagnoses such as prefill-pool, decode-pool, and transfer pressure
|
| 85 |
+
|
| 86 |
+
### Interactive tools
|
| 87 |
+
|
| 88 |
+
**Serving Lab** - run one colocated or P/D scenario and inspect live metrics, timelines, cache behavior, and diagnostics.
|
| 89 |
+
|
| 90 |
+
**Scheduler Arena** - run every colocated scheduler against the identical deterministic workload.
|
| 91 |
+
|
| 92 |
+
**Capacity Planner** - robust binary search for the highest request rate where **every repetition** satisfies the target and drains. The trace exposes mean SLO attainment, worst repetition, target, and min/max range.
|
| 93 |
+
|
| 94 |
+
**Modern Serving Lab** - compare four controlled variants on one workload:
|
| 95 |
+
|
| 96 |
+
1. colocated
|
| 97 |
+
2. colocated + prefix reuse
|
| 98 |
+
3. P/D disaggregated
|
| 99 |
+
4. P/D disaggregated + prefix reuse
|
| 100 |
+
|
| 101 |
+
**Design Explorer** - runs a bounded live sweep over scheduler, batch size, prefix caching, and P/D worker splits, then reports two non-dominated frontiers: **raw goodput vs p95 TTFT** and **goodput per accelerator vs p95 TTFT**. The second view makes the resource cost of multi-worker P/D layouts explicit.
|
| 102 |
+
|
| 103 |
+
All charts can be expanded and exported as meaningfully named PNG files. Result tables support clipboard copy and CSV export.
|
| 104 |
|
| 105 |
## Architecture
|
| 106 |
|
| 107 |
```text
|
| 108 |
Hugging Face Static Space
|
| 109 |
+
|
|
| 110 |
+
| serves files only
|
| 111 |
+
v
|
| 112 |
+
+----------------------------------------------------------------+
|
| 113 |
+
| Browser |
|
| 114 |
+
| |
|
| 115 |
+
| UI / Chart.js Pyodide Web Worker |
|
| 116 |
+
| | | |
|
| 117 |
+
| +------------------------------->| |
|
| 118 |
+
| v |
|
| 119 |
+
| Python inferscale package |
|
| 120 |
+
| | |
|
| 121 |
+
| +--------------------------------+-------------------+ |
|
| 122 |
+
| | | | | |
|
| 123 |
+
| workload schedulers KV cache profiles |
|
| 124 |
+
| | | | | |
|
| 125 |
+
| +--------------------+-----------------+-------------+ |
|
| 126 |
+
| | |
|
| 127 |
+
| +--------------------+------------------+ |
|
| 128 |
+
| | | |
|
| 129 |
+
| colocated simulator P/D simulator |
|
| 130 |
+
| | |
|
| 131 |
+
| prefill -> transfer -> decode|
|
| 132 |
+
| | | |
|
| 133 |
+
| +--------------------+------------------+ |
|
| 134 |
+
| v |
|
| 135 |
+
| metrics / SLO / search |
|
| 136 |
+
+----------------------------------------------------------------+
|
| 137 |
```
|
| 138 |
|
| 139 |
+
The discrete-event engine never sleeps for simulated compute time. A predicted 40 ms decode step advances virtual time by 0.040 seconds immediately.
|
| 140 |
+
|
| 141 |
## Run locally
|
| 142 |
|
| 143 |
+
The Python simulator has **zero runtime dependencies** beyond Python 3.10+.
|
| 144 |
|
| 145 |
```bash
|
| 146 |
python -m venv .venv
|
|
|
|
| 152 |
python scripts/release_check.py
|
| 153 |
```
|
| 154 |
|
| 155 |
+
Run one simulation:
|
| 156 |
|
| 157 |
```bash
|
| 158 |
python scripts/run_simulation.py
|
| 159 |
```
|
| 160 |
|
| 161 |
+
Serve the browser app:
|
| 162 |
|
| 163 |
```bash
|
| 164 |
python -m http.server 8000
|
| 165 |
```
|
| 166 |
|
| 167 |
+
Open `http://localhost:8000`. The first page load downloads Pyodide and Chart.js from the CDN; simulation runs then execute locally in the worker.
|
| 168 |
|
| 169 |
## Deploy to Hugging Face
|
| 170 |
|
| 171 |
+
Create or reuse a **Static Space** and push this repository. Root metadata already contains:
|
| 172 |
|
| 173 |
```yaml
|
| 174 |
sdk: static
|
|
|
|
| 177 |
|
| 178 |
No Hugging Face secret is required.
|
| 179 |
|
| 180 |
+
Before deployment:
|
| 181 |
|
| 182 |
```bash
|
| 183 |
python scripts/sync_web_python.py
|
|
|
|
| 184 |
pytest -q
|
| 185 |
+
python scripts/release_check.py
|
| 186 |
```
|
| 187 |
|
| 188 |
+
`sync_web_python.py` mirrors the canonical `src/inferscale/` package into `py/inferscale/`, which the Pyodide worker imports. CI fails if the mirror is stale.
|
| 189 |
|
| 190 |
+
## Core modeling choices
|
| 191 |
|
| 192 |
### Goodput
|
| 193 |
|
| 194 |
+
```text
|
| 195 |
+
goodput = requests satisfying TTFT and E2E SLOs / simulated makespan
|
| 196 |
+
```
|
| 197 |
+
|
| 198 |
+
Raw throughput can reward overload. Goodput penalizes requests that finish too late to be useful.
|
| 199 |
+
|
| 200 |
+
### KV-cache model
|
| 201 |
+
|
| 202 |
+
Per-token KV bytes are approximated as:
|
| 203 |
|
| 204 |
```text
|
| 205 |
+
2 x layers x KV heads x head dimension x 2 bytes
|
| 206 |
```
|
| 207 |
|
| 208 |
+
for K and V with FP16 KV state. Paged allocation rounds live per-request state to configurable token blocks.
|
| 209 |
|
| 210 |
+
With prefix reuse enabled, the shared prefix is represented once as persistent KV state and request allocations contain only the uncached suffix plus generated tokens.
|
| 211 |
|
| 212 |
+
### P/D transfer model
|
| 213 |
|
| 214 |
+
After prefill, newly computed prompt KV is moved to the decode pool through a serialized reference link:
|
| 215 |
|
| 216 |
+
```text
|
| 217 |
+
transfer_time = base_latency + KV_bytes / interconnect_bandwidth
|
| 218 |
+
```
|
| 219 |
|
| 220 |
+
The link is intentionally simple and explicit. It does not claim to reproduce NCCL, NIXL, RDMA, PCIe, or NVLink behavior.
|
| 221 |
|
| 222 |
+
### Latency-profile honesty
|
| 223 |
|
| 224 |
+
`AnalyticalLatencyModel` estimates operation duration from model architecture, accelerator peak FP16 compute, memory bandwidth, quantization footprint, context length, and conservative efficiency factors. It is an analytical proxy.
|
| 225 |
+
|
| 226 |
+
A future empirical interpolator can replace that backend without rewriting workload generation, scheduling, KV logic, P/D orchestration, or metrics.
|
| 227 |
|
| 228 |
## Research lineage
|
| 229 |
|
| 230 |
+
- **Vidur: A Large-Scale Simulation Framework for LLM Inference** (MLSys 2024) - profiling + simulation + deployment search. https://arxiv.org/abs/2405.05465
|
| 231 |
+
- **SGLang: Efficient Execution of Structured Language Model Programs** (NeurIPS 2024) - RadixAttention motivates exact shared-prefix reuse. https://arxiv.org/abs/2312.07104
|
| 232 |
+
- **TokenSim** (2025) - extensible scheduling and memory-management simulation. https://arxiv.org/abs/2503.08415
|
| 233 |
+
- **Revati: Transparent GPU-Free Time-Warp Emulation for LLM Serving** (2026) - real control logic with virtualized GPU time. https://arxiv.org/abs/2601.00397
|
| 234 |
+
- **LLMServingSim 2.0** (2026) - heterogeneous/disaggregated infrastructure and runtime interactions. https://arxiv.org/abs/2602.23036
|
| 235 |
+
- **Frontier: Towards Comprehensive and Accurate LLM Inference Simulation** (2026) - P/D and Attention-FFN disaggregation, runtime optimizations, stateful workloads, and Pareto exploration. https://arxiv.org/abs/2605.21312
|
| 236 |
+
- **When Does Disaggregation Pay? / HeteroPanacea** (Aug 2026) - heterogeneous P/D/attention/FFN specialization and cross-stack design exploration. https://arxiv.org/abs/2608.03741
|
| 237 |
|
| 238 |
+
See `docs/research.md`, `docs/methodology.md`, and `docs/validation.md` for scope and limitations.
|
| 239 |
|
| 240 |
## Repository
|
| 241 |
|
| 242 |
```text
|
| 243 |
.
|
| 244 |
+
|-- src/inferscale/ canonical Python simulator
|
| 245 |
+
|-- py/inferscale/ generated browser mirror
|
| 246 |
+
|-- tests/ deterministic unit tests
|
| 247 |
+
|-- scripts/ local runner + release tooling
|
| 248 |
+
|-- docs/ architecture / methodology / research notes
|
| 249 |
+
|-- index.html HF Static Space entry point
|
| 250 |
+
|-- app.js UI + charts + export tooling
|
| 251 |
+
|-- worker.mjs Pyodide Web Worker bridge
|
| 252 |
+
`-- styles.css
|
| 253 |
```
|
| 254 |
|
| 255 |
## Roadmap
|
| 256 |
|
| 257 |
+
v0.3 deliberately stops short of pretending to be a production-runtime emulator. Strong next steps would be:
|
|
|
|
|
|
|
| 258 |
|
| 259 |
+
- empirical latency-profile import and held-out calibration report
|
| 260 |
+
- multi-turn / agentic session traces
|
| 261 |
+
- speculative decoding
|
| 262 |
+
- multi-replica routing and tenant fairness
|
| 263 |
+
- richer prefix-tree eviction/scheduling
|
| 264 |
+
- attention/FFN disaggregation
|
|
|
|
| 265 |
|
| 266 |
## License
|
| 267 |
|
app.js
CHANGED
|
@@ -1,23 +1,35 @@
|
|
| 1 |
const $ = (id) => document.getElementById(id);
|
| 2 |
-
const runtimePill = $("runtimePill");
|
| 3 |
-
const runtimeText = $("runtimeText");
|
| 4 |
const charts = {};
|
| 5 |
const pending = new Map();
|
|
|
|
|
|
|
| 6 |
let lastResult = null;
|
| 7 |
let lastArenaRows = [];
|
| 8 |
let lastCapacityTrace = [];
|
| 9 |
-
let
|
| 10 |
-
let
|
| 11 |
|
| 12 |
-
const COLORS = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
-
const worker = new Worker("./worker.mjs", {type:"module"});
|
| 15 |
worker.addEventListener("message", (event) => {
|
| 16 |
const data = event.data || {};
|
| 17 |
if (data.type === "ready") {
|
| 18 |
runtimePill.classList.add("ready");
|
| 19 |
runtimeText.textContent = "Python runtime ready";
|
| 20 |
-
["runBtn","arenaBtn","capacityBtn"].forEach((id) => $(id).disabled = false);
|
| 21 |
window.setTimeout(() => document.body.classList.add("runtime-ready"), 650);
|
| 22 |
return;
|
| 23 |
}
|
|
@@ -28,59 +40,601 @@ worker.addEventListener("message", (event) => {
|
|
| 28 |
return;
|
| 29 |
}
|
| 30 |
if (!pending.has(data.id)) return;
|
| 31 |
-
const {resolve,reject} = pending.get(data.id);
|
| 32 |
pending.delete(data.id);
|
| 33 |
data.error ? reject(new Error(data.error)) : resolve(data.result);
|
| 34 |
});
|
| 35 |
|
| 36 |
-
function callPython(action,payload)
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
function
|
| 46 |
-
|
| 47 |
-
function
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
function
|
| 85 |
-
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
const $ = (id) => document.getElementById(id);
|
|
|
|
|
|
|
| 2 |
const charts = {};
|
| 3 |
const pending = new Map();
|
| 4 |
+
let requestId = 0;
|
| 5 |
+
let toastTimer = null;
|
| 6 |
let lastResult = null;
|
| 7 |
let lastArenaRows = [];
|
| 8 |
let lastCapacityTrace = [];
|
| 9 |
+
let lastTopologyRows = [];
|
| 10 |
+
let lastDesignRows = [];
|
| 11 |
|
| 12 |
+
const COLORS = {
|
| 13 |
+
blue: "#79a7ff",
|
| 14 |
+
blue2: "#5d8fe9",
|
| 15 |
+
steel: "#92a2b5",
|
| 16 |
+
green: "#69c99a",
|
| 17 |
+
amber: "#dfb966",
|
| 18 |
+
red: "#df7d89",
|
| 19 |
+
gray: "#667384",
|
| 20 |
+
grid: "rgba(140,155,175,.14)",
|
| 21 |
+
};
|
| 22 |
+
|
| 23 |
+
const runtimePill = $("runtimePill");
|
| 24 |
+
const runtimeText = $("runtimeText");
|
| 25 |
+
const worker = new Worker("./worker.mjs", { type: "module" });
|
| 26 |
|
|
|
|
| 27 |
worker.addEventListener("message", (event) => {
|
| 28 |
const data = event.data || {};
|
| 29 |
if (data.type === "ready") {
|
| 30 |
runtimePill.classList.add("ready");
|
| 31 |
runtimeText.textContent = "Python runtime ready";
|
| 32 |
+
["runBtn", "arenaBtn", "capacityBtn", "topologyBtn", "designBtn"].forEach((id) => { $(id).disabled = false; });
|
| 33 |
window.setTimeout(() => document.body.classList.add("runtime-ready"), 650);
|
| 34 |
return;
|
| 35 |
}
|
|
|
|
| 40 |
return;
|
| 41 |
}
|
| 42 |
if (!pending.has(data.id)) return;
|
| 43 |
+
const { resolve, reject } = pending.get(data.id);
|
| 44 |
pending.delete(data.id);
|
| 45 |
data.error ? reject(new Error(data.error)) : resolve(data.result);
|
| 46 |
});
|
| 47 |
|
| 48 |
+
function callPython(action, payload) {
|
| 49 |
+
const id = ++requestId;
|
| 50 |
+
return new Promise((resolve, reject) => {
|
| 51 |
+
pending.set(id, { resolve, reject });
|
| 52 |
+
worker.postMessage({ id, action, payload });
|
| 53 |
+
});
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
function num(id) { return Number($(id).value); }
|
| 57 |
+
function boolSelect(id) { return $(id).value === "on"; }
|
| 58 |
+
|
| 59 |
+
function configFromUI(overrides = {}) {
|
| 60 |
+
return {
|
| 61 |
+
model: $("model").value,
|
| 62 |
+
accelerator: $("accelerator").value,
|
| 63 |
+
prefill_accelerator: $("prefillAccelerator").value,
|
| 64 |
+
decode_accelerator: $("decodeAccelerator").value,
|
| 65 |
+
prefill_workers: num("prefillWorkers"),
|
| 66 |
+
decode_workers: num("decodeWorkers"),
|
| 67 |
+
interconnect_gbps: num("interconnect"),
|
| 68 |
+
transfer_base_ms: num("transferBase"),
|
| 69 |
+
topology: $("topology").value,
|
| 70 |
+
scheduler: $("scheduler").value,
|
| 71 |
+
quantization: $("quantization").value,
|
| 72 |
+
prefix_cache_enabled: boolSelect("prefixCache"),
|
| 73 |
+
shared_prefix_tokens: num("sharedPrefix"),
|
| 74 |
+
prefix_reuse_fraction: num("prefixReuse"),
|
| 75 |
+
arrival_process: $("arrival").value,
|
| 76 |
+
request_rate_rps: num("rate"),
|
| 77 |
+
duration_s: num("duration"),
|
| 78 |
+
prompt_tokens_mean: num("promptMean"),
|
| 79 |
+
prompt_tokens_cv: num("promptCv"),
|
| 80 |
+
output_tokens_mean: num("outputMean"),
|
| 81 |
+
output_tokens_cv: num("outputCv"),
|
| 82 |
+
max_batch_size: num("maxBatch"),
|
| 83 |
+
max_batch_tokens: num("maxBatchTokens"),
|
| 84 |
+
chunk_size: num("chunkSize"),
|
| 85 |
+
kv_block_tokens: num("kvBlock"),
|
| 86 |
+
burst_multiplier: num("burstMultiplier"),
|
| 87 |
+
burst_period_s: num("burstPeriod"),
|
| 88 |
+
seed: num("seed"),
|
| 89 |
+
slo_ttft_ms: num("sloTtft"),
|
| 90 |
+
slo_e2e_ms: num("sloE2e"),
|
| 91 |
+
slo_attainment_target: 0.99,
|
| 92 |
+
...overrides,
|
| 93 |
+
};
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
function fmt(value, digits = 1) {
|
| 97 |
+
if (!Number.isFinite(value)) return "N/A";
|
| 98 |
+
return value.toLocaleString(undefined, { maximumFractionDigits: digits });
|
| 99 |
+
}
|
| 100 |
+
function pct(value, digits = 1) { return `${fmt(value * 100, digits)}%`; }
|
| 101 |
+
function escapeHtml(value) {
|
| 102 |
+
return String(value).replace(/[&<>'"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", "'": "'", '"': """ })[c]);
|
| 103 |
+
}
|
| 104 |
+
function slug(value) {
|
| 105 |
+
return String(value).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
| 106 |
+
}
|
| 107 |
+
function stamp() {
|
| 108 |
+
const d = new Date();
|
| 109 |
+
const pad = (v) => String(v).padStart(2, "0");
|
| 110 |
+
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
|
| 111 |
+
}
|
| 112 |
+
function showToast(message) {
|
| 113 |
+
const toast = $("toast");
|
| 114 |
+
toast.textContent = message;
|
| 115 |
+
toast.classList.add("show");
|
| 116 |
+
if (toastTimer) clearTimeout(toastTimer);
|
| 117 |
+
toastTimer = setTimeout(() => toast.classList.remove("show"), 1800);
|
| 118 |
+
}
|
| 119 |
+
async function copyText(text, message) {
|
| 120 |
+
try {
|
| 121 |
+
await navigator.clipboard.writeText(text);
|
| 122 |
+
showToast(message);
|
| 123 |
+
} catch {
|
| 124 |
+
const area = document.createElement("textarea");
|
| 125 |
+
area.value = text;
|
| 126 |
+
area.style.position = "fixed";
|
| 127 |
+
area.style.opacity = "0";
|
| 128 |
+
document.body.appendChild(area);
|
| 129 |
+
area.select();
|
| 130 |
+
document.execCommand("copy");
|
| 131 |
+
area.remove();
|
| 132 |
+
showToast(message);
|
| 133 |
+
}
|
| 134 |
+
}
|
| 135 |
+
function csvCell(value) {
|
| 136 |
+
const text = String(value ?? "");
|
| 137 |
+
return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
|
| 138 |
+
}
|
| 139 |
+
function tableText(headers, rows, separator = "\t") {
|
| 140 |
+
return [headers, ...rows].map((row) => row.join(separator)).join("\n");
|
| 141 |
+
}
|
| 142 |
+
function downloadCsv(filename, headers, rows) {
|
| 143 |
+
const csv = [headers, ...rows].map((row) => row.map(csvCell).join(",")).join("\n");
|
| 144 |
+
const blob = new Blob([csv], { type: "text/csv;charset=utf-8" });
|
| 145 |
+
const url = URL.createObjectURL(blob);
|
| 146 |
+
const a = document.createElement("a");
|
| 147 |
+
a.href = url;
|
| 148 |
+
a.download = filename;
|
| 149 |
+
a.click();
|
| 150 |
+
URL.revokeObjectURL(url);
|
| 151 |
+
}
|
| 152 |
+
function downloadJson(result) {
|
| 153 |
+
const cfg = result.config || configFromUI();
|
| 154 |
+
const topology = cfg.topology === "disaggregated_pd" ? `pd-${cfg.prefill_workers}p-${cfg.decode_workers}d` : `colocated-${cfg.accelerator}`;
|
| 155 |
+
const name = `inferscale-run_${slug(cfg.model)}_${slug(topology)}_${slug(cfg.scheduler)}_${stamp()}.json`;
|
| 156 |
+
const blob = new Blob([JSON.stringify(result, null, 2)], { type: "application/json" });
|
| 157 |
+
const url = URL.createObjectURL(blob);
|
| 158 |
+
const a = document.createElement("a");
|
| 159 |
+
a.href = url;
|
| 160 |
+
a.download = name;
|
| 161 |
+
a.click();
|
| 162 |
+
URL.revokeObjectURL(url);
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
Chart.defaults.color = "#929dab";
|
| 166 |
+
Chart.defaults.borderColor = COLORS.grid;
|
| 167 |
+
Chart.defaults.font.family = getComputedStyle(document.body).fontFamily;
|
| 168 |
+
Chart.defaults.animation.duration = 160;
|
| 169 |
+
|
| 170 |
+
function commonChartOptions() {
|
| 171 |
+
return { responsive: true, maintainAspectRatio: false };
|
| 172 |
+
}
|
| 173 |
+
function lineLegend() {
|
| 174 |
+
return {
|
| 175 |
+
labels: {
|
| 176 |
+
usePointStyle: true,
|
| 177 |
+
pointStyle: "line",
|
| 178 |
+
pointStyleWidth: 26,
|
| 179 |
+
boxWidth: 26,
|
| 180 |
+
boxHeight: 2,
|
| 181 |
+
padding: 14,
|
| 182 |
+
},
|
| 183 |
+
};
|
| 184 |
+
}
|
| 185 |
+
function pointLegend() {
|
| 186 |
+
return {
|
| 187 |
+
labels: {
|
| 188 |
+
usePointStyle: true,
|
| 189 |
+
pointStyle: "circle",
|
| 190 |
+
boxWidth: 8,
|
| 191 |
+
boxHeight: 8,
|
| 192 |
+
padding: 14,
|
| 193 |
+
},
|
| 194 |
+
};
|
| 195 |
+
}
|
| 196 |
+
function destroyChart(name) {
|
| 197 |
+
if (charts[name]) {
|
| 198 |
+
charts[name].destroy();
|
| 199 |
+
delete charts[name];
|
| 200 |
+
}
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
function chartFileName(card) {
|
| 204 |
+
const cfg = configFromUI();
|
| 205 |
+
const chartName = card.dataset.chartName || "figure";
|
| 206 |
+
const topology = cfg.topology === "disaggregated_pd"
|
| 207 |
+
? `pd-${cfg.prefill_workers}p-${cfg.decode_workers}d-${cfg.prefill_accelerator}-${cfg.decode_accelerator}`
|
| 208 |
+
: `colocated-${cfg.accelerator}`;
|
| 209 |
+
return `inferscale_${slug(chartName)}_${slug(cfg.model)}_${slug(topology)}_${slug(cfg.scheduler)}_${stamp()}.png`;
|
| 210 |
+
}
|
| 211 |
+
function downloadChart(card) {
|
| 212 |
+
const canvas = card.querySelector("canvas");
|
| 213 |
+
const chart = Chart.getChart(canvas);
|
| 214 |
+
if (!chart) return;
|
| 215 |
+
const exportCanvas = document.createElement("canvas");
|
| 216 |
+
exportCanvas.width = canvas.width;
|
| 217 |
+
exportCanvas.height = canvas.height;
|
| 218 |
+
const ctx = exportCanvas.getContext("2d");
|
| 219 |
+
ctx.fillStyle = "#0a0f15";
|
| 220 |
+
ctx.fillRect(0, 0, exportCanvas.width, exportCanvas.height);
|
| 221 |
+
ctx.drawImage(canvas, 0, 0);
|
| 222 |
+
exportCanvas.toBlob((blob) => {
|
| 223 |
+
if (!blob) return;
|
| 224 |
+
const url = URL.createObjectURL(blob);
|
| 225 |
+
const a = document.createElement("a");
|
| 226 |
+
a.href = url;
|
| 227 |
+
a.download = chartFileName(card);
|
| 228 |
+
a.click();
|
| 229 |
+
URL.revokeObjectURL(url);
|
| 230 |
+
showToast("Chart PNG downloaded");
|
| 231 |
+
}, "image/png");
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
function renderSimulation(result) {
|
| 235 |
+
lastResult = result;
|
| 236 |
+
$("emptyState").classList.add("hidden");
|
| 237 |
+
$("resultContent").classList.remove("hidden");
|
| 238 |
+
$("exportBtn").disabled = false;
|
| 239 |
+
$("copyResultBtn").disabled = false;
|
| 240 |
+
|
| 241 |
+
const s = result.summary;
|
| 242 |
+
const l = result.latency;
|
| 243 |
+
const r = result.resource;
|
| 244 |
+
const d = result.diagnostics || {};
|
| 245 |
+
$("mTtft").textContent = `${fmt(l.ttft_ms.p95)} ms`;
|
| 246 |
+
$("mE2e").textContent = `${fmt(l.e2e_ms.p95)} ms`;
|
| 247 |
+
$("mGoodput").textContent = `${fmt(s.goodput_rps, 2)} req/s`;
|
| 248 |
+
$("mSlo").textContent = pct(s.slo_attainment);
|
| 249 |
+
$("mReq").textContent = `${fmt(s.request_throughput_rps, 2)} req/s`;
|
| 250 |
+
$("mKv").textContent = `${fmt(r.peak_kv_gb, 2)} GB`;
|
| 251 |
+
|
| 252 |
+
const tag = $("runState");
|
| 253 |
+
tag.textContent = `${s.requests_completed}/${s.requests_generated} completed`;
|
| 254 |
+
tag.className = `tag ${s.slo_attainment >= .99 && s.requests_unfinished === 0 ? "good" : "bad"}`;
|
| 255 |
+
|
| 256 |
+
$("mBottleneck").textContent = d.label || "N/A";
|
| 257 |
+
$("mDiagnosis").textContent = d.explanation || "No simulator diagnosis available.";
|
| 258 |
+
$("mRecommendation").textContent = d.recommendation ? `Next check: ${d.recommendation}` : "";
|
| 259 |
+
const e = d.evidence || {};
|
| 260 |
+
const evidence = [
|
| 261 |
+
`topology ${r.topology || result.config.topology || "colocated"}`,
|
| 262 |
+
`busy ${pct(e.busy_fraction ?? s.busy_fraction)}`,
|
| 263 |
+
`KV ${pct(e.peak_kv_utilization ?? r.peak_kv_utilization)}`,
|
| 264 |
+
`TTFT pass ${pct(e.ttft_slo_attainment ?? s.ttft_slo_attainment)}`,
|
| 265 |
+
`E2E pass ${pct(e.e2e_slo_attainment ?? s.e2e_slo_attainment)}`,
|
| 266 |
+
`queue p95 ${fmt(e.queue_p95_ms ?? l.queue_ms.p95)} ms`,
|
| 267 |
+
];
|
| 268 |
+
if ((r.prefix_cache_hit_rate ?? 0) > 0) evidence.push(`cache hit ${pct(r.prefix_cache_hit_rate)}`, `prefill saved ${fmt(r.prefill_tokens_saved, 0)} tok`);
|
| 269 |
+
if (r.topology === "disaggregated_pd") evidence.push(
|
| 270 |
+
`prefill busy ${pct(r.prefill_busy_fraction)}`,
|
| 271 |
+
`decode busy ${pct(r.decode_busy_fraction)}`,
|
| 272 |
+
`link busy ${pct(r.transfer_busy_fraction)}`,
|
| 273 |
+
`transfer p95 ${fmt(r.p95_transfer_ms, 2)} ms`,
|
| 274 |
+
);
|
| 275 |
+
$("mEvidence").innerHTML = evidence.map((x) => `<span>${escapeHtml(x)}</span>`).join("");
|
| 276 |
+
|
| 277 |
+
destroyChart("latency");
|
| 278 |
+
charts.latency = new Chart($("latencyChart"), {
|
| 279 |
+
type: "bar",
|
| 280 |
+
data: {
|
| 281 |
+
labels: ["TTFT p50", "TTFT p95", "E2E p50", "E2E p95", "Queue p95"],
|
| 282 |
+
datasets: [{
|
| 283 |
+
label: "Milliseconds",
|
| 284 |
+
data: [l.ttft_ms.p50, l.ttft_ms.p95, l.e2e_ms.p50, l.e2e_ms.p95, l.queue_ms.p95],
|
| 285 |
+
backgroundColor: [COLORS.steel, COLORS.blue, "#8196b1", COLORS.blue2, COLORS.amber],
|
| 286 |
+
}],
|
| 287 |
+
},
|
| 288 |
+
options: { ...commonChartOptions(), plugins: { legend: { display: false } }, scales: { y: { beginAtZero: true } } },
|
| 289 |
+
});
|
| 290 |
+
|
| 291 |
+
const timeline = result.timeline || [];
|
| 292 |
+
const lineDatasets = [
|
| 293 |
+
{ label: r.topology === "disaggregated_pd" ? "Prefill queue" : "Waiting", data: timeline.map((x) => ({ x: x.time_s, y: x.waiting })), borderColor: COLORS.amber, pointRadius: 0, tension: .08, yAxisID: "y" },
|
| 294 |
+
{ label: "Decoding", data: timeline.map((x) => ({ x: x.time_s, y: x.decoding })), borderColor: COLORS.blue, pointRadius: 0, tension: .08, yAxisID: "y" },
|
| 295 |
+
{ label: "KV GB", data: timeline.map((x) => ({ x: x.time_s, y: x.kv_used_gb })), borderColor: COLORS.green, pointRadius: 0, tension: .08, yAxisID: "y1" },
|
| 296 |
+
];
|
| 297 |
+
if (r.topology === "disaggregated_pd") {
|
| 298 |
+
lineDatasets.splice(1, 0, { label: "Decode queue", data: timeline.map((x) => ({ x: x.time_s, y: x.decode_ready || 0 })), borderColor: COLORS.steel, borderDash: [4, 3], pointRadius: 0, tension: .08, yAxisID: "y" });
|
| 299 |
+
}
|
| 300 |
+
destroyChart("timeline");
|
| 301 |
+
charts.timeline = new Chart($("timelineChart"), {
|
| 302 |
+
type: "line",
|
| 303 |
+
data: { datasets: lineDatasets },
|
| 304 |
+
options: {
|
| 305 |
+
...commonChartOptions(),
|
| 306 |
+
parsing: false,
|
| 307 |
+
interaction: { mode: "nearest", intersect: false },
|
| 308 |
+
plugins: { legend: lineLegend() },
|
| 309 |
+
scales: {
|
| 310 |
+
x: { type: "linear", title: { display: true, text: "Virtual time (s)" } },
|
| 311 |
+
y: { beginAtZero: true, title: { display: true, text: "Requests" } },
|
| 312 |
+
y1: { beginAtZero: true, position: "right", grid: { drawOnChartArea: false }, title: { display: true, text: "KV GB" } },
|
| 313 |
+
},
|
| 314 |
+
},
|
| 315 |
+
});
|
| 316 |
+
|
| 317 |
+
const sample = result.requests || [];
|
| 318 |
+
destroyChart("scatter");
|
| 319 |
+
charts.scatter = new Chart($("scatterChart"), {
|
| 320 |
+
type: "scatter",
|
| 321 |
+
data: { datasets: [{ label: "Requests", data: sample.map((x) => ({ x: x.prompt_tokens, y: x.ttft_ms })), backgroundColor: "rgba(121,167,255,.55)", pointRadius: 2.2 }] },
|
| 322 |
+
options: { ...commonChartOptions(), plugins: { legend: { display: false } }, scales: { x: { title: { display: true, text: "Prompt tokens" } }, y: { title: { display: true, text: "TTFT (ms)" }, beginAtZero: true } } },
|
| 323 |
+
});
|
| 324 |
+
|
| 325 |
+
const warnings = $("warnings");
|
| 326 |
+
const allWarnings = [...(result.warnings || [])];
|
| 327 |
+
if (result.provenance?.profile_warning) allWarnings.unshift(result.provenance.profile_warning);
|
| 328 |
+
if (allWarnings.length) {
|
| 329 |
+
warnings.innerHTML = allWarnings.map((w) => `<div>${escapeHtml(w)}</div>`).join("");
|
| 330 |
+
warnings.classList.remove("hidden");
|
| 331 |
+
} else {
|
| 332 |
+
warnings.classList.add("hidden");
|
| 333 |
+
}
|
| 334 |
+
}
|
| 335 |
+
|
| 336 |
+
$("runBtn").addEventListener("click", async () => {
|
| 337 |
+
const button = $("runBtn");
|
| 338 |
+
const state = $("runState");
|
| 339 |
+
button.disabled = true;
|
| 340 |
+
button.textContent = "Running simulation...";
|
| 341 |
+
state.textContent = "Simulating...";
|
| 342 |
+
state.className = "tag neutral";
|
| 343 |
+
try {
|
| 344 |
+
renderSimulation(await callPython("simulate", configFromUI()));
|
| 345 |
+
} catch (error) {
|
| 346 |
+
state.textContent = "Error";
|
| 347 |
+
state.className = "tag bad";
|
| 348 |
+
alert(`Simulation failed: ${error.message}`);
|
| 349 |
+
} finally {
|
| 350 |
+
button.textContent = "Run simulation";
|
| 351 |
+
button.disabled = false;
|
| 352 |
+
}
|
| 353 |
+
});
|
| 354 |
+
$("copyResultBtn").addEventListener("click", () => { if (lastResult) copyText(JSON.stringify(lastResult, null, 2), "Result JSON copied"); });
|
| 355 |
+
$("exportBtn").addEventListener("click", () => { if (lastResult) downloadJson(lastResult); });
|
| 356 |
+
|
| 357 |
+
function schedulerLabel(value) {
|
| 358 |
+
return ({
|
| 359 |
+
static_fcfs: "Static FCFS",
|
| 360 |
+
continuous_fcfs: "Continuous FCFS",
|
| 361 |
+
continuous_sjf: "Continuous SJF",
|
| 362 |
+
continuous_slo: "Continuous SLO",
|
| 363 |
+
chunked_slo: "Chunked SLO",
|
| 364 |
+
})[value] || value;
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
function arenaTableRows(rows) {
|
| 368 |
+
return rows.map((r) => [schedulerLabel(r.scheduler), `${fmt(r.goodput_rps, 2)} req/s`, pct(r.slo_attainment), `${fmt(r.p95_ttft_ms)} ms`, `${fmt(r.p95_e2e_ms)} ms`, pct(r.peak_kv_utilization), fmt(r.unfinished, 0), r.bottleneck || "N/A"]);
|
| 369 |
+
}
|
| 370 |
+
function renderArena(rows) {
|
| 371 |
+
lastArenaRows = rows;
|
| 372 |
+
$("arenaEmpty").classList.add("hidden");
|
| 373 |
+
$("arenaContent").classList.remove("hidden");
|
| 374 |
+
$("arenaCopyBtn").disabled = false;
|
| 375 |
+
$("arenaCsvBtn").disabled = false;
|
| 376 |
+
$("arenaRows").innerHTML = rows.map((r, index) => `<tr><td>${escapeHtml(schedulerLabel(r.scheduler))}${index === 0 ? '<span class="best-label">Best</span>' : ""}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.goodput_per_accelerator, 2)} req/s/GPU</td><td>${pct(r.slo_attainment)}</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${pct(r.peak_kv_utilization)}</td><td>${fmt(r.unfinished, 0)}</td><td>${escapeHtml(r.bottleneck || "N/A")}</td></tr>`).join("");
|
| 377 |
+
destroyChart("arena");
|
| 378 |
+
charts.arena = new Chart($("arenaChart"), {
|
| 379 |
+
type: "bar",
|
| 380 |
+
data: { labels: rows.map((r) => schedulerLabel(r.scheduler)), datasets: [{ label: "Goodput (req/s)", data: rows.map((r) => r.goodput_rps), backgroundColor: COLORS.blue }, { label: "Raw throughput (req/s)", data: rows.map((r) => r.request_throughput_rps), backgroundColor: "#53677f" }] },
|
| 381 |
+
options: { ...commonChartOptions(), scales: { y: { beginAtZero: true } } },
|
| 382 |
+
});
|
| 383 |
+
}
|
| 384 |
+
$("arenaBtn").addEventListener("click", async () => {
|
| 385 |
+
const button = $("arenaBtn");
|
| 386 |
+
button.disabled = true;
|
| 387 |
+
button.textContent = "Comparing...";
|
| 388 |
+
try { renderArena((await callPython("compare", { config: configFromUI() })).rows); }
|
| 389 |
+
catch (error) { alert(`Scheduler comparison failed: ${error.message}`); }
|
| 390 |
+
finally { button.disabled = false; button.textContent = "Compare schedulers"; }
|
| 391 |
+
});
|
| 392 |
+
const arenaHeaders = ["Scheduler", "Goodput", "SLO attainment", "p95 TTFT", "p95 E2E", "KV peak", "Unfinished", "Diagnosis"];
|
| 393 |
+
$("arenaCopyBtn").addEventListener("click", () => copyText(tableText(arenaHeaders, arenaTableRows(lastArenaRows)), "Scheduler table copied"));
|
| 394 |
+
$("arenaCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_scheduler-arena_${stamp()}.csv`, arenaHeaders, arenaTableRows(lastArenaRows)));
|
| 395 |
+
|
| 396 |
+
function capacityTableRows(trace) {
|
| 397 |
+
return trace.map((r) => [
|
| 398 |
+
`${fmt(r.rate_rps, 2)} req/s`,
|
| 399 |
+
r.passed ? "PASS" : "FAIL",
|
| 400 |
+
pct(r.slo_attainment),
|
| 401 |
+
pct(r.slo_attainment_min),
|
| 402 |
+
pct(r.target ?? num("targetSlo")),
|
| 403 |
+
`${pct(r.slo_attainment_min)} - ${pct(r.slo_attainment_max)}`,
|
| 404 |
+
`${fmt(r.goodput_rps, 2)} req/s`,
|
| 405 |
+
`${fmt(r.p95_ttft_ms)} ms`,
|
| 406 |
+
`${fmt(r.p95_e2e_ms)} ms`,
|
| 407 |
+
]);
|
| 408 |
+
}
|
| 409 |
+
function renderCapacity(result) {
|
| 410 |
+
lastCapacityTrace = result.trace || [];
|
| 411 |
+
$("plannerEmpty").classList.add("hidden");
|
| 412 |
+
$("plannerContent").classList.remove("hidden");
|
| 413 |
+
$("capacityCopyBtn").disabled = false;
|
| 414 |
+
$("capacityCsvBtn").disabled = false;
|
| 415 |
+
$("pCapacity").textContent = `${fmt(result.capacity_rps, 2)} req/s`;
|
| 416 |
+
$("pRecommended").textContent = `${fmt(result.recommended_rps, 2)} req/s`;
|
| 417 |
+
$("pHeadroom").textContent = pct(result.headroom ?? num("headroom"));
|
| 418 |
+
$("pStatus").textContent = result.status.replaceAll("_", " ");
|
| 419 |
+
const state = $("plannerState");
|
| 420 |
+
state.textContent = result.status === "ok" ? "Search complete" : result.status.replaceAll("_", " ");
|
| 421 |
+
state.className = `tag ${result.capacity_rps > 0 ? "good" : "bad"}`;
|
| 422 |
+
|
| 423 |
+
$("capacityRows").innerHTML = lastCapacityTrace.map((r) => `<tr><td>${fmt(r.rate_rps, 2)} req/s</td><td class="${r.passed ? "pass" : "fail"}">${r.passed ? "PASS" : "FAIL"}</td><td>${pct(r.slo_attainment)}</td><td>${pct(r.slo_attainment_min)}</td><td>${pct(r.target ?? num("targetSlo"))}</td><td>${pct(r.slo_attainment_min)} - ${pct(r.slo_attainment_max)}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td></tr>`).join("");
|
| 424 |
+
|
| 425 |
+
if (!lastCapacityTrace.length) return;
|
| 426 |
+
const target = num("targetSlo");
|
| 427 |
+
const mean = lastCapacityTrace.map((r) => ({ x: r.rate_rps, y: r.slo_attainment }));
|
| 428 |
+
const worst = lastCapacityTrace.map((r) => ({ x: r.rate_rps, y: r.slo_attainment_min }));
|
| 429 |
+
const xs = lastCapacityTrace.map((r) => r.rate_rps);
|
| 430 |
+
const minX = Math.min(...xs);
|
| 431 |
+
const maxX = Math.max(...xs);
|
| 432 |
+
destroyChart("capacity");
|
| 433 |
+
charts.capacity = new Chart($("capacityChart"), {
|
| 434 |
+
type: "line",
|
| 435 |
+
data: { datasets: [
|
| 436 |
+
{ label: "Mean SLO attainment", data: mean, borderColor: COLORS.blue, backgroundColor: COLORS.blue, fill: false, tension: .06, pointRadius: 3 },
|
| 437 |
+
{ label: "Worst repetition", data: worst, borderColor: COLORS.amber, backgroundColor: COLORS.amber, borderDash: [5, 4], fill: false, pointRadius: 2, tension: .06 },
|
| 438 |
+
{ label: "Target", data: [{ x: minX, y: target }, { x: maxX, y: target }], borderColor: COLORS.green, backgroundColor: COLORS.green, borderDash: [6, 5], fill: false, pointRadius: 0 },
|
| 439 |
+
] },
|
| 440 |
+
options: { ...commonChartOptions(), parsing: false, plugins: { legend: lineLegend() }, scales: { x: { type: "linear", title: { display: true, text: "Offered load (req/s)" }, ticks: { maxTicksLimit: 8 } }, y: { min: 0, max: 1, ticks: { callback: (v) => `${Math.round(v * 100)}%` } } } },
|
| 441 |
+
});
|
| 442 |
+
}
|
| 443 |
+
$("capacityBtn").addEventListener("click", async () => {
|
| 444 |
+
const button = $("capacityBtn");
|
| 445 |
+
const state = $("plannerState");
|
| 446 |
+
button.disabled = true;
|
| 447 |
+
button.textContent = "Searching...";
|
| 448 |
+
state.textContent = "Running simulations...";
|
| 449 |
+
state.className = "tag neutral";
|
| 450 |
+
try {
|
| 451 |
+
renderCapacity(await callPython("capacity", {
|
| 452 |
+
config: configFromUI({ slo_attainment_target: num("targetSlo") }),
|
| 453 |
+
min_rate: num("minRate"),
|
| 454 |
+
max_rate: num("maxRate"),
|
| 455 |
+
iterations: num("searchIter"),
|
| 456 |
+
repetitions: num("repetitions"),
|
| 457 |
+
headroom: num("headroom"),
|
| 458 |
+
}));
|
| 459 |
+
} catch (error) {
|
| 460 |
+
state.textContent = "Error";
|
| 461 |
+
state.className = "tag bad";
|
| 462 |
+
alert(`Capacity search failed: ${error.message}`);
|
| 463 |
+
} finally {
|
| 464 |
+
button.disabled = false;
|
| 465 |
+
button.textContent = "Find sustainable capacity";
|
| 466 |
+
}
|
| 467 |
+
});
|
| 468 |
+
const capacityHeaders = ["Rate", "Pass", "Mean SLO", "Worst repetition", "Target", "SLO range", "Goodput", "p95 TTFT", "p95 E2E"];
|
| 469 |
+
$("capacityCopyBtn").addEventListener("click", () => copyText(tableText(capacityHeaders, capacityTableRows(lastCapacityTrace)), "Capacity table copied"));
|
| 470 |
+
$("capacityCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_capacity-trace_${stamp()}.csv`, capacityHeaders, capacityTableRows(lastCapacityTrace)));
|
| 471 |
+
|
| 472 |
+
function scenarioLabel(value) {
|
| 473 |
+
return ({
|
| 474 |
+
colocated: "Colocated",
|
| 475 |
+
colocated_cache: "Colocated + prefix cache",
|
| 476 |
+
disaggregated_pd: "P/D disaggregated",
|
| 477 |
+
disaggregated_pd_cache: "P/D + prefix cache",
|
| 478 |
+
})[value] || value;
|
| 479 |
+
}
|
| 480 |
+
function topologyTableRows(rows) {
|
| 481 |
+
return rows.map((r) => [
|
| 482 |
+
scenarioLabel(r.scenario),
|
| 483 |
+
fmt(r.accelerator_instances, 0),
|
| 484 |
+
`${fmt(r.goodput_rps, 2)} req/s`,
|
| 485 |
+
`${fmt(r.goodput_per_accelerator, 2)} req/s/GPU`,
|
| 486 |
+
pct(r.slo_attainment),
|
| 487 |
+
`${fmt(r.p95_ttft_ms)} ms`,
|
| 488 |
+
`${fmt(r.p95_e2e_ms)} ms`,
|
| 489 |
+
`${fmt(r.p95_transfer_ms, 2)} ms`,
|
| 490 |
+
pct(r.prefix_hit_rate),
|
| 491 |
+
`${fmt(r.prefill_tokens_saved, 0)} tok`,
|
| 492 |
+
r.bottleneck || "N/A",
|
| 493 |
+
]);
|
| 494 |
+
}
|
| 495 |
+
function renderTopology(rows) {
|
| 496 |
+
lastTopologyRows = rows;
|
| 497 |
+
$("topologyEmpty").classList.add("hidden");
|
| 498 |
+
$("topologyContent").classList.remove("hidden");
|
| 499 |
+
$("topologyCopyBtn").disabled = false;
|
| 500 |
+
$("topologyCsvBtn").disabled = false;
|
| 501 |
+
$("topologyRows").innerHTML = rows.map((r, index) => `<tr><td>${escapeHtml(scenarioLabel(r.scenario))}${index === 0 ? '<span class="best-label">Best</span>' : ""}</td><td>${fmt(r.accelerator_instances, 0)}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.goodput_per_accelerator, 2)} req/s/GPU</td><td>${pct(r.slo_attainment)}</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${fmt(r.p95_transfer_ms, 2)} ms</td><td>${pct(r.prefix_hit_rate)}</td><td>${fmt(r.prefill_tokens_saved, 0)} tok</td><td>${escapeHtml(r.bottleneck || "N/A")}</td></tr>`).join("");
|
| 502 |
+
|
| 503 |
+
const palette = [COLORS.blue, COLORS.green, COLORS.amber, COLORS.red];
|
| 504 |
+
destroyChart("topology");
|
| 505 |
+
charts.topology = new Chart($("topologyChart"), {
|
| 506 |
+
type: "scatter",
|
| 507 |
+
data: { datasets: rows.map((r, i) => ({ label: scenarioLabel(r.scenario), data: [{ x: r.p95_ttft_ms, y: r.goodput_rps }], backgroundColor: palette[i % palette.length], borderColor: palette[i % palette.length], pointRadius: 6, pointHoverRadius: 8 })) },
|
| 508 |
+
options: { ...commonChartOptions(), plugins: { legend: pointLegend() }, scales: { x: { title: { display: true, text: "p95 TTFT (ms)" }, beginAtZero: true }, y: { title: { display: true, text: "Goodput (req/s)" }, beginAtZero: true } } },
|
| 509 |
+
});
|
| 510 |
+
}
|
| 511 |
+
$("topologyBtn").addEventListener("click", async () => {
|
| 512 |
+
const button = $("topologyBtn");
|
| 513 |
+
button.disabled = true;
|
| 514 |
+
button.textContent = "Comparing...";
|
| 515 |
+
try { renderTopology((await callPython("topology_compare", { config: configFromUI() })).rows); }
|
| 516 |
+
catch (error) { alert(`Topology comparison failed: ${error.message}`); }
|
| 517 |
+
finally { button.disabled = false; button.textContent = "Compare 4 scenarios"; }
|
| 518 |
+
});
|
| 519 |
+
const topologyHeaders = ["Scenario", "GPU instances", "Goodput", "Goodput / GPU", "SLO attainment", "p95 TTFT", "p95 E2E", "p95 KV transfer", "Cache hit", "Prefill saved", "Diagnosis"];
|
| 520 |
+
$("topologyCopyBtn").addEventListener("click", () => copyText(tableText(topologyHeaders, topologyTableRows(lastTopologyRows)), "Topology table copied"));
|
| 521 |
+
$("topologyCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_topology-cache-study_${stamp()}.csv`, topologyHeaders, topologyTableRows(lastTopologyRows)));
|
| 522 |
+
|
| 523 |
+
function designTableRows(rows) {
|
| 524 |
+
return rows.map((r) => [
|
| 525 |
+
r.label,
|
| 526 |
+
r.pareto ? "YES" : "NO",
|
| 527 |
+
r.efficiency_pareto ? "YES" : "NO",
|
| 528 |
+
r.slo_pass ? "PASS" : "FAIL",
|
| 529 |
+
fmt(r.accelerator_instances, 0),
|
| 530 |
+
`${fmt(r.goodput_rps, 2)} req/s`,
|
| 531 |
+
`${fmt(r.goodput_per_accelerator, 2)} req/s/GPU`,
|
| 532 |
+
`${fmt(r.p95_ttft_ms)} ms`,
|
| 533 |
+
`${fmt(r.p95_e2e_ms)} ms`,
|
| 534 |
+
pct(r.peak_kv_utilization),
|
| 535 |
+
r.bottleneck || "N/A",
|
| 536 |
+
]);
|
| 537 |
+
}
|
| 538 |
+
function renderDesign(result) {
|
| 539 |
+
lastDesignRows = result.rows || [];
|
| 540 |
+
$("designEmpty").classList.add("hidden");
|
| 541 |
+
$("designContent").classList.remove("hidden");
|
| 542 |
+
$("designCopyBtn").disabled = false;
|
| 543 |
+
$("designCsvBtn").disabled = false;
|
| 544 |
+
$("dCandidates").textContent = fmt(result.candidate_count, 0);
|
| 545 |
+
$("dPareto").textContent = fmt(result.pareto_count, 0);
|
| 546 |
+
$("dEfficiencyPareto").textContent = fmt(result.efficiency_pareto_count, 0);
|
| 547 |
+
$("designRows").innerHTML = lastDesignRows.map((r) => `<tr><td>${escapeHtml(r.label)}${r.pareto ? '<span class="pareto-label">Perf</span>' : ""}${r.efficiency_pareto ? '<span class="pareto-label">Eff</span>' : ""}</td><td class="${r.pareto ? "pass" : ""}">${r.pareto ? "YES" : "NO"}</td><td class="${r.efficiency_pareto ? "pass" : ""}">${r.efficiency_pareto ? "YES" : "NO"}</td><td class="${r.slo_pass ? "pass" : "fail"}">${r.slo_pass ? "PASS" : "FAIL"}</td><td>${fmt(r.accelerator_instances, 0)}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.goodput_per_accelerator, 2)} req/s/GPU</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${pct(r.peak_kv_utilization)}</td><td>${escapeHtml(r.bottleneck || "N/A")}</td></tr>`).join("");
|
| 548 |
+
|
| 549 |
+
const passPoints = lastDesignRows.filter((r) => r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_rps }));
|
| 550 |
+
const failPoints = lastDesignRows.filter((r) => !r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_rps }));
|
| 551 |
+
const frontier = lastDesignRows.filter((r) => r.pareto).sort((a, b) => a.p95_ttft_ms - b.p95_ttft_ms).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_rps }));
|
| 552 |
+
destroyChart("design");
|
| 553 |
+
charts.design = new Chart($("designChart"), {
|
| 554 |
+
type: "scatter",
|
| 555 |
+
data: { datasets: [
|
| 556 |
+
{ label: "SLO pass", data: passPoints, backgroundColor: COLORS.green, borderColor: COLORS.green, pointRadius: 4 },
|
| 557 |
+
{ label: "SLO fail", data: failPoints, backgroundColor: COLORS.gray, borderColor: COLORS.gray, pointRadius: 4 },
|
| 558 |
+
{ type: "line", label: "Performance frontier", data: frontier, borderColor: COLORS.blue, backgroundColor: COLORS.blue, pointBackgroundColor: COLORS.blue, pointRadius: 5, fill: false, tension: 0 },
|
| 559 |
+
] },
|
| 560 |
+
options: { ...commonChartOptions(), parsing: false, plugins: { legend: pointLegend() }, scales: { x: { title: { display: true, text: "p95 TTFT (ms)" }, beginAtZero: true }, y: { title: { display: true, text: "Goodput (req/s)" }, beginAtZero: true } } },
|
| 561 |
+
});
|
| 562 |
+
|
| 563 |
+
const efficiencyPass = lastDesignRows.filter((r) => r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_per_accelerator }));
|
| 564 |
+
const efficiencyFail = lastDesignRows.filter((r) => !r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_per_accelerator }));
|
| 565 |
+
const efficiencyFrontier = lastDesignRows.filter((r) => r.efficiency_pareto).sort((a, b) => a.p95_ttft_ms - b.p95_ttft_ms).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_per_accelerator }));
|
| 566 |
+
destroyChart("designEfficiency");
|
| 567 |
+
charts.designEfficiency = new Chart($("designEfficiencyChart"), {
|
| 568 |
+
type: "scatter",
|
| 569 |
+
data: { datasets: [
|
| 570 |
+
{ label: "SLO pass", data: efficiencyPass, backgroundColor: COLORS.green, borderColor: COLORS.green, pointRadius: 4 },
|
| 571 |
+
{ label: "SLO fail", data: efficiencyFail, backgroundColor: COLORS.gray, borderColor: COLORS.gray, pointRadius: 4 },
|
| 572 |
+
{ type: "line", label: "Efficiency frontier", data: efficiencyFrontier, borderColor: COLORS.amber, backgroundColor: COLORS.amber, pointBackgroundColor: COLORS.amber, pointRadius: 5, fill: false, tension: 0 },
|
| 573 |
+
] },
|
| 574 |
+
options: { ...commonChartOptions(), parsing: false, plugins: { legend: pointLegend() }, scales: { x: { title: { display: true, text: "p95 TTFT (ms)" }, beginAtZero: true }, y: { title: { display: true, text: "Goodput / accelerator (req/s/GPU)" }, beginAtZero: true } } },
|
| 575 |
+
});
|
| 576 |
+
}
|
| 577 |
+
$("designBtn").addEventListener("click", async () => {
|
| 578 |
+
const button = $("designBtn");
|
| 579 |
+
button.disabled = true;
|
| 580 |
+
button.textContent = "Exploring...";
|
| 581 |
+
try { renderDesign(await callPython("design_space", { config: configFromUI(), include_disaggregated: $("includePd").checked })); }
|
| 582 |
+
catch (error) { alert(`Design-space exploration failed: ${error.message}`); }
|
| 583 |
+
finally { button.disabled = false; button.textContent = "Explore design space"; }
|
| 584 |
+
});
|
| 585 |
+
const designHeaders = ["Candidate", "Perf Pareto", "Efficiency Pareto", "SLO pass", "GPU instances", "Goodput", "Goodput / GPU", "p95 TTFT", "p95 E2E", "KV peak", "Diagnosis"];
|
| 586 |
+
$("designCopyBtn").addEventListener("click", () => copyText(tableText(designHeaders, designTableRows(lastDesignRows)), "Design table copied"));
|
| 587 |
+
$("designCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_design-space_${stamp()}.csv`, designHeaders, designTableRows(lastDesignRows)));
|
| 588 |
+
|
| 589 |
+
function syncConditionalControls() {
|
| 590 |
+
const pd = $("topology").value === "disaggregated_pd";
|
| 591 |
+
$("pdControls").classList.toggle("hidden", !pd);
|
| 592 |
+
$("colocatedAcceleratorLabel").classList.toggle("hidden", pd);
|
| 593 |
+
if (pd && $("scheduler").value === "static_fcfs") {
|
| 594 |
+
$("scheduler").value = "continuous_fcfs";
|
| 595 |
+
showToast("Static FCFS switched to Continuous FCFS for P/D topology");
|
| 596 |
+
}
|
| 597 |
+
$("prefixControls").classList.toggle("hidden", !boolSelect("prefixCache"));
|
| 598 |
+
$("burstControls").classList.toggle("hidden", $("arrival").value !== "bursty");
|
| 599 |
+
}
|
| 600 |
+
$("topology").addEventListener("change", syncConditionalControls);
|
| 601 |
+
$("prefixCache").addEventListener("change", syncConditionalControls);
|
| 602 |
+
$("arrival").addEventListener("change", syncConditionalControls);
|
| 603 |
+
syncConditionalControls();
|
| 604 |
+
|
| 605 |
+
for (const tab of document.querySelectorAll(".tab")) {
|
| 606 |
+
tab.addEventListener("click", () => {
|
| 607 |
+
document.querySelectorAll(".tab").forEach((item) => item.classList.remove("active"));
|
| 608 |
+
document.querySelectorAll(".tab-panel").forEach((panel) => panel.classList.remove("active"));
|
| 609 |
+
tab.classList.add("active");
|
| 610 |
+
$(tab.dataset.tab).classList.add("active");
|
| 611 |
+
setTimeout(() => Object.values(charts).forEach((chart) => chart.resize()), 20);
|
| 612 |
+
});
|
| 613 |
+
}
|
| 614 |
+
|
| 615 |
+
function closeExpandedChart() {
|
| 616 |
+
const card = document.querySelector(".chart-card.chart-expanded");
|
| 617 |
+
if (!card) return;
|
| 618 |
+
card.classList.remove("chart-expanded");
|
| 619 |
+
const button = card.querySelector(".chart-expand");
|
| 620 |
+
if (button) button.textContent = "Expand";
|
| 621 |
+
document.body.classList.remove("chart-open");
|
| 622 |
+
setTimeout(() => Chart.getChart(card.querySelector("canvas"))?.resize(), 20);
|
| 623 |
+
}
|
| 624 |
+
for (const button of document.querySelectorAll(".chart-expand")) {
|
| 625 |
+
button.addEventListener("click", () => {
|
| 626 |
+
const card = button.closest(".chart-card");
|
| 627 |
+
const wasExpanded = card.classList.contains("chart-expanded");
|
| 628 |
+
closeExpandedChart();
|
| 629 |
+
if (!wasExpanded) {
|
| 630 |
+
card.classList.add("chart-expanded");
|
| 631 |
+
button.textContent = "Close";
|
| 632 |
+
document.body.classList.add("chart-open");
|
| 633 |
+
setTimeout(() => Chart.getChart(card.querySelector("canvas"))?.resize(), 20);
|
| 634 |
+
}
|
| 635 |
+
});
|
| 636 |
+
}
|
| 637 |
+
for (const button of document.querySelectorAll(".chart-download")) {
|
| 638 |
+
button.addEventListener("click", () => downloadChart(button.closest(".chart-card")));
|
| 639 |
+
}
|
| 640 |
+
document.addEventListener("keydown", (event) => { if (event.key === "Escape") closeExpandedChart(); });
|
docs/architecture.md
CHANGED
|
@@ -1,31 +1,52 @@
|
|
| 1 |
# Architecture
|
| 2 |
|
| 3 |
-
InferScale-Sim
|
| 4 |
|
| 5 |
-
##
|
| 6 |
|
| 7 |
-
1. `workloads.py`
|
| 8 |
-
2. `simulator.py`
|
| 9 |
-
3. `
|
| 10 |
-
4. `
|
| 11 |
-
5. `
|
| 12 |
-
6. `
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
## Browser execution
|
| 17 |
|
| 18 |
-
|
| 19 |
|
| 20 |
-
The
|
| 21 |
|
| 22 |
## Extension boundary
|
| 23 |
|
| 24 |
-
`AnalyticalLatencyModel`
|
| 25 |
|
| 26 |
- `prefill_seconds(token_counts)`
|
| 27 |
- `decode_step_seconds(context_lengths)`
|
| 28 |
- `model_weight_gb`
|
| 29 |
- `kv_bytes_per_token()`
|
| 30 |
|
| 31 |
-
without
|
|
|
|
| 1 |
# Architecture
|
| 2 |
|
| 3 |
+
InferScale-Sim separates **serving-system logic** from **latency estimation**.
|
| 4 |
|
| 5 |
+
## Main Python modules
|
| 6 |
|
| 7 |
+
1. `workloads.py` creates deterministic workload traces and shared-prefix hit assignments.
|
| 8 |
+
2. `simulator.py` implements the colocated serving loop.
|
| 9 |
+
3. `disaggregated.py` implements separate prefill/decode worker pools plus KV transfer.
|
| 10 |
+
4. `kv_cache.py` handles memory/admission and shared-prefix allocation.
|
| 11 |
+
5. `latency.py` predicts reference prefill/decode operation durations.
|
| 12 |
+
6. `metrics.py` derives request-level TTFT, TPOT, E2E, queueing, throughput, and goodput.
|
| 13 |
+
7. `diagnostics.py` converts simulated telemetry into explicit heuristic bottleneck labels.
|
| 14 |
+
8. `optimizer.py` implements capacity search, scheduler comparison, topology/cache comparison, and the bounded Pareto sweep.
|
| 15 |
+
9. `api.py` exposes JSON-like actions to both local Python and Pyodide.
|
| 16 |
|
| 17 |
+
## Colocated path
|
| 18 |
+
|
| 19 |
+
```text
|
| 20 |
+
arrival -> waiting -> prefill -> active decode batch -> complete
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
## P/D path
|
| 24 |
+
|
| 25 |
+
```text
|
| 26 |
+
arrival
|
| 27 |
+
-> prefill queue
|
| 28 |
+
-> prefill worker batch
|
| 29 |
+
-> KV-transfer link
|
| 30 |
+
-> decode-ready queue
|
| 31 |
+
-> continuous decode worker
|
| 32 |
+
-> complete
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
The P/D path is a genuine discrete-event loop: prefill workers, decode workers, and the transfer link can have overlapping virtual activity.
|
| 36 |
|
| 37 |
## Browser execution
|
| 38 |
|
| 39 |
+
Canonical source lives in `src/inferscale`. `scripts/sync_web_python.py` mirrors it into `py/inferscale`. A module Web Worker loads Pyodide, writes those files into Pyodide's virtual filesystem, imports `inferscale.api`, and exchanges JSON messages with the UI.
|
| 40 |
|
| 41 |
+
The simulator therefore runs away from the browser main thread and uses the same Python implementation as the local tests.
|
| 42 |
|
| 43 |
## Extension boundary
|
| 44 |
|
| 45 |
+
`AnalyticalLatencyModel` can later be replaced by an empirical profile interpolator exposing:
|
| 46 |
|
| 47 |
- `prefill_seconds(token_counts)`
|
| 48 |
- `decode_step_seconds(context_lengths)`
|
| 49 |
- `model_weight_gb`
|
| 50 |
- `kv_bytes_per_token()`
|
| 51 |
|
| 52 |
+
without rewriting scheduling, cache, P/D orchestration, SLO metrics, or search tooling.
|
docs/methodology.md
CHANGED
|
@@ -1,69 +1,113 @@
|
|
| 1 |
# Methodology and limitations
|
| 2 |
|
| 3 |
-
## What v0.
|
| 4 |
|
| 5 |
-
InferScale models request arrival, queueing, admission, prefill, autoregressive decode, dynamic batch membership, KV-cache memory and request completion.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
## Scheduler semantics
|
| 8 |
|
| 9 |
-
- `static_fcfs`: admits one batch and drains it before admitting new requests.
|
| 10 |
- `continuous_fcfs`: admits FCFS work whenever decode slots become available.
|
| 11 |
-
- `continuous_sjf`: prioritizes shorter
|
| 12 |
-
- `continuous_slo`:
|
| 13 |
-
- `chunked_slo`: combines
|
| 14 |
|
| 15 |
-
These are
|
|
|
|
|
|
|
| 16 |
|
| 17 |
## Workload semantics
|
| 18 |
|
| 19 |
-
Poisson arrivals use exponentially distributed inter-arrival times. Constant arrivals are evenly spaced. Bursty arrivals alternate lower and higher rate periods
|
| 20 |
|
| 21 |
-
Prompt/output lengths are sampled from log-normal distributions parameterized by mean and coefficient of variation (CV)
|
| 22 |
|
| 23 |
-
##
|
| 24 |
|
| 25 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
|
| 27 |
-
|
| 28 |
-
- attention adds a context-length-dependent term;
|
| 29 |
-
- decode includes a shared model-weight memory stream and context-dependent KV reads;
|
| 30 |
-
- operation time is approximated from the larger of compute and memory costs plus a small launch/scheduling proxy;
|
| 31 |
-
- conservative efficiency factors prevent peak-spec numbers from being treated as achieved throughput.
|
| 32 |
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
-
##
|
|
|
|
|
|
|
| 36 |
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
-
|
| 40 |
|
| 41 |
-
KV
|
|
|
|
|
|
|
| 42 |
|
| 43 |
```text
|
| 44 |
-
2
|
| 45 |
```
|
| 46 |
|
| 47 |
for K and V with FP16 KV state. Paged allocation rounds live sequence lengths to `kv_block_tokens`. Static batching reserves full prompt+requested-output capacity.
|
| 48 |
|
| 49 |
## Capacity search
|
| 50 |
|
| 51 |
-
|
| 52 |
|
| 53 |
1. reaches the configured SLO-attainment target; and
|
| 54 |
-
2.
|
| 55 |
|
| 56 |
-
A bounded binary search
|
| 57 |
|
| 58 |
-
##
|
|
|
|
|
|
|
| 59 |
|
| 60 |
-
|
|
|
|
|
|
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
-
|
| 64 |
|
| 65 |
-
InferScale derives a compact explanation from the simulator's own state: component SLO attainment, p95 queue latency, virtual device busy fraction, peak KV utilization, and unfinished requests. These are heuristic simulator diagnoses, not hardware-counter diagnoses of a real GPU.
|
| 66 |
|
| 67 |
-
##
|
| 68 |
|
| 69 |
-
|
|
|
|
| 1 |
# Methodology and limitations
|
| 2 |
|
| 3 |
+
## What v0.3 simulates
|
| 4 |
|
| 5 |
+
InferScale models request arrival, queueing, admission, prefill, autoregressive decode, dynamic batch membership, KV-cache memory, and request completion.
|
| 6 |
+
|
| 7 |
+
For P/D disaggregation it additionally models independent prefill/decode worker pools and explicit prompt-KV transfer before decode admission.
|
| 8 |
+
|
| 9 |
+
Metrics are computed from per-request virtual timestamps.
|
| 10 |
|
| 11 |
## Scheduler semantics
|
| 12 |
|
| 13 |
+
- `static_fcfs`: admits one colocated batch and drains it before admitting new requests.
|
| 14 |
- `continuous_fcfs`: admits FCFS work whenever decode slots become available.
|
| 15 |
+
- `continuous_sjf`: prioritizes shorter estimated jobs at admission.
|
| 16 |
+
- `continuous_slo`: uses a least-slack-style ordering based on E2E deadline and an analytical remaining-service estimate.
|
| 17 |
+
- `chunked_slo`: combines least-slack ordering with chunked prompt prefill.
|
| 18 |
|
| 19 |
+
These are transparent approximations, not line-by-line reproductions of vLLM or SGLang.
|
| 20 |
+
|
| 21 |
+
Static batching is intentionally excluded from the P/D comparison; if requested for P/D, the simulator warns and uses continuous semantics.
|
| 22 |
|
| 23 |
## Workload semantics
|
| 24 |
|
| 25 |
+
Poisson arrivals use exponentially distributed inter-arrival times. Constant arrivals are evenly spaced. Bursty arrivals alternate lower and higher rate periods.
|
| 26 |
|
| 27 |
+
Prompt/output lengths are sampled from log-normal distributions parameterized by mean and coefficient of variation (CV).
|
| 28 |
|
| 29 |
+
## Shared-prefix model
|
| 30 |
|
| 31 |
+
v0.3 models one reusable exact prefix:
|
| 32 |
+
|
| 33 |
+
1. cache hits are assigned deterministically with a separate seeded RNG so enabling cache does **not** change the underlying arrival/prompt/output trace;
|
| 34 |
+
2. a hit reduces prefill work by the configured reusable-prefix length, bounded by the request prompt;
|
| 35 |
+
3. shared prefix KV consumes one persistent allocation per serving worker rather than being duplicated per request;
|
| 36 |
+
4. decode still uses the full logical context length for attention-cost estimation.
|
| 37 |
+
|
| 38 |
+
This isolates prefix reuse without implementing radix-tree lookup, eviction, or prefix-aware scheduling.
|
| 39 |
+
|
| 40 |
+
## P/D disaggregation
|
| 41 |
+
|
| 42 |
+
P/D uses a global discrete-event queue with four main event classes:
|
| 43 |
+
|
| 44 |
+
```text
|
| 45 |
+
arrival
|
| 46 |
+
prefill_done
|
| 47 |
+
transfer_done
|
| 48 |
+
decode_done
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
Prefill workers batch queued requests independently. Completed prompt state is placed on a serialized transfer link. Decode workers admit transferred requests only between decode iterations, preserving continuous-batching semantics.
|
| 52 |
|
| 53 |
+
The transfer model is:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
+
```text
|
| 56 |
+
base_latency + bytes / bandwidth
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
where transferred bytes correspond to newly computed prompt KV. With a cache hit, shared-prefix state is assumed resident in both role pools and only the uncached suffix is transferred.
|
| 60 |
+
|
| 61 |
+
The model does not claim protocol-level fidelity to PCIe, NVLink, RDMA, NIXL, or NCCL.
|
| 62 |
|
| 63 |
+
## Latency model
|
| 64 |
+
|
| 65 |
+
The default reference model is roofline-inspired:
|
| 66 |
|
| 67 |
+
- dense transformer FLOPs scale with parameter count and processed tokens;
|
| 68 |
+
- attention adds context-length-dependent work;
|
| 69 |
+
- decode includes model-weight traffic and context-dependent KV reads;
|
| 70 |
+
- time is approximated from compute/memory costs plus a launch/scheduling proxy;
|
| 71 |
+
- conservative efficiency factors keep accelerator peak specs from being treated as achieved throughput.
|
| 72 |
|
| 73 |
+
This creates useful qualitative dynamics but is **not empirically calibrated**.
|
| 74 |
|
| 75 |
+
## KV cache
|
| 76 |
+
|
| 77 |
+
KV bytes per token are approximated as:
|
| 78 |
|
| 79 |
```text
|
| 80 |
+
2 x layers x KV heads x head dimension x 2 bytes
|
| 81 |
```
|
| 82 |
|
| 83 |
for K and V with FP16 KV state. Paged allocation rounds live sequence lengths to `kv_block_tokens`. Static batching reserves full prompt+requested-output capacity.
|
| 84 |
|
| 85 |
## Capacity search
|
| 86 |
|
| 87 |
+
Each offered rate is evaluated over deterministic seed offsets and is feasible only when **every repetition**:
|
| 88 |
|
| 89 |
1. reaches the configured SLO-attainment target; and
|
| 90 |
+
2. fully drains all generated requests.
|
| 91 |
|
| 92 |
+
The result retains mean, worst, best, and standard-deviation evidence. A bounded binary search estimates the highest feasible rate and a recommended rate after user-selected headroom.
|
| 93 |
|
| 94 |
+
## Design-space explorer
|
| 95 |
+
|
| 96 |
+
The browser-safe v0.3 sweep evaluates:
|
| 97 |
|
| 98 |
+
- three colocated continuous schedulers over batch sizes 8/16/32;
|
| 99 |
+
- one additional cached SLO-aware colocated point at the user's current batch size;
|
| 100 |
+
- optionally P/D worker splits 1P:1D, 1P:2D, and 2P:1D, each with cache off/on.
|
| 101 |
|
| 102 |
+
A candidate is Pareto-optimal when no other candidate has both **at least as much goodput** and **no worse p95 TTFT**, with at least one strict improvement.
|
| 103 |
+
|
| 104 |
+
This is a bounded interactive study, not exhaustive global optimization.
|
| 105 |
+
|
| 106 |
+
## Validation status
|
| 107 |
|
| 108 |
+
v0.3 validates deterministic software invariants and expected qualitative behavior through tests. It does **not** provide held-out GPU calibration error. Empirical profile import/calibration remains a future extension rather than being fabricated for this release.
|
| 109 |
|
|
|
|
| 110 |
|
| 111 |
+
## Resource-normalized design comparison
|
| 112 |
|
| 113 |
+
P/D candidates can use more simulated accelerator instances than colocated candidates. v0.3 therefore reports both raw goodput and `goodput / accelerator`. The Design Explorer computes a separate Pareto frontier for each throughput objective against p95 TTFT. This avoids treating additional hardware as a free improvement and keeps raw performance distinct from resource efficiency.
|
docs/research.md
CHANGED
|
@@ -1,37 +1,67 @@
|
|
| 1 |
# Research context
|
| 2 |
|
| 3 |
-
InferScale-Sim is a portfolio-scale implementation situated within
|
| 4 |
|
| 5 |
## Vidur (2024)
|
| 6 |
|
| 7 |
-
Vidur combines experimental profiling, predictive models and end-to-end inference simulation. Its paper reports
|
| 8 |
|
| 9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
## TokenSim (2025)
|
| 12 |
|
| 13 |
-
TokenSim emphasizes extensible exploration of scheduling and memory-management choices
|
|
|
|
|
|
|
| 14 |
|
| 15 |
-
|
| 16 |
|
| 17 |
## Revati (2026)
|
| 18 |
|
| 19 |
-
Revati
|
|
|
|
|
|
|
| 20 |
|
| 21 |
-
|
| 22 |
|
| 23 |
## LLMServingSim 2.0 (2026)
|
| 24 |
|
| 25 |
-
LLMServingSim 2.0 focuses on heterogeneous and disaggregated infrastructure
|
| 26 |
|
| 27 |
-
|
| 28 |
|
| 29 |
-
|
| 30 |
|
| 31 |
-
Frontier
|
| 32 |
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
## Scope boundary
|
| 36 |
|
| 37 |
-
InferScale-Sim v0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# Research context
|
| 2 |
|
| 3 |
+
InferScale-Sim is a portfolio-scale implementation situated within the research line on replacing expensive deployment sweeps with modeling, simulation, or emulation.
|
| 4 |
|
| 5 |
## Vidur (2024)
|
| 6 |
|
| 7 |
+
Vidur combines experimental profiling, predictive models, and end-to-end inference simulation. Its paper reports under 9% inference-latency estimation error across the evaluated range and a configuration-search example where LLaMA2-70B exploration took about one CPU-hour versus an estimated 42K GPU-hours (~$218K) for deployment-based exploration.
|
| 8 |
|
| 9 |
+
InferScale inspiration: workload-aware configuration search, explicit performance-profile provenance, and separation between latency prediction and event simulation.
|
| 10 |
+
|
| 11 |
+
Source: https://arxiv.org/abs/2405.05465
|
| 12 |
+
|
| 13 |
+
## SGLang / RadixAttention (2024)
|
| 14 |
+
|
| 15 |
+
SGLang introduced RadixAttention for automatic KV-cache reuse across shared prompt prefixes. A radix tree retains reusable prompt/generation state and combines prefix lookup with cache-aware scheduling and eviction.
|
| 16 |
+
|
| 17 |
+
InferScale v0.3 does not implement a radix tree. It adds a deliberately smaller controlled shared-prefix abstraction so the effect of avoiding repeated prefill and duplicate KV allocation can be studied independently.
|
| 18 |
+
|
| 19 |
+
Source: https://arxiv.org/abs/2312.07104
|
| 20 |
|
| 21 |
## TokenSim (2025)
|
| 22 |
|
| 23 |
+
TokenSim emphasizes extensible exploration of scheduling and memory-management choices and reports sub-1% error in its evaluated workloads.
|
| 24 |
+
|
| 25 |
+
InferScale inspiration: modular scheduler/memory experiments.
|
| 26 |
|
| 27 |
+
Source: https://arxiv.org/abs/2503.08415
|
| 28 |
|
| 29 |
## Revati (2026)
|
| 30 |
|
| 31 |
+
Revati avoids reimplementing every evolving serving-system control decision: it executes real vLLM/SGLang control paths while virtualizing CUDA time. The paper reports less than 5% prediction error across its evaluated scenarios and 5-17x faster execution than real GPU serving runs.
|
| 32 |
+
|
| 33 |
+
InferScale lesson: pure simulation has a control-path fidelity ceiling. Our schedulers therefore document their semantics rather than claiming framework equivalence.
|
| 34 |
|
| 35 |
+
Source: https://arxiv.org/abs/2601.00397
|
| 36 |
|
| 37 |
## LLMServingSim 2.0 (2026)
|
| 38 |
|
| 39 |
+
LLMServingSim 2.0 focuses on heterogeneous and disaggregated infrastructure and integrates batching, routing, offloading, memory, communication, and power into a unified runtime loop. The 2026 paper reports about 0.97% average error in its validation.
|
| 40 |
|
| 41 |
+
InferScale v0.3 inspiration: role-specific resources and explicit data movement rather than assuming a monolithic replica.
|
| 42 |
|
| 43 |
+
Source: https://arxiv.org/abs/2602.23036
|
| 44 |
|
| 45 |
+
## Frontier (May 2026)
|
| 46 |
|
| 47 |
+
Frontier models co-location, Prefill-Decode Disaggregation, Attention-FFN Disaggregation, runtime optimizations such as speculative decoding, and stateful workloads. It reports average throughput error below 4% on its 16-H800 evaluation and explicitly supports SLA-dependent Pareto-frontier exploration.
|
| 48 |
+
|
| 49 |
+
InferScale v0.3 inspiration:
|
| 50 |
+
|
| 51 |
+
- separate prefill and decode worker pools;
|
| 52 |
+
- communication/KV-transfer cost;
|
| 53 |
+
- role-specific bottleneck telemetry;
|
| 54 |
+
- an interactive bounded Pareto design-space study.
|
| 55 |
+
|
| 56 |
+
Source: https://arxiv.org/abs/2605.21312
|
| 57 |
|
| 58 |
## Scope boundary
|
| 59 |
|
| 60 |
+
InferScale-Sim v0.3 is not intended to compete with these research systems on fidelity, hardware scale, or runtime compatibility. Its goal is an inspectable, dependency-light Python implementation with a zero-backend interactive interface for understanding the systems mechanisms that motivate them.
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
## HeteroPanacea / When Does Disaggregation Pay? (Aug 2026)
|
| 64 |
+
|
| 65 |
+
**When Does Disaggregation Pay? Simulating Prefill--Decode--Attention--FFN Specialization for Agentic LLM Inference** pushes the simulator design space toward heterogeneous stage-specific hardware and automated cross-stack exploration. InferScale-Sim v0.3 does not model Attention--FFN disaggregation or custom NPUs, but its P/D topology already permits independent prefill and decode accelerator profiles and reports resource-normalized efficiency so heterogeneous layouts can be compared without treating extra accelerators as free.
|
| 66 |
+
|
| 67 |
+
- arXiv: https://arxiv.org/abs/2608.03741
|
docs/validation.md
CHANGED
|
@@ -1,35 +1,51 @@
|
|
| 1 |
-
# v0.
|
| 2 |
|
| 3 |
-
The release
|
| 4 |
|
| 5 |
## Automated
|
| 6 |
|
| 7 |
- deterministic workload generation
|
| 8 |
-
- constant
|
| 9 |
-
- prefill latency monotonicity
|
| 10 |
- quantization footprint ordering
|
| 11 |
-
-
|
| 12 |
-
-
|
| 13 |
-
-
|
| 14 |
-
-
|
| 15 |
-
-
|
| 16 |
-
-
|
| 17 |
-
-
|
| 18 |
-
-
|
| 19 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
- Hugging Face `short_description` <= 60 characters
|
| 21 |
- `sdk: static` metadata
|
| 22 |
-
- canonical Python source
|
|
|
|
| 23 |
- provenance remains `analytical-reference`
|
| 24 |
- JavaScript syntax parse
|
| 25 |
- Python compilation
|
| 26 |
|
| 27 |
-
## Not claimed in v0.
|
| 28 |
|
| 29 |
- empirical L4/A10G/A100 latency accuracy
|
| 30 |
- exact vLLM/SGLang scheduler equivalence
|
| 31 |
- CUDA-kernel modeling
|
| 32 |
-
-
|
| 33 |
-
-
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# v0.3 validation checklist
|
| 2 |
|
| 3 |
+
The release checks prevent software, deployment, and provenance mistakes; they do not pretend the analytical profile has measured-hardware fidelity.
|
| 4 |
|
| 5 |
## Automated
|
| 6 |
|
| 7 |
- deterministic workload generation
|
| 8 |
+
- constant/Poisson/bursty workload behavior
|
| 9 |
+
- prefill latency monotonicity
|
| 10 |
- quantization footprint ordering
|
| 11 |
+
- colocated end-to-end completion
|
| 12 |
+
- static vs continuous behavioral difference
|
| 13 |
+
- component TTFT/E2E SLO accounting
|
| 14 |
+
- robust repetition-aware capacity search
|
| 15 |
+
- bottleneck-diagnosis provenance
|
| 16 |
+
- prefix hits do not alter the underlying generated request trace
|
| 17 |
+
- prefix reuse reduces modeled prefill work
|
| 18 |
+
- P/D pipeline completion
|
| 19 |
+
- non-zero P/D transfer telemetry
|
| 20 |
+
- configurable P/D worker counts
|
| 21 |
+
- four-scenario topology/cache comparison
|
| 22 |
+
- bounded design-space sweep and Pareto marking
|
| 23 |
+
- ASCII-only public UI labels
|
| 24 |
+
- chart export controls present
|
| 25 |
+
- explicit planner worst-repetition and target columns
|
| 26 |
+
- no product-style footer
|
| 27 |
- Hugging Face `short_description` <= 60 characters
|
| 28 |
- `sdk: static` metadata
|
| 29 |
+
- canonical Python source equals browser mirror
|
| 30 |
+
- every Python module is included by the worker
|
| 31 |
- provenance remains `analytical-reference`
|
| 32 |
- JavaScript syntax parse
|
| 33 |
- Python compilation
|
| 34 |
|
| 35 |
+
## Not claimed in v0.3
|
| 36 |
|
| 37 |
- empirical L4/A10G/A100 latency accuracy
|
| 38 |
- exact vLLM/SGLang scheduler equivalence
|
| 39 |
- CUDA-kernel modeling
|
| 40 |
+
- real network-protocol fidelity
|
| 41 |
+
- radix-tree prefix-cache eviction/scheduling
|
| 42 |
+
- speculative decoding
|
| 43 |
+
- Attention-FFN disaggregation
|
| 44 |
+
- multi-turn/agentic session fidelity
|
| 45 |
|
| 46 |
+
These remain explicit future extensions.
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
## Design-space sanity checks
|
| 50 |
+
|
| 51 |
+
The release suite verifies that the bounded design sweep returns at least one raw-performance Pareto point and at least one resource-efficiency Pareto point. P/D candidates also expose accelerator-instance counts so raw and normalized throughput can be interpreted separately.
|
index.html
CHANGED
|
@@ -11,51 +11,268 @@
|
|
| 11 |
</head>
|
| 12 |
<body>
|
| 13 |
<header class="topbar" id="topbar">
|
| 14 |
-
<div class="brand-wrap">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
<div class="runtime-pill" id="runtimePill"><span class="dot"></span><span id="runtimeText">Loading Python runtime...</span></div>
|
| 16 |
</header>
|
| 17 |
|
| 18 |
<main class="shell">
|
| 19 |
<section class="intro">
|
| 20 |
-
<div class="intro-copy">
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
</section>
|
| 23 |
-
<div class="reference-note"><strong>Reference-profile mode.</strong> Absolute L4, A10G, and A100 latency values are analytical predictions, not measured GPU benchmarks. Scheduler, queueing, KV-cache, and SLO behavior is simulated live.</div>
|
| 24 |
-
|
| 25 |
-
<nav class="tabs" aria-label="InferScale sections"><button class="tab active" data-tab="lab">Serving Lab</button><button class="tab" data-tab="arena">Scheduler Arena</button><button class="tab" data-tab="planner">Capacity Planner</button><button class="tab" data-tab="method">Methodology</button></nav>
|
| 26 |
-
|
| 27 |
-
<section id="lab" class="tab-panel active"><div class="workspace">
|
| 28 |
-
<aside class="panel controls-panel">
|
| 29 |
-
<div class="panel-title-row"><h2>Experiment</h2><span class="tag">Deterministic seed</span></div>
|
| 30 |
-
<div class="field-grid two"><label>Model<select id="model"><option>Qwen2.5-3B</option><option>Llama-3.1-8B</option><option>Mistral-7B-v0.3</option></select></label><label>Accelerator<select id="accelerator"><option value="L4">NVIDIA L4</option><option value="A10G">NVIDIA A10G</option><option value="A100-40GB">NVIDIA A100 40GB</option></select></label></div>
|
| 31 |
-
<div class="field-grid two"><label>Scheduler<select id="scheduler"><option value="continuous_fcfs">Continuous - FCFS</option><option value="continuous_sjf">Continuous - SJF</option><option value="continuous_slo">Continuous - SLO-aware</option><option value="chunked_slo">Chunked prefill - SLO-aware</option><option value="static_fcfs">Static batching - FCFS</option></select></label><label>Weight precision<select id="quantization"><option value="fp16">FP16</option><option value="int8" selected>INT8 scenario</option><option value="int4">INT4 scenario</option></select></label></div>
|
| 32 |
-
<hr /><div class="section-kicker">Workload</div>
|
| 33 |
-
<div class="field-grid two"><label>Arrival process<select id="arrival"><option value="poisson">Poisson</option><option value="constant">Constant</option><option value="bursty">Bursty</option></select></label><label>Request rate<input id="rate" type="number" min="0.1" step="0.1" value="4" /><span class="unit">req/s</span></label><label>Duration<input id="duration" type="number" min="2" step="1" value="30" /><span class="unit">sim s</span></label><label>Seed<input id="seed" type="number" step="1" value="7" /></label></div>
|
| 34 |
-
<div id="burstControls" class="field-grid two hidden"><label>Burst multiplier<input id="burstMultiplier" type="number" min="1" step="0.25" value="3" /><span class="unit">x</span></label><label>Burst period<input id="burstPeriod" type="number" min="0.5" step="0.5" value="10" /><span class="unit">s</span></label></div>
|
| 35 |
-
<div class="field-grid two"><label>Prompt mean<input id="promptMean" type="number" min="16" value="512" /><span class="unit">tokens</span></label><label>Prompt CV<input id="promptCv" type="number" min="0" max="2" step="0.05" value="0.50" /></label><label>Output mean<input id="outputMean" type="number" min="1" value="64" /><span class="unit">tokens</span></label><label>Output CV<input id="outputCv" type="number" min="0" max="2" step="0.05" value="0.60" /></label></div>
|
| 36 |
-
<hr /><div class="section-kicker">Serving controls</div>
|
| 37 |
-
<div class="field-grid two"><label>Max batch size<input id="maxBatch" type="number" min="1" max="128" value="16" /></label><label>Max batch tokens<input id="maxBatchTokens" type="number" min="128" step="128" value="8192" /></label><label>Prefill chunk<input id="chunkSize" type="number" min="64" step="64" value="512" /><span class="unit">tokens</span></label><label>KV block<input id="kvBlock" type="number" min="1" value="16" /><span class="unit">tokens</span></label></div>
|
| 38 |
-
<hr /><div class="section-kicker">SLO</div><div class="field-grid two"><label>TTFT limit<input id="sloTtft" type="number" min="1" value="500" /><span class="unit">ms</span></label><label>E2E limit<input id="sloE2e" type="number" min="100" value="5000" /><span class="unit">ms</span></label></div>
|
| 39 |
-
<button id="runBtn" class="primary" disabled>Run simulation</button><div class="button-row"><button id="copyResultBtn" class="secondary" disabled>Copy JSON</button><button id="exportBtn" class="secondary" disabled>Download JSON</button></div>
|
| 40 |
-
</aside>
|
| 41 |
-
<div class="results-column"><section class="panel result-panel"><div class="panel-title-row"><h2>Run summary</h2><span id="runState" class="tag neutral">Waiting</span></div><div id="emptyState" class="empty-state"><h3>Configure a workload and run it</h3><p>The Python simulator executes in a background Web Worker and returns request-level virtual timestamps.</p></div><div id="resultContent" class="hidden">
|
| 42 |
-
<div class="metric-grid"><div class="metric"><span>p95 TTFT</span><strong id="mTtft">N/A</strong></div><div class="metric"><span>p95 E2E</span><strong id="mE2e">N/A</strong></div><div class="metric"><span>Goodput</span><strong id="mGoodput">N/A</strong></div><div class="metric"><span>SLO attainment</span><strong id="mSlo">N/A</strong></div><div class="metric"><span>Throughput</span><strong id="mReq">N/A</strong></div><div class="metric"><span>Peak KV</span><strong id="mKv">N/A</strong></div></div>
|
| 43 |
-
<section class="diagnostic-card" id="diagnosticCard"><div class="diagnostic-head"><span>Simulator diagnosis</span><strong id="mBottleneck">N/A</strong></div><p id="mDiagnosis">Run a simulation to generate a bottleneck explanation.</p><p class="diagnostic-action" id="mRecommendation"></p><div class="evidence-row" id="mEvidence"></div></section>
|
| 44 |
-
<div class="chart-grid"><div class="chart-card" data-chart-card><div class="chart-head"><div class="chart-title">Latency percentiles</div><button class="chart-expand" type="button">Expand</button></div><div class="chart-body"><canvas id="latencyChart"></canvas></div></div><div class="chart-card" data-chart-card><div class="chart-head"><div class="chart-title">Queue, decode, and KV timeline</div><button class="chart-expand" type="button">Expand</button></div><div class="chart-body"><canvas id="timelineChart"></canvas></div></div></div>
|
| 45 |
-
<div class="chart-card full" data-chart-card><div class="chart-head"><div class="chart-title">Request TTFT vs prompt length</div><button class="chart-expand" type="button">Expand</button></div><div class="chart-body"><canvas id="scatterChart"></canvas></div></div><div id="warnings" class="warnings hidden"></div>
|
| 46 |
-
</div></section></div>
|
| 47 |
-
</div></section>
|
| 48 |
-
|
| 49 |
-
<section id="arena" class="tab-panel"><div class="panel wide-panel"><div class="panel-title-row arena-title-row"><div><div class="section-kicker">Same workload - same seed</div><h2>Scheduler Arena</h2><p class="muted">Run every scheduler against the current Serving Lab configuration and rank by SLO attainment, then goodput.</p></div><button id="arenaBtn" class="primary compact" disabled>Compare schedulers</button></div><div id="arenaEmpty" class="empty-state small"><h3>No comparison yet</h3><p>Your Serving Lab controls are reused automatically.</p></div><div id="arenaContent" class="hidden"><div class="chart-card full" data-chart-card><div class="chart-head"><div class="chart-title">Useful vs raw request throughput</div><button class="chart-expand" type="button">Expand</button></div><div class="chart-body large"><canvas id="arenaChart"></canvas></div></div><div class="table-toolbar"><span>Scheduler comparison</span><div><button id="arenaCopyBtn" class="mini-button" disabled>Copy table</button><button id="arenaCsvBtn" class="mini-button" disabled>Download CSV</button></div></div><div class="table-wrap"><table><thead><tr><th>Scheduler</th><th>Goodput</th><th>SLO attainment</th><th>p95 TTFT</th><th>p95 E2E</th><th>KV peak</th><th>Unfinished</th><th>Diagnosis</th></tr></thead><tbody id="arenaRows"></tbody></table></div></div></div></section>
|
| 50 |
-
|
| 51 |
-
<section id="planner" class="tab-panel"><div class="workspace planner-grid"><aside class="panel controls-panel"><div class="panel-title-row"><h2>Capacity search</h2><span class="tag">Robust binary search</span></div><p class="muted">Find the highest offered load where every repetition meets the SLO target and fully drains.</p><label>Required SLO attainment<input id="targetSlo" type="number" min="0.5" max="1" step="0.001" value="0.99" /></label><div class="field-grid two"><label>Minimum rate<input id="minRate" type="number" min="0.05" step="0.1" value="0.25" /><span class="unit">req/s</span></label><label>Maximum rate<input id="maxRate" type="number" min="0.1" step="1" value="20" /><span class="unit">req/s</span></label><label>Search iterations<input id="searchIter" type="number" min="2" max="12" value="7" /></label><label>Repetitions / rate<input id="repetitions" type="number" min="1" max="5" value="2" /></label></div><label>Safety headroom<input id="headroom" type="number" min="0" max="0.8" step="0.05" value="0.20" /><span class="unit">fraction</span></label><button id="capacityBtn" class="primary" disabled>Find sustainable capacity</button></aside>
|
| 52 |
-
<section class="panel result-panel"><div class="panel-title-row"><h2>Planner result</h2><span id="plannerState" class="tag neutral">Waiting</span></div><div id="plannerEmpty" class="empty-state"><h3>No search yet</h3><p>The planner repeatedly runs the simulator at different offered loads.</p></div><div id="plannerContent" class="hidden"><div class="metric-grid four"><div class="metric emphasis"><span>Estimated capacity</span><strong id="pCapacity">N/A</strong></div><div class="metric"><span>Recommended load</span><strong id="pRecommended">N/A</strong></div><div class="metric"><span>Safety headroom</span><strong id="pHeadroom">N/A</strong></div><div class="metric"><span>Status</span><strong id="pStatus">N/A</strong></div></div><div class="planner-note" id="plannerCriterion">A rate passes only if every repetition meets the target and drains all generated requests.</div><div class="chart-card full" data-chart-card><div class="chart-head"><div class="chart-title">SLO attainment across searched rates</div><button class="chart-expand" type="button">Expand</button></div><div class="chart-body large"><canvas id="capacityChart"></canvas></div></div><div class="table-toolbar"><span>Search trace</span><div><button id="capacityCopyBtn" class="mini-button" disabled>Copy table</button><button id="capacityCsvBtn" class="mini-button" disabled>Download CSV</button></div></div><div class="table-wrap"><table><thead><tr><th>Rate</th><th>Pass</th><th>Mean SLO</th><th>SLO range</th><th>Goodput</th><th>p95 TTFT</th><th>p95 E2E</th></tr></thead><tbody id="capacityRows"></tbody></table></div></div></section>
|
| 53 |
-
</div></section>
|
| 54 |
-
|
| 55 |
-
<section id="method" class="tab-panel"><div class="method-grid"><article class="panel prose"><div class="section-kicker">What is simulated?</div><h2>A serving loop, not CUDA kernels.</h2><p>InferScale advances virtual time through request arrivals, queueing, prefill, autoregressive decode, batch membership changes, KV allocation, and completion. A predicted 40 ms decode step becomes <code>simulated_time += 0.040</code>; the browser never waits 40 ms.</p><p>The latency backend is roofline-inspired and replaceable. v0.2 uses it to expose systems interactions, not to claim measured L4/A100 latency.</p></article><article class="panel prose"><div class="section-kicker">Why goodput?</div><h2>Throughput can reward overload.</h2><p>Goodput counts completed requests satisfying both TTFT and E2E SLOs per simulated second.</p><div class="formula">goodput = SLO-compliant completions / simulated time</div></article><article class="panel prose"><div class="section-kicker">v0.2 analysis</div><h2>Explain the limiting regime.</h2><p>Each run derives a simulator-side diagnosis from SLO failures, queue latency, busy fraction, unfinished work, and KV utilization. These labels explain simulated state, not hardware counters.</p><div class="formula">healthy | queue/prefill | decode/E2E | KV pressure | over-capacity</div></article><article class="panel prose"><div class="section-kicker">Capacity criterion</div><h2>Do not average away a bad repetition.</h2><p>A searched rate passes only when every configured repetition reaches the SLO target and fully drains. Mean, minimum, and maximum attainment remain visible in the trace.</p></article><article class="panel prose wide-method"><div class="section-kicker">Research lineage</div><h2>From Vidur to modern disaggregated simulators.</h2><div class="paper-grid"><div><strong>Vidur - 2024</strong><span>Predictive profiling and deployment search. Reported about 1 CPU-hour versus an estimated 42K GPU-hours (about $218K) for one LLaMA2-70B search.</span></div><div><strong>TokenSim - 2025</strong><span>Extensible hardware/software exploration with scheduler and memory-management modeling.</span></div><div><strong>Revati - 2026</strong><span>GPU-free time-warp emulation using serving control logic.</span></div><div><strong>LLMServingSim 2.0 - 2026</strong><span>Heterogeneous and disaggregated serving with runtime-driven batching, routing, memory, and power.</span></div><div><strong>Frontier - 2026</strong><span>Modern disaggregation, runtime optimizations, stateful workloads, and large-scale configuration exploration.</span></div></div></article></div></section>
|
| 56 |
</main>
|
| 57 |
-
|
| 58 |
-
<div id="toast" class="toast" role="status"
|
| 59 |
-
<script
|
| 60 |
</body>
|
| 61 |
</html>
|
|
|
|
| 11 |
</head>
|
| 12 |
<body>
|
| 13 |
<header class="topbar" id="topbar">
|
| 14 |
+
<div class="brand-wrap">
|
| 15 |
+
<div>
|
| 16 |
+
<div class="brand">InferScale-Sim</div>
|
| 17 |
+
<div class="subtitle">interactive serving-systems research prototype</div>
|
| 18 |
+
</div>
|
| 19 |
+
</div>
|
| 20 |
<div class="runtime-pill" id="runtimePill"><span class="dot"></span><span id="runtimeText">Loading Python runtime...</span></div>
|
| 21 |
</header>
|
| 22 |
|
| 23 |
<main class="shell">
|
| 24 |
<section class="intro">
|
| 25 |
+
<div class="intro-copy">
|
| 26 |
+
<div class="eyebrow">InferScale-Sim / v0.3.0</div>
|
| 27 |
+
<h1>Explore serving policies, cache reuse, and P/D disaggregation without provisioning a GPU.</h1>
|
| 28 |
+
<p>Generate workloads, compare schedulers, model KV pressure and prefix reuse, separate prefill from decode, and search the SLO-constrained design space. The simulator is Python running locally in a Pyodide Web Worker.</p>
|
| 29 |
+
</div>
|
| 30 |
+
<dl class="project-facts">
|
| 31 |
+
<div><dt>Backend</dt><dd>None</dd></div>
|
| 32 |
+
<div><dt>Runtime</dt><dd>Python / WASM</dd></div>
|
| 33 |
+
<div><dt>Network data</dt><dd>None after load</dd></div>
|
| 34 |
+
<div><dt>Profiles</dt><dd>Analytical reference</dd></div>
|
| 35 |
+
</dl>
|
| 36 |
+
</section>
|
| 37 |
+
|
| 38 |
+
<div class="reference-note"><strong>Reference-profile mode.</strong> Absolute L4, A10G, and A100 latency values are analytical predictions, not measured GPU benchmarks. Queueing, scheduling, cache, transfer, and SLO behavior is simulated live.</div>
|
| 39 |
+
|
| 40 |
+
<nav class="tabs" aria-label="InferScale sections">
|
| 41 |
+
<button class="tab active" data-tab="lab">Serving Lab</button>
|
| 42 |
+
<button class="tab" data-tab="arena">Scheduler Arena</button>
|
| 43 |
+
<button class="tab" data-tab="planner">Capacity Planner</button>
|
| 44 |
+
<button class="tab" data-tab="modern">Modern Serving</button>
|
| 45 |
+
<button class="tab" data-tab="design">Design Explorer</button>
|
| 46 |
+
<button class="tab" data-tab="method">Methodology</button>
|
| 47 |
+
</nav>
|
| 48 |
+
|
| 49 |
+
<section id="lab" class="tab-panel active">
|
| 50 |
+
<div class="workspace">
|
| 51 |
+
<aside class="panel controls-panel">
|
| 52 |
+
<div class="panel-title-row"><h2>Experiment</h2><span class="tag">Deterministic seed</span></div>
|
| 53 |
+
<div class="field-grid two">
|
| 54 |
+
<label>Model<select id="model"><option>Qwen2.5-3B</option><option>Llama-3.1-8B</option><option>Mistral-7B-v0.3</option></select></label>
|
| 55 |
+
<label>Weight precision<select id="quantization"><option value="fp16">FP16</option><option value="int8" selected>INT8 scenario</option><option value="int4">INT4 scenario</option></select></label>
|
| 56 |
+
</div>
|
| 57 |
+
<div class="field-grid two">
|
| 58 |
+
<label>Topology<select id="topology"><option value="colocated">Colocated</option><option value="disaggregated_pd">P/D disaggregated</option></select></label>
|
| 59 |
+
<label id="colocatedAcceleratorLabel">Accelerator<select id="accelerator"><option value="L4">NVIDIA L4</option><option value="A10G">NVIDIA A10G</option><option value="A100-40GB">NVIDIA A100 40GB</option></select></label>
|
| 60 |
+
</div>
|
| 61 |
+
<div class="field-grid two">
|
| 62 |
+
<label>Scheduler<select id="scheduler"><option value="continuous_fcfs">Continuous - FCFS</option><option value="continuous_sjf">Continuous - SJF</option><option value="continuous_slo">Continuous - SLO-aware</option><option value="chunked_slo">Chunked prefill - SLO-aware</option><option value="static_fcfs">Static batching - FCFS</option></select></label>
|
| 63 |
+
<label>Prefix reuse<select id="prefixCache"><option value="off">Off</option><option value="on">On</option></select></label>
|
| 64 |
+
</div>
|
| 65 |
+
|
| 66 |
+
<div id="prefixControls" class="subcontrols hidden">
|
| 67 |
+
<div class="section-kicker">Shared-prefix scenario</div>
|
| 68 |
+
<div class="field-grid two">
|
| 69 |
+
<label>Shared prefix<input id="sharedPrefix" type="number" min="0" step="32" value="256" /><span class="unit">tokens</span></label>
|
| 70 |
+
<label>Reuse fraction<input id="prefixReuse" type="number" min="0" max="1" step="0.05" value="0.60" /><span class="unit">fraction</span></label>
|
| 71 |
+
</div>
|
| 72 |
+
</div>
|
| 73 |
+
|
| 74 |
+
<div id="pdControls" class="subcontrols hidden">
|
| 75 |
+
<div class="section-kicker">P/D topology</div>
|
| 76 |
+
<div class="field-grid two">
|
| 77 |
+
<label>Prefill accelerator<select id="prefillAccelerator"><option value="L4">NVIDIA L4</option><option value="A10G">NVIDIA A10G</option><option value="A100-40GB">NVIDIA A100 40GB</option></select></label>
|
| 78 |
+
<label>Decode accelerator<select id="decodeAccelerator"><option value="L4">NVIDIA L4</option><option value="A10G">NVIDIA A10G</option><option value="A100-40GB">NVIDIA A100 40GB</option></select></label>
|
| 79 |
+
<label>Prefill workers<input id="prefillWorkers" type="number" min="1" max="8" step="1" value="1" /></label>
|
| 80 |
+
<label>Decode workers<input id="decodeWorkers" type="number" min="1" max="8" step="1" value="1" /></label>
|
| 81 |
+
<label>Interconnect<input id="interconnect" type="number" min="1" step="1" value="50" /><span class="unit">GB/s</span></label>
|
| 82 |
+
<label>Transfer base<input id="transferBase" type="number" min="0" step="0.05" value="0.20" /><span class="unit">ms</span></label>
|
| 83 |
+
</div>
|
| 84 |
+
</div>
|
| 85 |
+
|
| 86 |
+
<hr />
|
| 87 |
+
<div class="section-kicker">Workload</div>
|
| 88 |
+
<div class="field-grid two">
|
| 89 |
+
<label>Arrival process<select id="arrival"><option value="poisson">Poisson</option><option value="constant">Constant</option><option value="bursty">Bursty</option></select></label>
|
| 90 |
+
<label>Request rate<input id="rate" type="number" min="0.1" step="0.1" value="4" /><span class="unit">req/s</span></label>
|
| 91 |
+
<label>Duration<input id="duration" type="number" min="2" step="1" value="30" /><span class="unit">sim s</span></label>
|
| 92 |
+
<label>Seed<input id="seed" type="number" step="1" value="7" /></label>
|
| 93 |
+
</div>
|
| 94 |
+
<div id="burstControls" class="field-grid two hidden">
|
| 95 |
+
<label>Burst multiplier<input id="burstMultiplier" type="number" min="1" step="0.25" value="3" /><span class="unit">x</span></label>
|
| 96 |
+
<label>Burst period<input id="burstPeriod" type="number" min="0.5" step="0.5" value="10" /><span class="unit">s</span></label>
|
| 97 |
+
</div>
|
| 98 |
+
<div class="field-grid two">
|
| 99 |
+
<label>Prompt mean<input id="promptMean" type="number" min="16" value="512" /><span class="unit">tokens</span></label>
|
| 100 |
+
<label>Prompt CV<input id="promptCv" type="number" min="0" max="2" step="0.05" value="0.50" /></label>
|
| 101 |
+
<label>Output mean<input id="outputMean" type="number" min="1" value="64" /><span class="unit">tokens</span></label>
|
| 102 |
+
<label>Output CV<input id="outputCv" type="number" min="0" max="2" step="0.05" value="0.60" /></label>
|
| 103 |
+
</div>
|
| 104 |
+
|
| 105 |
+
<hr />
|
| 106 |
+
<div class="section-kicker">Serving controls</div>
|
| 107 |
+
<div class="field-grid two">
|
| 108 |
+
<label>Max batch size<input id="maxBatch" type="number" min="1" max="128" value="16" /></label>
|
| 109 |
+
<label>Max batch tokens<input id="maxBatchTokens" type="number" min="128" step="128" value="8192" /></label>
|
| 110 |
+
<label>Prefill chunk<input id="chunkSize" type="number" min="64" step="64" value="512" /><span class="unit">tokens</span></label>
|
| 111 |
+
<label>KV block<input id="kvBlock" type="number" min="1" value="16" /><span class="unit">tokens</span></label>
|
| 112 |
+
</div>
|
| 113 |
+
|
| 114 |
+
<hr />
|
| 115 |
+
<div class="section-kicker">SLO</div>
|
| 116 |
+
<div class="field-grid two">
|
| 117 |
+
<label>TTFT limit<input id="sloTtft" type="number" min="1" value="500" /><span class="unit">ms</span></label>
|
| 118 |
+
<label>E2E limit<input id="sloE2e" type="number" min="100" value="5000" /><span class="unit">ms</span></label>
|
| 119 |
+
</div>
|
| 120 |
+
|
| 121 |
+
<button id="runBtn" class="primary" disabled>Run simulation</button>
|
| 122 |
+
<div class="button-row"><button id="copyResultBtn" class="secondary" disabled>Copy JSON</button><button id="exportBtn" class="secondary" disabled>Download JSON</button></div>
|
| 123 |
+
</aside>
|
| 124 |
+
|
| 125 |
+
<div class="results-column">
|
| 126 |
+
<section class="panel result-panel">
|
| 127 |
+
<div class="panel-title-row"><h2>Run summary</h2><span id="runState" class="tag neutral">Waiting</span></div>
|
| 128 |
+
<div id="emptyState" class="empty-state"><h3>Configure a workload and run it</h3><p>The Python simulator executes in a background Web Worker and returns request-level virtual timestamps.</p></div>
|
| 129 |
+
<div id="resultContent" class="hidden">
|
| 130 |
+
<div class="metric-grid">
|
| 131 |
+
<div class="metric"><span>p95 TTFT</span><strong id="mTtft">N/A</strong></div>
|
| 132 |
+
<div class="metric"><span>p95 E2E</span><strong id="mE2e">N/A</strong></div>
|
| 133 |
+
<div class="metric"><span>Goodput</span><strong id="mGoodput">N/A</strong></div>
|
| 134 |
+
<div class="metric"><span>SLO attainment</span><strong id="mSlo">N/A</strong></div>
|
| 135 |
+
<div class="metric"><span>Throughput</span><strong id="mReq">N/A</strong></div>
|
| 136 |
+
<div class="metric"><span>Peak KV</span><strong id="mKv">N/A</strong></div>
|
| 137 |
+
</div>
|
| 138 |
+
<section class="diagnostic-card" id="diagnosticCard">
|
| 139 |
+
<div class="diagnostic-head"><span>Simulator diagnosis</span><strong id="mBottleneck">N/A</strong></div>
|
| 140 |
+
<p id="mDiagnosis">Run a simulation to generate a bottleneck explanation.</p>
|
| 141 |
+
<p class="diagnostic-action" id="mRecommendation"></p>
|
| 142 |
+
<div class="evidence-row" id="mEvidence"></div>
|
| 143 |
+
</section>
|
| 144 |
+
<div class="chart-grid">
|
| 145 |
+
<div class="chart-card" data-chart-card data-chart-name="latency-percentiles">
|
| 146 |
+
<div class="chart-head"><div class="chart-title">Latency percentiles</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 147 |
+
<div class="chart-body"><canvas id="latencyChart"></canvas></div>
|
| 148 |
+
</div>
|
| 149 |
+
<div class="chart-card" data-chart-card data-chart-name="queue-decode-kv-timeline">
|
| 150 |
+
<div class="chart-head"><div class="chart-title">Queue, decode, and KV timeline</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 151 |
+
<div class="chart-body"><canvas id="timelineChart"></canvas></div>
|
| 152 |
+
</div>
|
| 153 |
+
</div>
|
| 154 |
+
<div class="chart-card full" data-chart-card data-chart-name="ttft-vs-prompt-length">
|
| 155 |
+
<div class="chart-head"><div class="chart-title">Request TTFT vs prompt length</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 156 |
+
<div class="chart-body"><canvas id="scatterChart"></canvas></div>
|
| 157 |
+
</div>
|
| 158 |
+
<div id="warnings" class="warnings hidden"></div>
|
| 159 |
+
</div>
|
| 160 |
+
</section>
|
| 161 |
+
</div>
|
| 162 |
+
</div>
|
| 163 |
+
</section>
|
| 164 |
+
|
| 165 |
+
<section id="arena" class="tab-panel">
|
| 166 |
+
<div class="panel wide-panel">
|
| 167 |
+
<div class="panel-title-row arena-title-row">
|
| 168 |
+
<div><div class="section-kicker">Same workload - same seed</div><h2>Scheduler Arena</h2><p class="muted">Run every colocated v0.3 scheduler against the current Serving Lab workload and rank by SLO attainment, then goodput.</p></div>
|
| 169 |
+
<button id="arenaBtn" class="primary compact" disabled>Compare schedulers</button>
|
| 170 |
+
</div>
|
| 171 |
+
<div id="arenaEmpty" class="empty-state small"><h3>No comparison yet</h3><p>Your Serving Lab workload and model controls are reused automatically.</p></div>
|
| 172 |
+
<div id="arenaContent" class="hidden">
|
| 173 |
+
<div class="chart-card full" data-chart-card data-chart-name="scheduler-throughput-comparison">
|
| 174 |
+
<div class="chart-head"><div class="chart-title">Useful vs raw request throughput</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 175 |
+
<div class="chart-body large"><canvas id="arenaChart"></canvas></div>
|
| 176 |
+
</div>
|
| 177 |
+
<div class="table-toolbar"><span>Scheduler comparison</span><div><button id="arenaCopyBtn" class="mini-button" disabled>Copy table</button><button id="arenaCsvBtn" class="mini-button" disabled>Download CSV</button></div></div>
|
| 178 |
+
<div class="table-wrap"><table><thead><tr><th>Scheduler</th><th>Goodput</th><th>SLO attainment</th><th>p95 TTFT</th><th>p95 E2E</th><th>KV peak</th><th>Unfinished</th><th>Diagnosis</th></tr></thead><tbody id="arenaRows"></tbody></table></div>
|
| 179 |
+
</div>
|
| 180 |
+
</div>
|
| 181 |
+
</section>
|
| 182 |
+
|
| 183 |
+
<section id="planner" class="tab-panel">
|
| 184 |
+
<div class="workspace planner-grid">
|
| 185 |
+
<aside class="panel controls-panel">
|
| 186 |
+
<div class="panel-title-row"><h2>Capacity search</h2><span class="tag">Robust binary search</span></div>
|
| 187 |
+
<p class="muted">Find the highest offered load where every repetition meets the SLO target and fully drains.</p>
|
| 188 |
+
<label>Required SLO attainment<input id="targetSlo" type="number" min="0.5" max="1" step="0.001" value="0.99" /></label>
|
| 189 |
+
<div class="field-grid two">
|
| 190 |
+
<label>Minimum rate<input id="minRate" type="number" min="0.05" step="0.1" value="0.25" /><span class="unit">req/s</span></label>
|
| 191 |
+
<label>Maximum rate<input id="maxRate" type="number" min="0.1" step="1" value="20" /><span class="unit">req/s</span></label>
|
| 192 |
+
<label>Search iterations<input id="searchIter" type="number" min="2" max="12" value="7" /></label>
|
| 193 |
+
<label>Repetitions / rate<input id="repetitions" type="number" min="1" max="5" value="2" /></label>
|
| 194 |
+
</div>
|
| 195 |
+
<label>Safety headroom<input id="headroom" type="number" min="0" max="0.8" step="0.05" value="0.20" /><span class="unit">fraction</span></label>
|
| 196 |
+
<button id="capacityBtn" class="primary" disabled>Find sustainable capacity</button>
|
| 197 |
+
</aside>
|
| 198 |
+
|
| 199 |
+
<section class="panel result-panel">
|
| 200 |
+
<div class="panel-title-row"><h2>Planner result</h2><span id="plannerState" class="tag neutral">Waiting</span></div>
|
| 201 |
+
<div id="plannerEmpty" class="empty-state"><h3>No search yet</h3><p>The planner repeatedly runs the simulator at different offered loads.</p></div>
|
| 202 |
+
<div id="plannerContent" class="hidden">
|
| 203 |
+
<div class="metric-grid four">
|
| 204 |
+
<div class="metric emphasis"><span>Estimated capacity</span><strong id="pCapacity">N/A</strong></div>
|
| 205 |
+
<div class="metric"><span>Recommended load</span><strong id="pRecommended">N/A</strong></div>
|
| 206 |
+
<div class="metric"><span>Safety headroom</span><strong id="pHeadroom">N/A</strong></div>
|
| 207 |
+
<div class="metric"><span>Status</span><strong id="pStatus">N/A</strong></div>
|
| 208 |
+
</div>
|
| 209 |
+
<div class="planner-note" id="plannerCriterion">A rate passes only if every repetition meets the target and drains all generated requests.</div>
|
| 210 |
+
<div class="chart-card full" data-chart-card data-chart-name="capacity-slo-curve">
|
| 211 |
+
<div class="chart-head"><div class="chart-title">SLO attainment across searched rates</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 212 |
+
<div class="chart-body large"><canvas id="capacityChart"></canvas></div>
|
| 213 |
+
</div>
|
| 214 |
+
<div class="table-toolbar"><span>Search trace</span><div><button id="capacityCopyBtn" class="mini-button" disabled>Copy table</button><button id="capacityCsvBtn" class="mini-button" disabled>Download CSV</button></div></div>
|
| 215 |
+
<div class="table-wrap"><table><thead><tr><th>Rate</th><th>Pass</th><th>Mean SLO</th><th>Worst repetition</th><th>Target</th><th>SLO range</th><th>Goodput</th><th>p95 TTFT</th><th>p95 E2E</th></tr></thead><tbody id="capacityRows"></tbody></table></div>
|
| 216 |
+
</div>
|
| 217 |
+
</section>
|
| 218 |
+
</div>
|
| 219 |
+
</section>
|
| 220 |
+
|
| 221 |
+
<section id="modern" class="tab-panel">
|
| 222 |
+
<div class="panel wide-panel">
|
| 223 |
+
<div class="panel-title-row arena-title-row">
|
| 224 |
+
<div><div class="section-kicker">P/D disaggregation + prefix reuse</div><h2>Modern Serving Lab</h2><p class="muted">Compare colocated and prefill/decode-disaggregated serving, each with and without the current shared-prefix reuse scenario. P/D worker counts and interconnect settings come from Serving Lab.</p></div>
|
| 225 |
+
<button id="topologyBtn" class="primary compact" disabled>Compare 4 scenarios</button>
|
| 226 |
+
</div>
|
| 227 |
+
<div id="topologyEmpty" class="empty-state small"><h3>No topology study yet</h3><p>Set shared-prefix and P/D parameters in Serving Lab, then run this controlled comparison.</p></div>
|
| 228 |
+
<div id="topologyContent" class="hidden">
|
| 229 |
+
<div class="chart-card full" data-chart-card data-chart-name="topology-goodput-ttft">
|
| 230 |
+
<div class="chart-head"><div class="chart-title">Goodput vs p95 TTFT</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 231 |
+
<div class="chart-body large"><canvas id="topologyChart"></canvas></div>
|
| 232 |
+
</div>
|
| 233 |
+
<div class="table-toolbar"><span>Topology and cache comparison</span><div><button id="topologyCopyBtn" class="mini-button" disabled>Copy table</button><button id="topologyCsvBtn" class="mini-button" disabled>Download CSV</button></div></div>
|
| 234 |
+
<div class="table-wrap"><table><thead><tr><th>Scenario</th><th>GPU instances</th><th>Goodput</th><th>Goodput / GPU</th><th>SLO attainment</th><th>p95 TTFT</th><th>p95 E2E</th><th>p95 KV transfer</th><th>Cache hit</th><th>Prefill saved</th><th>Diagnosis</th></tr></thead><tbody id="topologyRows"></tbody></table></div>
|
| 235 |
+
</div>
|
| 236 |
+
</div>
|
| 237 |
+
</section>
|
| 238 |
+
|
| 239 |
+
<section id="design" class="tab-panel">
|
| 240 |
+
<div class="panel wide-panel">
|
| 241 |
+
<div class="panel-title-row arena-title-row">
|
| 242 |
+
<div><div class="section-kicker">Bounded what-if sweep</div><h2>Design Explorer</h2><p class="muted">Sweep 10 colocated candidates plus six P/D/cache variants, then identify the non-dominated goodput/TTFT frontier. This is an interactive design study, not an exhaustive optimizer.</p></div>
|
| 243 |
+
<div class="action-stack"><label class="checkline"><input id="includePd" type="checkbox" checked /> Include P/D candidates</label><button id="designBtn" class="primary compact" disabled>Explore design space</button></div>
|
| 244 |
+
</div>
|
| 245 |
+
<div id="designEmpty" class="empty-state small"><h3>No design sweep yet</h3><p>The current workload, model, SLO, and P/D parameters are reused from Serving Lab.</p></div>
|
| 246 |
+
<div id="designContent" class="hidden">
|
| 247 |
+
<div class="metric-grid four"><div class="metric emphasis"><span>Candidates</span><strong id="dCandidates">N/A</strong></div><div class="metric"><span>Performance Pareto</span><strong id="dPareto">N/A</strong></div><div class="metric"><span>Efficiency Pareto</span><strong id="dEfficiencyPareto">N/A</strong></div><div class="metric"><span>Profile</span><strong class="metric-small">Analytical</strong></div></div>
|
| 248 |
+
<div class="chart-grid">
|
| 249 |
+
<div class="chart-card" data-chart-card data-chart-name="design-performance-pareto-frontier">
|
| 250 |
+
<div class="chart-head"><div class="chart-title">Raw goodput vs p95 TTFT</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 251 |
+
<div class="chart-body large"><canvas id="designChart"></canvas></div>
|
| 252 |
+
</div>
|
| 253 |
+
<div class="chart-card" data-chart-card data-chart-name="design-efficiency-pareto-frontier">
|
| 254 |
+
<div class="chart-head"><div class="chart-title">Goodput per accelerator vs p95 TTFT</div><div class="chart-actions"><button class="chart-download" type="button">Download PNG</button><button class="chart-expand" type="button">Expand</button></div></div>
|
| 255 |
+
<div class="chart-body large"><canvas id="designEfficiencyChart"></canvas></div>
|
| 256 |
+
</div>
|
| 257 |
+
</div>
|
| 258 |
+
<div class="table-toolbar"><span>Candidate configurations</span><div><button id="designCopyBtn" class="mini-button" disabled>Copy table</button><button id="designCsvBtn" class="mini-button" disabled>Download CSV</button></div></div>
|
| 259 |
+
<div class="table-wrap"><table><thead><tr><th>Candidate</th><th>Perf Pareto</th><th>Efficiency Pareto</th><th>SLO pass</th><th>GPU instances</th><th>Goodput</th><th>Goodput / GPU</th><th>p95 TTFT</th><th>p95 E2E</th><th>KV peak</th><th>Diagnosis</th></tr></thead><tbody id="designRows"></tbody></table></div>
|
| 260 |
+
</div>
|
| 261 |
+
</div>
|
| 262 |
+
</section>
|
| 263 |
+
|
| 264 |
+
<section id="method" class="tab-panel">
|
| 265 |
+
<div class="method-grid">
|
| 266 |
+
<article class="panel prose"><div class="section-kicker">Simulation core</div><h2>What is actually simulated?</h2><p>Requests are generated from deterministic workload distributions and advanced through virtual time. Colocated runs model admission, prefill, paged KV allocation, dynamic decode batches, and completion. P/D runs use separate prefill and decode worker pools plus an explicit serialized KV-transfer link.</p><div class="formula">request -> queue -> prefill -> [KV transfer] -> decode -> completion</div></article>
|
| 267 |
+
<article class="panel prose"><div class="section-kicker">Prefix reuse</div><h2>Cache without pretending to implement a radix tree</h2><p>v0.3 models a single shared prompt prefix with configurable length and reuse fraction. Cache hits avoid redundant prefill work and share one persistent KV allocation. It is deliberately a controlled what-if abstraction, not a claim to reproduce SGLang's full RadixAttention policy.</p></article>
|
| 268 |
+
<article class="panel prose"><div class="section-kicker">P/D disaggregation</div><h2>Role-specific resources and transfer cost</h2><p>Prefill and decode have separate accelerator profiles and worker counts. Prompt KV state crosses a modeled interconnect before decode admission. The simulator reports role utilization, transfer latency, and transfer pressure so the benefit of isolation can be weighed against data-movement overhead.</p></article>
|
| 269 |
+
<article class="panel prose"><div class="section-kicker">Design search</div><h2>Pareto, not one magic configuration</h2><p>The Design Explorer reports both a raw-performance frontier and a resource-normalized frontier using goodput per accelerator. That prevents a multi-GPU P/D layout from looking unconditionally better merely because it uses more simulated hardware.</p></article>
|
| 270 |
+
<article class="panel prose wide-method"><div class="section-kicker">Research lineage</div><h2>Why these features are in v0.3</h2><p>Vidur established the value of simulation for avoiding expensive deployment sweeps. Recent systems have pushed toward heterogeneous and disaggregated serving, communication-aware modeling, runtime optimizations, and SLA-dependent design-space exploration. InferScale-Sim remains intentionally smaller and inspectable.</p><div class="paper-grid"><div><strong>Vidur / 2024</strong><span>Predictive profiling, workload-aware serving simulation, configuration search.</span></div><div><strong>TokenSim / 2025</strong><span>Extensible scheduling and memory-management simulation.</span></div><div><strong>Revati / 2026</strong><span>GPU-free time-warp emulation of serving control logic.</span></div><div><strong>LLMServingSim 2.0 / 2026</strong><span>Heterogeneous and disaggregated infrastructure, memory and communication.</span></div><div><strong>Frontier / May 2026</strong><span>P/D disaggregation, runtime optimizations, stateful workloads, Pareto exploration.</span></div><div><strong>HeteroPanacea / Aug 2026</strong><span>Heterogeneous stage specialization motivates resource-aware P/D comparison.</span></div><div><strong>SGLang / RadixAttention</strong><span>Automatic shared-prefix KV reuse motivates the controlled cache scenario.</span></div></div></article>
|
| 271 |
+
</div>
|
| 272 |
</section>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
</main>
|
| 274 |
+
|
| 275 |
+
<div id="toast" class="toast" role="status"></div>
|
| 276 |
+
<script src="app.js"></script>
|
| 277 |
</body>
|
| 278 |
</html>
|
py/inferscale/__init__.py
CHANGED
|
@@ -1,15 +1,17 @@
|
|
| 1 |
from .api import execute, metadata
|
| 2 |
from .models import SimulationConfig
|
| 3 |
-
from .optimizer import capacity_search, compare_schedulers
|
| 4 |
from .simulator import run_simulation
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"SimulationConfig",
|
| 8 |
"capacity_search",
|
| 9 |
"compare_schedulers",
|
|
|
|
|
|
|
| 10 |
"execute",
|
| 11 |
"metadata",
|
| 12 |
"run_simulation",
|
| 13 |
]
|
| 14 |
|
| 15 |
-
__version__ = "0.
|
|
|
|
| 1 |
from .api import execute, metadata
|
| 2 |
from .models import SimulationConfig
|
| 3 |
+
from .optimizer import capacity_search, compare_schedulers, compare_topologies, design_space_search
|
| 4 |
from .simulator import run_simulation
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"SimulationConfig",
|
| 8 |
"capacity_search",
|
| 9 |
"compare_schedulers",
|
| 10 |
+
"compare_topologies",
|
| 11 |
+
"design_space_search",
|
| 12 |
"execute",
|
| 13 |
"metadata",
|
| 14 |
"run_simulation",
|
| 15 |
]
|
| 16 |
|
| 17 |
+
__version__ = "0.3.0"
|
py/inferscale/api.py
CHANGED
|
@@ -1,16 +1,17 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
from .optimizer import capacity_search, compare_schedulers
|
| 4 |
from .profiles import ACCELERATORS, MODELS
|
| 5 |
from .simulator import SCHEDULERS, run_simulation
|
| 6 |
|
| 7 |
|
| 8 |
def metadata() -> dict:
|
| 9 |
return {
|
| 10 |
-
"version": "0.
|
| 11 |
"models": list(MODELS.keys()),
|
| 12 |
"accelerators": list(ACCELERATORS.keys()),
|
| 13 |
"schedulers": sorted(SCHEDULERS),
|
|
|
|
| 14 |
"profile_type": "analytical-reference",
|
| 15 |
}
|
| 16 |
|
|
@@ -30,8 +31,13 @@ def execute(action: str, payload: dict) -> dict:
|
|
| 30 |
)
|
| 31 |
if action == "compare":
|
| 32 |
config = payload.get("config", payload)
|
| 33 |
-
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
if action == "metadata":
|
| 36 |
return metadata()
|
| 37 |
raise ValueError(f"Unknown action: {action}")
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
from .optimizer import capacity_search, compare_schedulers, compare_topologies, design_space_search
|
| 4 |
from .profiles import ACCELERATORS, MODELS
|
| 5 |
from .simulator import SCHEDULERS, run_simulation
|
| 6 |
|
| 7 |
|
| 8 |
def metadata() -> dict:
|
| 9 |
return {
|
| 10 |
+
"version": "0.3.0",
|
| 11 |
"models": list(MODELS.keys()),
|
| 12 |
"accelerators": list(ACCELERATORS.keys()),
|
| 13 |
"schedulers": sorted(SCHEDULERS),
|
| 14 |
+
"topologies": ["colocated", "disaggregated_pd"],
|
| 15 |
"profile_type": "analytical-reference",
|
| 16 |
}
|
| 17 |
|
|
|
|
| 31 |
)
|
| 32 |
if action == "compare":
|
| 33 |
config = payload.get("config", payload)
|
| 34 |
+
return compare_schedulers(config, payload.get("schedulers"))
|
| 35 |
+
if action == "topology_compare":
|
| 36 |
+
config = payload.get("config", payload)
|
| 37 |
+
return compare_topologies(config)
|
| 38 |
+
if action == "design_space":
|
| 39 |
+
config = payload.get("config", payload)
|
| 40 |
+
return design_space_search(config, bool(payload.get("include_disaggregated", True)))
|
| 41 |
if action == "metadata":
|
| 42 |
return metadata()
|
| 43 |
raise ValueError(f"Unknown action: {action}")
|
py/inferscale/diagnostics.py
CHANGED
|
@@ -6,7 +6,13 @@ from .models import SimulationConfig
|
|
| 6 |
|
| 7 |
|
| 8 |
def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dict[str, Any], cfg: SimulationConfig) -> dict[str, Any]:
|
| 9 |
-
"""Explain the simulator's limiting regime from its own state.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
target = cfg.slo_attainment_target
|
| 11 |
attainment = float(summary.get("slo_attainment", 0.0))
|
| 12 |
ttft_attainment = float(summary.get("ttft_slo_attainment", 0.0))
|
|
@@ -15,37 +21,53 @@ def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dic
|
|
| 15 |
busy = float(summary.get("busy_fraction", 0.0))
|
| 16 |
kv_util = float(resource.get("peak_kv_utilization", 0.0))
|
| 17 |
queue_p95 = float(latency.get("queue_ms", {}).get("p95", 0.0))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
code = "mixed_pressure"
|
| 20 |
label = "Mixed SLO pressure"
|
| 21 |
explanation = "Several constraints are active; inspect queueing, KV pressure, and tail latency together."
|
| 22 |
recommendation = "Compare schedulers and run the capacity planner to isolate the limiting regime."
|
| 23 |
|
| 24 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
code, label = "kv_pressure", "KV memory pressure"
|
| 26 |
explanation = "Requests remain unfinished while the simulated KV cache approaches its configured capacity."
|
| 27 |
recommendation = "Reduce sequence pressure, lower offered load, use a smaller KV block, or choose a larger-memory profile."
|
| 28 |
elif unfinished > 0 and busy >= 0.92:
|
| 29 |
code, label = "over_capacity", "Offered load above capacity"
|
| 30 |
-
explanation = "The virtual
|
| 31 |
recommendation = "Reduce offered load or use the capacity planner to find a sustainable operating point."
|
| 32 |
elif unfinished > 0:
|
| 33 |
code, label = "admission_stall", "Admission stall"
|
| 34 |
explanation = "Some requests cannot be admitted or completed under the current serving constraints."
|
| 35 |
-
recommendation = "Inspect batch-token and KV-cache limits, then compare a less restrictive
|
| 36 |
elif attainment >= target:
|
| 37 |
if busy >= 0.90:
|
| 38 |
code, label = "healthy_near_saturation", "Healthy, little headroom"
|
| 39 |
-
explanation = "The workload meets the configured SLO target, but simulated
|
| 40 |
recommendation = "Run the capacity planner before increasing traffic; the current point is close to saturation."
|
| 41 |
else:
|
| 42 |
code, label = "healthy", "Healthy"
|
| 43 |
explanation = "The workload drains and meets the configured SLO-attainment target with visible headroom."
|
| 44 |
-
recommendation = "Use the capacity planner
|
| 45 |
elif kv_util >= 0.90:
|
| 46 |
code, label = "kv_pressure", "KV memory pressure"
|
| 47 |
explanation = "Peak simulated KV utilization is high enough to constrain admission and batching flexibility."
|
| 48 |
-
recommendation = "Reduce sequence pressure, use smaller KV blocks, or select a larger-memory
|
| 49 |
elif ttft_attainment < target and queue_p95 >= max(25.0, cfg.slo_ttft_ms * 0.20):
|
| 50 |
code, label = "queue_prefill_pressure", "Queue / prefill pressure"
|
| 51 |
explanation = "TTFT misses coincide with substantial queueing before prefill begins."
|
|
@@ -53,14 +75,14 @@ def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dic
|
|
| 53 |
elif ttft_attainment < target:
|
| 54 |
code, label = "ttft_pressure", "TTFT pressure"
|
| 55 |
explanation = "First-token latency is the dominant SLO failure even though queueing is not the only contributor."
|
| 56 |
-
recommendation = "Compare prefill-oriented
|
| 57 |
elif e2e_attainment < target:
|
| 58 |
code, label = "decode_pressure", "Decode / E2E pressure"
|
| 59 |
explanation = "Most first tokens arrive within target, but end-to-end latency still misses the configured SLO."
|
| 60 |
-
recommendation = "Reduce output-length pressure or offered load; larger decode
|
| 61 |
elif busy >= 0.95:
|
| 62 |
code, label = "compute_saturation", "Compute saturation"
|
| 63 |
-
explanation = "The simulated
|
| 64 |
recommendation = "Reduce load or switch to a higher-throughput accelerator profile."
|
| 65 |
|
| 66 |
return {
|
|
@@ -76,6 +98,9 @@ def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dic
|
|
| 76 |
"peak_kv_utilization": kv_util,
|
| 77 |
"queue_p95_ms": queue_p95,
|
| 78 |
"unfinished_requests": unfinished,
|
|
|
|
|
|
|
|
|
|
| 79 |
},
|
| 80 |
"provenance": "heuristic-simulator-diagnosis",
|
| 81 |
}
|
|
|
|
| 6 |
|
| 7 |
|
| 8 |
def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dict[str, Any], cfg: SimulationConfig) -> dict[str, Any]:
|
| 9 |
+
"""Explain the simulator's limiting regime from its own state.
|
| 10 |
+
|
| 11 |
+
Diagnostics are heuristic labels over simulated telemetry; they are not
|
| 12 |
+
hardware-counter measurements and must be interpreted with the profile
|
| 13 |
+
provenance attached to every run.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
target = cfg.slo_attainment_target
|
| 17 |
attainment = float(summary.get("slo_attainment", 0.0))
|
| 18 |
ttft_attainment = float(summary.get("ttft_slo_attainment", 0.0))
|
|
|
|
| 21 |
busy = float(summary.get("busy_fraction", 0.0))
|
| 22 |
kv_util = float(resource.get("peak_kv_utilization", 0.0))
|
| 23 |
queue_p95 = float(latency.get("queue_ms", {}).get("p95", 0.0))
|
| 24 |
+
prefill_busy = float(resource.get("prefill_busy_fraction", 0.0))
|
| 25 |
+
decode_busy = float(resource.get("decode_busy_fraction", 0.0))
|
| 26 |
+
transfer_busy = float(resource.get("transfer_busy_fraction", 0.0))
|
| 27 |
+
topology = str(resource.get("topology", cfg.topology))
|
| 28 |
|
| 29 |
code = "mixed_pressure"
|
| 30 |
label = "Mixed SLO pressure"
|
| 31 |
explanation = "Several constraints are active; inspect queueing, KV pressure, and tail latency together."
|
| 32 |
recommendation = "Compare schedulers and run the capacity planner to isolate the limiting regime."
|
| 33 |
|
| 34 |
+
if topology == "disaggregated_pd" and transfer_busy >= 0.85 and attainment < target:
|
| 35 |
+
code, label = "kv_transfer_pressure", "KV transfer pressure"
|
| 36 |
+
explanation = "The simulated P/D interconnect is highly utilized and transfer time is contributing to first-token latency."
|
| 37 |
+
recommendation = "Increase link bandwidth, reduce prompt pressure, or compare against colocated serving."
|
| 38 |
+
elif topology == "disaggregated_pd" and prefill_busy >= 0.92 and prefill_busy > decode_busy + 0.08:
|
| 39 |
+
code, label = "prefill_pool_pressure", "Prefill pool pressure"
|
| 40 |
+
explanation = "Prefill workers are the hottest role in the disaggregated pipeline."
|
| 41 |
+
recommendation = "Add prefill capacity, use prefix reuse, or reduce prompt-side pressure."
|
| 42 |
+
elif topology == "disaggregated_pd" and decode_busy >= 0.92 and decode_busy > prefill_busy + 0.08:
|
| 43 |
+
code, label = "decode_pool_pressure", "Decode pool pressure"
|
| 44 |
+
explanation = "Decode workers are the hottest role in the disaggregated pipeline."
|
| 45 |
+
recommendation = "Add decode capacity or reduce output-length / offered-load pressure."
|
| 46 |
+
elif unfinished > 0 and kv_util >= 0.85:
|
| 47 |
code, label = "kv_pressure", "KV memory pressure"
|
| 48 |
explanation = "Requests remain unfinished while the simulated KV cache approaches its configured capacity."
|
| 49 |
recommendation = "Reduce sequence pressure, lower offered load, use a smaller KV block, or choose a larger-memory profile."
|
| 50 |
elif unfinished > 0 and busy >= 0.92:
|
| 51 |
code, label = "over_capacity", "Offered load above capacity"
|
| 52 |
+
explanation = "The virtual serving system remains busy while the workload fails to drain."
|
| 53 |
recommendation = "Reduce offered load or use the capacity planner to find a sustainable operating point."
|
| 54 |
elif unfinished > 0:
|
| 55 |
code, label = "admission_stall", "Admission stall"
|
| 56 |
explanation = "Some requests cannot be admitted or completed under the current serving constraints."
|
| 57 |
+
recommendation = "Inspect batch-token and KV-cache limits, then compare a less restrictive configuration."
|
| 58 |
elif attainment >= target:
|
| 59 |
if busy >= 0.90:
|
| 60 |
code, label = "healthy_near_saturation", "Healthy, little headroom"
|
| 61 |
+
explanation = "The workload meets the configured SLO target, but simulated resource utilization is already high."
|
| 62 |
recommendation = "Run the capacity planner before increasing traffic; the current point is close to saturation."
|
| 63 |
else:
|
| 64 |
code, label = "healthy", "Healthy"
|
| 65 |
explanation = "The workload drains and meets the configured SLO-attainment target with visible headroom."
|
| 66 |
+
recommendation = "Use the capacity planner or design explorer to quantify additional headroom."
|
| 67 |
elif kv_util >= 0.90:
|
| 68 |
code, label = "kv_pressure", "KV memory pressure"
|
| 69 |
explanation = "Peak simulated KV utilization is high enough to constrain admission and batching flexibility."
|
| 70 |
+
recommendation = "Reduce sequence pressure, use smaller KV blocks, enable reusable prefixes, or select a larger-memory profile."
|
| 71 |
elif ttft_attainment < target and queue_p95 >= max(25.0, cfg.slo_ttft_ms * 0.20):
|
| 72 |
code, label = "queue_prefill_pressure", "Queue / prefill pressure"
|
| 73 |
explanation = "TTFT misses coincide with substantial queueing before prefill begins."
|
|
|
|
| 75 |
elif ttft_attainment < target:
|
| 76 |
code, label = "ttft_pressure", "TTFT pressure"
|
| 77 |
explanation = "First-token latency is the dominant SLO failure even though queueing is not the only contributor."
|
| 78 |
+
recommendation = "Compare prefill-oriented scheduling, prefix reuse, and P/D disaggregation scenarios."
|
| 79 |
elif e2e_attainment < target:
|
| 80 |
code, label = "decode_pressure", "Decode / E2E pressure"
|
| 81 |
explanation = "Most first tokens arrive within target, but end-to-end latency still misses the configured SLO."
|
| 82 |
+
recommendation = "Reduce output-length pressure or offered load; larger decode capacity may help when memory allows."
|
| 83 |
elif busy >= 0.95:
|
| 84 |
code, label = "compute_saturation", "Compute saturation"
|
| 85 |
+
explanation = "The simulated serving resources are effectively saturated even though no single SLO component dominates."
|
| 86 |
recommendation = "Reduce load or switch to a higher-throughput accelerator profile."
|
| 87 |
|
| 88 |
return {
|
|
|
|
| 98 |
"peak_kv_utilization": kv_util,
|
| 99 |
"queue_p95_ms": queue_p95,
|
| 100 |
"unfinished_requests": unfinished,
|
| 101 |
+
"prefill_busy_fraction": prefill_busy,
|
| 102 |
+
"decode_busy_fraction": decode_busy,
|
| 103 |
+
"transfer_busy_fraction": transfer_busy,
|
| 104 |
},
|
| 105 |
"provenance": "heuristic-simulator-diagnosis",
|
| 106 |
}
|
py/inferscale/disaggregated.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import heapq
|
| 4 |
+
from dataclasses import asdict, dataclass, field
|
| 5 |
+
from itertools import count
|
| 6 |
+
|
| 7 |
+
from .diagnostics import diagnose_run
|
| 8 |
+
from .kv_cache import KVCacheModel
|
| 9 |
+
from .latency import AnalyticalLatencyModel
|
| 10 |
+
from .metrics import percentile, summarize
|
| 11 |
+
from .models import Request, SimulationConfig, SimulationResult, TimelinePoint
|
| 12 |
+
from .profiles import get_accelerator, get_model
|
| 13 |
+
from .workloads import generate_workload
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass
|
| 17 |
+
class PrefillWorker:
|
| 18 |
+
worker_id: int
|
| 19 |
+
latency: AnalyticalLatencyModel
|
| 20 |
+
kv: KVCacheModel
|
| 21 |
+
busy: bool = False
|
| 22 |
+
busy_time_s: float = 0.0
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@dataclass
|
| 26 |
+
class DecodeWorker:
|
| 27 |
+
worker_id: int
|
| 28 |
+
latency: AnalyticalLatencyModel
|
| 29 |
+
kv: KVCacheModel
|
| 30 |
+
active: list[Request] = field(default_factory=list)
|
| 31 |
+
busy: bool = False
|
| 32 |
+
busy_time_s: float = 0.0
|
| 33 |
+
peak_kv_gb: float = 0.0
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class DisaggregatedSimulator:
|
| 37 |
+
"""Two-stage prefill/decode discrete-event simulator.
|
| 38 |
+
|
| 39 |
+
v0.3 models role-specific worker pools plus a serialized analytical KV-transfer
|
| 40 |
+
link. It is intentionally a systems abstraction, not a distributed-runtime
|
| 41 |
+
emulator: transport, compute and memory timings remain reference-model
|
| 42 |
+
predictions and carry explicit provenance in every result.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
def __init__(self, cfg: SimulationConfig):
|
| 46 |
+
if cfg.prefill_workers < 1 or cfg.decode_workers < 1:
|
| 47 |
+
raise ValueError("prefill_workers and decode_workers must be >= 1")
|
| 48 |
+
if cfg.interconnect_gbps <= 0:
|
| 49 |
+
raise ValueError("interconnect_gbps must be > 0")
|
| 50 |
+
|
| 51 |
+
self.cfg = cfg
|
| 52 |
+
self.model = get_model(cfg.model)
|
| 53 |
+
self.prefill_accelerator = get_accelerator(cfg.prefill_accelerator)
|
| 54 |
+
self.decode_accelerator = get_accelerator(cfg.decode_accelerator)
|
| 55 |
+
self.prefill_latency = AnalyticalLatencyModel(self.model, self.prefill_accelerator, cfg.quantization)
|
| 56 |
+
self.decode_latency = AnalyticalLatencyModel(self.model, self.decode_accelerator, cfg.quantization)
|
| 57 |
+
|
| 58 |
+
self.prefill_workers = [
|
| 59 |
+
PrefillWorker(i, self.prefill_latency, KVCacheModel(self.prefill_latency, cfg))
|
| 60 |
+
for i in range(cfg.prefill_workers)
|
| 61 |
+
]
|
| 62 |
+
self.decode_workers = [
|
| 63 |
+
DecodeWorker(i, self.decode_latency, KVCacheModel(self.decode_latency, cfg))
|
| 64 |
+
for i in range(cfg.decode_workers)
|
| 65 |
+
]
|
| 66 |
+
|
| 67 |
+
self.requests = generate_workload(cfg)
|
| 68 |
+
self.prefill_waiting: list[Request] = []
|
| 69 |
+
self.decode_waiting: list[Request] = []
|
| 70 |
+
self.completed: list[Request] = []
|
| 71 |
+
self.events: list[tuple[float, int, str, object]] = []
|
| 72 |
+
self.seq = count()
|
| 73 |
+
self.now = 0.0
|
| 74 |
+
self.transfer_busy_until = 0.0
|
| 75 |
+
self.transfer_busy_time_s = 0.0
|
| 76 |
+
self.transfer_gb = 0.0
|
| 77 |
+
self.transfer_latencies_ms: list[float] = []
|
| 78 |
+
self.transfer_pending = 0
|
| 79 |
+
self.timeline: list[TimelinePoint] = []
|
| 80 |
+
self.peak_total_kv_gb = 0.0
|
| 81 |
+
self.warnings: list[str] = []
|
| 82 |
+
|
| 83 |
+
for req in self.requests:
|
| 84 |
+
self._push(req.arrival_time, "arrival", req)
|
| 85 |
+
|
| 86 |
+
def _push(self, time_s: float, kind: str, payload: object) -> None:
|
| 87 |
+
heapq.heappush(self.events, (max(time_s, self.now), next(self.seq), kind, payload))
|
| 88 |
+
|
| 89 |
+
def _waiting_sorted(self, requests: list[Request]) -> list[Request]:
|
| 90 |
+
if self.cfg.scheduler == "continuous_sjf":
|
| 91 |
+
return sorted(requests, key=lambda r: (r.remaining_prefill + r.output_tokens, r.arrival_time))
|
| 92 |
+
if self.cfg.scheduler in {"continuous_slo", "chunked_slo"}:
|
| 93 |
+
def slack(req: Request) -> tuple[float, float]:
|
| 94 |
+
prefill = self.prefill_latency.prefill_seconds([max(req.remaining_prefill, 1)])
|
| 95 |
+
midpoint_context = req.prompt_tokens + max(req.output_tokens // 2, 1)
|
| 96 |
+
decode = req.output_tokens * self.decode_latency.decode_step_seconds([midpoint_context])
|
| 97 |
+
return (req.deadline_time - self.now - prefill - decode, req.arrival_time)
|
| 98 |
+
|
| 99 |
+
return sorted(requests, key=slack)
|
| 100 |
+
return sorted(requests, key=lambda r: r.arrival_time)
|
| 101 |
+
|
| 102 |
+
def _record_timeline(self, force: bool = False) -> None:
|
| 103 |
+
target = max(self.cfg.timeline_points, 20)
|
| 104 |
+
total_kv = sum(w.kv.used_gb(w.active) for w in self.decode_workers)
|
| 105 |
+
total_capacity = sum(w.kv.capacity_gb for w in self.decode_workers)
|
| 106 |
+
self.peak_total_kv_gb = max(self.peak_total_kv_gb, total_kv)
|
| 107 |
+
for worker in self.decode_workers:
|
| 108 |
+
worker.peak_kv_gb = max(worker.peak_kv_gb, worker.kv.used_gb(worker.active))
|
| 109 |
+
point = TimelinePoint(
|
| 110 |
+
time_s=self.now,
|
| 111 |
+
waiting=len(self.prefill_waiting),
|
| 112 |
+
prefill_pending=sum(1 for w in self.prefill_workers if w.busy),
|
| 113 |
+
decoding=sum(len(w.active) for w in self.decode_workers),
|
| 114 |
+
completed=len(self.completed),
|
| 115 |
+
kv_used_gb=total_kv,
|
| 116 |
+
kv_capacity_gb=total_capacity,
|
| 117 |
+
transfer_pending=self.transfer_pending,
|
| 118 |
+
decode_ready=len(self.decode_waiting),
|
| 119 |
+
prefill_active=sum(1 for w in self.prefill_workers if w.busy),
|
| 120 |
+
)
|
| 121 |
+
if not self.timeline or force or self.now - self.timeline[-1].time_s >= max(self.cfg.duration_s / target, 0.05):
|
| 122 |
+
self.timeline.append(point)
|
| 123 |
+
if len(self.timeline) > target * 2:
|
| 124 |
+
self.timeline = self.timeline[::2]
|
| 125 |
+
|
| 126 |
+
def _prefill_batch_for(self, worker: PrefillWorker) -> list[tuple[Request, int]]:
|
| 127 |
+
if not self.prefill_waiting:
|
| 128 |
+
return []
|
| 129 |
+
selected: list[tuple[Request, int]] = []
|
| 130 |
+
budget = self.cfg.max_batch_tokens
|
| 131 |
+
virtual_selected: list[Request] = []
|
| 132 |
+
for req in self._waiting_sorted(self.prefill_waiting):
|
| 133 |
+
if len(selected) >= self.cfg.max_batch_size or budget <= 0:
|
| 134 |
+
break
|
| 135 |
+
chunk = req.remaining_prefill
|
| 136 |
+
if self.cfg.scheduler == "chunked_slo":
|
| 137 |
+
chunk = min(chunk, self.cfg.chunk_size)
|
| 138 |
+
chunk = min(chunk, budget)
|
| 139 |
+
if chunk <= 0:
|
| 140 |
+
continue
|
| 141 |
+
if not worker.kv.can_admit(req, [], virtual_selected):
|
| 142 |
+
continue
|
| 143 |
+
selected.append((req, chunk))
|
| 144 |
+
virtual_selected.append(req)
|
| 145 |
+
budget -= chunk
|
| 146 |
+
return selected
|
| 147 |
+
|
| 148 |
+
def _try_start_prefill(self) -> None:
|
| 149 |
+
for worker in self.prefill_workers:
|
| 150 |
+
if worker.busy:
|
| 151 |
+
continue
|
| 152 |
+
batch = self._prefill_batch_for(worker)
|
| 153 |
+
if not batch:
|
| 154 |
+
continue
|
| 155 |
+
selected_ids = {r.request_id for r, _ in batch}
|
| 156 |
+
self.prefill_waiting = [r for r in self.prefill_waiting if r.request_id not in selected_ids]
|
| 157 |
+
for req, _ in batch:
|
| 158 |
+
if req.first_prefill_time is None:
|
| 159 |
+
req.first_prefill_time = self.now
|
| 160 |
+
delta = worker.latency.prefill_seconds([chunk for _, chunk in batch])
|
| 161 |
+
worker.busy = True
|
| 162 |
+
worker.busy_time_s += delta
|
| 163 |
+
self._push(self.now + delta, "prefill_done", (worker.worker_id, batch))
|
| 164 |
+
|
| 165 |
+
def _schedule_transfer(self, req: Request) -> None:
|
| 166 |
+
# Cached shared-prefix KV is assumed resident in both role pools. Only
|
| 167 |
+
# newly computed prompt state must cross the P/D boundary.
|
| 168 |
+
bytes_to_transfer = req.uncached_prompt_tokens * self.prefill_latency.kv_bytes_per_token()
|
| 169 |
+
gb = bytes_to_transfer / 1e9
|
| 170 |
+
start = max(self.now, self.transfer_busy_until)
|
| 171 |
+
duration = self.cfg.transfer_base_ms / 1000.0 + bytes_to_transfer / (self.cfg.interconnect_gbps * 1e9)
|
| 172 |
+
end = start + duration
|
| 173 |
+
self.transfer_busy_until = end
|
| 174 |
+
self.transfer_busy_time_s += duration
|
| 175 |
+
self.transfer_gb += gb
|
| 176 |
+
self.transfer_latencies_ms.append(duration * 1000.0)
|
| 177 |
+
self.transfer_pending += 1
|
| 178 |
+
req.transfer_start_time = start
|
| 179 |
+
req.transfer_end_time = end
|
| 180 |
+
self._push(end, "transfer_done", req)
|
| 181 |
+
|
| 182 |
+
def _decode_order(self) -> list[Request]:
|
| 183 |
+
return self._waiting_sorted(self.decode_waiting)
|
| 184 |
+
|
| 185 |
+
def _try_schedule_decode(self) -> None:
|
| 186 |
+
# Admissions occur only between iterations. A worker marked busy has a
|
| 187 |
+
# decode step already in flight and cannot accept work until it completes.
|
| 188 |
+
for worker in self.decode_workers:
|
| 189 |
+
if worker.busy:
|
| 190 |
+
continue
|
| 191 |
+
slots = self.cfg.max_batch_size - len(worker.active)
|
| 192 |
+
if slots > 0 and self.decode_waiting:
|
| 193 |
+
for req in list(self._decode_order()):
|
| 194 |
+
if slots <= 0:
|
| 195 |
+
break
|
| 196 |
+
if worker.kv.can_admit(req, worker.active):
|
| 197 |
+
self.decode_waiting.remove(req)
|
| 198 |
+
req.decode_worker_id = worker.worker_id
|
| 199 |
+
worker.active.append(req)
|
| 200 |
+
slots -= 1
|
| 201 |
+
if not worker.active:
|
| 202 |
+
continue
|
| 203 |
+
contexts = [r.context_tokens for r in worker.active]
|
| 204 |
+
delta = worker.latency.decode_step_seconds(contexts)
|
| 205 |
+
worker.busy = True
|
| 206 |
+
worker.busy_time_s += delta
|
| 207 |
+
self._push(self.now + delta, "decode_done", worker.worker_id)
|
| 208 |
+
|
| 209 |
+
def _handle_event(self, kind: str, payload: object) -> None:
|
| 210 |
+
if kind == "arrival":
|
| 211 |
+
self.prefill_waiting.append(payload) # type: ignore[arg-type]
|
| 212 |
+
self._try_start_prefill()
|
| 213 |
+
return
|
| 214 |
+
|
| 215 |
+
if kind == "prefill_done":
|
| 216 |
+
worker_id, batch = payload # type: ignore[misc]
|
| 217 |
+
worker = self.prefill_workers[worker_id]
|
| 218 |
+
worker.busy = False
|
| 219 |
+
for req, chunk in batch:
|
| 220 |
+
req.remaining_prefill = max(0, req.remaining_prefill - chunk)
|
| 221 |
+
if req.remaining_prefill > 0:
|
| 222 |
+
self.prefill_waiting.append(req)
|
| 223 |
+
else:
|
| 224 |
+
req.prefill_complete_time = self.now
|
| 225 |
+
self._schedule_transfer(req)
|
| 226 |
+
self._try_start_prefill()
|
| 227 |
+
return
|
| 228 |
+
|
| 229 |
+
if kind == "transfer_done":
|
| 230 |
+
req = payload # type: ignore[assignment]
|
| 231 |
+
self.transfer_pending = max(0, self.transfer_pending - 1)
|
| 232 |
+
self.decode_waiting.append(req)
|
| 233 |
+
self._try_schedule_decode()
|
| 234 |
+
return
|
| 235 |
+
|
| 236 |
+
if kind == "decode_done":
|
| 237 |
+
worker = self.decode_workers[int(payload)]
|
| 238 |
+
worker.busy = False
|
| 239 |
+
for req in worker.active:
|
| 240 |
+
req.generated_tokens += 1
|
| 241 |
+
if req.first_token_time is None:
|
| 242 |
+
req.first_token_time = self.now
|
| 243 |
+
done = [r for r in worker.active if r.complete]
|
| 244 |
+
for req in done:
|
| 245 |
+
req.completion_time = self.now
|
| 246 |
+
self.completed.append(req)
|
| 247 |
+
if done:
|
| 248 |
+
done_ids = {r.request_id for r in done}
|
| 249 |
+
worker.active = [r for r in worker.active if r.request_id not in done_ids]
|
| 250 |
+
self._try_schedule_decode()
|
| 251 |
+
return
|
| 252 |
+
|
| 253 |
+
raise RuntimeError(f"Unknown event kind: {kind}")
|
| 254 |
+
|
| 255 |
+
def run(self) -> SimulationResult:
|
| 256 |
+
if self.cfg.scheduler == "static_fcfs":
|
| 257 |
+
self.warnings.append("Static FCFS is not defined for P/D disaggregation; using continuous FCFS semantics.")
|
| 258 |
+
self._record_timeline(force=True)
|
| 259 |
+
processed = 0
|
| 260 |
+
max_events = max(10000, len(self.requests) * max(self.cfg.output_tokens_mean, 1) * 20)
|
| 261 |
+
|
| 262 |
+
while self.events and len(self.completed) < len(self.requests):
|
| 263 |
+
time_s, _, kind, payload = heapq.heappop(self.events)
|
| 264 |
+
self.now = max(self.now, time_s)
|
| 265 |
+
self._handle_event(kind, payload)
|
| 266 |
+
self._record_timeline()
|
| 267 |
+
processed += 1
|
| 268 |
+
if processed > max_events:
|
| 269 |
+
self.warnings.append("Simulation stopped at the event safety limit.")
|
| 270 |
+
break
|
| 271 |
+
|
| 272 |
+
# If queued work remains with no events, the configuration is memory- or
|
| 273 |
+
# admission-constrained rather than silently considered complete.
|
| 274 |
+
if len(self.completed) < len(self.requests) and not self.events:
|
| 275 |
+
self.warnings.append("Disaggregated pipeline stalled before all requests completed.")
|
| 276 |
+
|
| 277 |
+
self._record_timeline(force=True)
|
| 278 |
+
makespan = max(self.now, self.cfg.duration_s if self.requests else 0.0)
|
| 279 |
+
prefill_util = sum(w.busy_time_s for w in self.prefill_workers) / max(makespan * len(self.prefill_workers), 1e-9)
|
| 280 |
+
decode_util = sum(w.busy_time_s for w in self.decode_workers) / max(makespan * len(self.decode_workers), 1e-9)
|
| 281 |
+
transfer_util = self.transfer_busy_time_s / max(makespan, 1e-9)
|
| 282 |
+
hottest = min(1.0, max(prefill_util, decode_util, transfer_util))
|
| 283 |
+
summary, latency = summarize(self.completed, self.cfg, makespan, hottest * makespan)
|
| 284 |
+
summary["requests_generated"] = len(self.requests)
|
| 285 |
+
summary["requests_unfinished"] = len(self.requests) - len(self.completed)
|
| 286 |
+
|
| 287 |
+
total_kv_capacity = sum(w.kv.capacity_gb for w in self.decode_workers)
|
| 288 |
+
peak_kv_gb = self.peak_total_kv_gb
|
| 289 |
+
peak_worker_util = max(
|
| 290 |
+
(w.peak_kv_gb / w.kv.capacity_gb if w.kv.capacity_gb > 0 else 0.0 for w in self.decode_workers),
|
| 291 |
+
default=0.0,
|
| 292 |
+
)
|
| 293 |
+
resource = {
|
| 294 |
+
"topology": "disaggregated_pd",
|
| 295 |
+
"model_weight_gb": self.decode_latency.model_weight_gb,
|
| 296 |
+
"kv_capacity_gb": total_kv_capacity,
|
| 297 |
+
"peak_kv_gb": peak_kv_gb,
|
| 298 |
+
"peak_kv_utilization": peak_worker_util,
|
| 299 |
+
"prefill_accelerator_vram_gb": self.prefill_accelerator.vram_gb,
|
| 300 |
+
"decode_accelerator_vram_gb": self.decode_accelerator.vram_gb,
|
| 301 |
+
"prefill_workers": len(self.prefill_workers),
|
| 302 |
+
"decode_workers": len(self.decode_workers),
|
| 303 |
+
"accelerator_instances": len(self.prefill_workers) + len(self.decode_workers),
|
| 304 |
+
"prefill_busy_fraction": min(1.0, prefill_util),
|
| 305 |
+
"decode_busy_fraction": min(1.0, decode_util),
|
| 306 |
+
"transfer_busy_fraction": min(1.0, transfer_util),
|
| 307 |
+
"kv_transfer_gb": self.transfer_gb,
|
| 308 |
+
"mean_transfer_ms": sum(self.transfer_latencies_ms) / len(self.transfer_latencies_ms) if self.transfer_latencies_ms else 0.0,
|
| 309 |
+
"p95_transfer_ms": percentile(self.transfer_latencies_ms, 0.95),
|
| 310 |
+
"interconnect_gbps": self.cfg.interconnect_gbps,
|
| 311 |
+
"prefix_cache_gb_per_decode_worker": self.decode_workers[0].kv.shared_prefix_gb if self.decode_workers else 0.0,
|
| 312 |
+
"prefix_cache_hits": sum(1 for r in self.requests if r.prefix_cache_hit),
|
| 313 |
+
"prefix_cache_hit_rate": (sum(1 for r in self.requests if r.prefix_cache_hit) / len(self.requests)) if self.requests else 0.0,
|
| 314 |
+
"prefill_tokens_saved": sum(r.cached_prefix_tokens for r in self.requests),
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
request_rows = []
|
| 318 |
+
for req in self.completed[:2000]:
|
| 319 |
+
transfer_ms = 0.0
|
| 320 |
+
if req.transfer_start_time is not None and req.transfer_end_time is not None:
|
| 321 |
+
transfer_ms = (req.transfer_end_time - req.transfer_start_time) * 1000.0
|
| 322 |
+
request_rows.append(
|
| 323 |
+
{
|
| 324 |
+
"request_id": req.request_id,
|
| 325 |
+
"arrival_time": req.arrival_time,
|
| 326 |
+
"prompt_tokens": req.prompt_tokens,
|
| 327 |
+
"output_tokens": req.output_tokens,
|
| 328 |
+
"cached_prefix_tokens": req.cached_prefix_tokens,
|
| 329 |
+
"decode_worker_id": req.decode_worker_id,
|
| 330 |
+
"transfer_ms": transfer_ms,
|
| 331 |
+
"ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
|
| 332 |
+
"e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
|
| 333 |
+
}
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
diagnostics = diagnose_run(summary, latency, resource, self.cfg)
|
| 337 |
+
provenance = {
|
| 338 |
+
"simulator": "InferScale-Sim",
|
| 339 |
+
"version": "0.3.0",
|
| 340 |
+
"latency_profile_type": "analytical-reference",
|
| 341 |
+
"profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
|
| 342 |
+
"model_profile_source": self.model.source,
|
| 343 |
+
"prefill_accelerator_profile_source": self.prefill_accelerator.source,
|
| 344 |
+
"decode_accelerator_profile_source": self.decode_accelerator.source,
|
| 345 |
+
"topology": "disaggregated_pd",
|
| 346 |
+
"transfer_model": "serialized-reference-link",
|
| 347 |
+
}
|
| 348 |
+
return SimulationResult(
|
| 349 |
+
config=self.cfg.to_dict(),
|
| 350 |
+
provenance=provenance,
|
| 351 |
+
summary=summary,
|
| 352 |
+
latency=latency,
|
| 353 |
+
resource=resource,
|
| 354 |
+
diagnostics=diagnostics,
|
| 355 |
+
requests=request_rows,
|
| 356 |
+
timeline=[asdict(p) for p in self.timeline],
|
| 357 |
+
warnings=self.warnings,
|
| 358 |
+
)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def run_disaggregated(config: dict) -> dict:
|
| 362 |
+
return DisaggregatedSimulator(SimulationConfig.from_dict(config)).run().to_dict()
|
py/inferscale/kv_cache.py
CHANGED
|
@@ -12,18 +12,20 @@ class KVCacheModel:
|
|
| 12 |
self.cfg = cfg
|
| 13 |
self.model_weight_gb = latency_model.model_weight_gb
|
| 14 |
total_vram = latency_model.accelerator.vram_gb
|
| 15 |
-
# Reserve non-KV space for runtime workspace and weights.
|
| 16 |
remaining = max(0.0, total_vram - self.model_weight_gb - 1.2)
|
| 17 |
self.capacity_gb = remaining * cfg.kv_memory_fraction
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
def _allocated_tokens(self, req: Request, include_output_reservation: bool = False) -> int:
|
| 20 |
-
|
|
|
|
|
|
|
| 21 |
if self.cfg.scheduler == "static_fcfs" or include_output_reservation:
|
| 22 |
-
|
| 23 |
-
# fragmentation / over-reservation versus paged allocation.
|
| 24 |
-
return req.prompt_tokens + req.output_tokens
|
| 25 |
block = max(self.cfg.kv_block_tokens, 1)
|
| 26 |
-
return int(math.ceil(live_tokens / block) * block)
|
| 27 |
|
| 28 |
def request_gb(self, req: Request, include_output_reservation: bool = False) -> float:
|
| 29 |
tokens = self._allocated_tokens(req, include_output_reservation)
|
|
@@ -33,7 +35,7 @@ class KVCacheModel:
|
|
| 33 |
requests = list(active)
|
| 34 |
if prefill_pending:
|
| 35 |
requests.extend(prefill_pending)
|
| 36 |
-
return sum(self.request_gb(r) for r in requests)
|
| 37 |
|
| 38 |
def can_admit(self, req: Request, active: list[Request], prefill_pending: list[Request] | None = None) -> bool:
|
| 39 |
return self.used_gb(active, prefill_pending) + self.request_gb(req) <= self.capacity_gb + 1e-12
|
|
|
|
| 12 |
self.cfg = cfg
|
| 13 |
self.model_weight_gb = latency_model.model_weight_gb
|
| 14 |
total_vram = latency_model.accelerator.vram_gb
|
|
|
|
| 15 |
remaining = max(0.0, total_vram - self.model_weight_gb - 1.2)
|
| 16 |
self.capacity_gb = remaining * cfg.kv_memory_fraction
|
| 17 |
+
self.shared_prefix_gb = 0.0
|
| 18 |
+
if cfg.prefix_cache_enabled and cfg.shared_prefix_tokens > 0 and cfg.prefix_reuse_fraction > 0:
|
| 19 |
+
self.shared_prefix_gb = cfg.shared_prefix_tokens * latency_model.kv_bytes_per_token() / 1e9
|
| 20 |
|
| 21 |
def _allocated_tokens(self, req: Request, include_output_reservation: bool = False) -> int:
|
| 22 |
+
# Cached prefix state is represented once by shared_prefix_gb. Per-request
|
| 23 |
+
# allocation therefore contains only the uncached suffix + generated state.
|
| 24 |
+
live_tokens = req.uncached_prompt_tokens + req.generated_tokens
|
| 25 |
if self.cfg.scheduler == "static_fcfs" or include_output_reservation:
|
| 26 |
+
return req.uncached_prompt_tokens + req.output_tokens
|
|
|
|
|
|
|
| 27 |
block = max(self.cfg.kv_block_tokens, 1)
|
| 28 |
+
return int(math.ceil(max(live_tokens, 0) / block) * block)
|
| 29 |
|
| 30 |
def request_gb(self, req: Request, include_output_reservation: bool = False) -> float:
|
| 31 |
tokens = self._allocated_tokens(req, include_output_reservation)
|
|
|
|
| 35 |
requests = list(active)
|
| 36 |
if prefill_pending:
|
| 37 |
requests.extend(prefill_pending)
|
| 38 |
+
return self.shared_prefix_gb + sum(self.request_gb(r) for r in requests)
|
| 39 |
|
| 40 |
def can_admit(self, req: Request, active: list[Request], prefill_pending: list[Request] | None = None) -> bool:
|
| 41 |
return self.used_gb(active, prefill_pending) + self.request_gb(req) <= self.capacity_gb + 1e-12
|
py/inferscale/latency.py
CHANGED
|
@@ -29,7 +29,7 @@ class AnalyticalLatencyModel:
|
|
| 29 |
return self.model.params_b * self.weight_bytes_per_param
|
| 30 |
|
| 31 |
def kv_bytes_per_token(self) -> float:
|
| 32 |
-
# K + V, all layers, KV heads only. KV state is assumed fp16 in v0.
|
| 33 |
return (
|
| 34 |
2
|
| 35 |
* self.model.layers
|
|
|
|
| 29 |
return self.model.params_b * self.weight_bytes_per_param
|
| 30 |
|
| 31 |
def kv_bytes_per_token(self) -> float:
|
| 32 |
+
# K + V, all layers, KV heads only. KV state is assumed fp16 in v0.3.
|
| 33 |
return (
|
| 34 |
2
|
| 35 |
* self.model.layers
|
py/inferscale/models.py
CHANGED
|
@@ -36,6 +36,7 @@ class SimulationConfig:
|
|
| 36 |
model: str = "Llama-3.1-8B"
|
| 37 |
accelerator: str = "L4"
|
| 38 |
scheduler: str = "continuous_fcfs"
|
|
|
|
| 39 |
arrival_process: str = "poisson"
|
| 40 |
request_rate_rps: float = 4.0
|
| 41 |
duration_s: float = 60.0
|
|
@@ -57,6 +58,21 @@ class SimulationConfig:
|
|
| 57 |
burst_period_s: float = 10.0
|
| 58 |
timeline_points: int = 300
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
@classmethod
|
| 61 |
def from_dict(cls, data: dict[str, Any]) -> "SimulationConfig":
|
| 62 |
allowed = cls.__dataclass_fields__.keys()
|
|
@@ -74,16 +90,29 @@ class Request:
|
|
| 74 |
output_tokens: int
|
| 75 |
deadline_time: float
|
| 76 |
remaining_prefill: int
|
|
|
|
| 77 |
generated_tokens: int = 0
|
| 78 |
first_prefill_time: float | None = None
|
|
|
|
|
|
|
|
|
|
| 79 |
first_token_time: float | None = None
|
| 80 |
completion_time: float | None = None
|
|
|
|
| 81 |
priority: int = 0
|
| 82 |
|
| 83 |
@property
|
| 84 |
def context_tokens(self) -> int:
|
| 85 |
return self.prompt_tokens + self.generated_tokens
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
@property
|
| 88 |
def complete(self) -> bool:
|
| 89 |
return self.generated_tokens >= self.output_tokens
|
|
@@ -113,6 +142,9 @@ class TimelinePoint:
|
|
| 113 |
completed: int
|
| 114 |
kv_used_gb: float
|
| 115 |
kv_capacity_gb: float
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
@dataclass
|
|
|
|
| 36 |
model: str = "Llama-3.1-8B"
|
| 37 |
accelerator: str = "L4"
|
| 38 |
scheduler: str = "continuous_fcfs"
|
| 39 |
+
topology: str = "colocated"
|
| 40 |
arrival_process: str = "poisson"
|
| 41 |
request_rate_rps: float = 4.0
|
| 42 |
duration_s: float = 60.0
|
|
|
|
| 58 |
burst_period_s: float = 10.0
|
| 59 |
timeline_points: int = 300
|
| 60 |
|
| 61 |
+
# Prefix-cache scenario. v0.3 intentionally models one reusable shared
|
| 62 |
+
# prefix rather than a full radix tree. Hits share one persistent KV entry.
|
| 63 |
+
prefix_cache_enabled: bool = False
|
| 64 |
+
shared_prefix_tokens: int = 256
|
| 65 |
+
prefix_reuse_fraction: float = 0.0
|
| 66 |
+
|
| 67 |
+
# Prefill/decode disaggregation scenario. These fields are ignored for the
|
| 68 |
+
# colocated topology. Interconnect bandwidth is expressed in GB/s.
|
| 69 |
+
prefill_accelerator: str = "L4"
|
| 70 |
+
decode_accelerator: str = "L4"
|
| 71 |
+
prefill_workers: int = 1
|
| 72 |
+
decode_workers: int = 1
|
| 73 |
+
interconnect_gbps: float = 50.0
|
| 74 |
+
transfer_base_ms: float = 0.20
|
| 75 |
+
|
| 76 |
@classmethod
|
| 77 |
def from_dict(cls, data: dict[str, Any]) -> "SimulationConfig":
|
| 78 |
allowed = cls.__dataclass_fields__.keys()
|
|
|
|
| 90 |
output_tokens: int
|
| 91 |
deadline_time: float
|
| 92 |
remaining_prefill: int
|
| 93 |
+
cached_prefix_tokens: int = 0
|
| 94 |
generated_tokens: int = 0
|
| 95 |
first_prefill_time: float | None = None
|
| 96 |
+
prefill_complete_time: float | None = None
|
| 97 |
+
transfer_start_time: float | None = None
|
| 98 |
+
transfer_end_time: float | None = None
|
| 99 |
first_token_time: float | None = None
|
| 100 |
completion_time: float | None = None
|
| 101 |
+
decode_worker_id: int | None = None
|
| 102 |
priority: int = 0
|
| 103 |
|
| 104 |
@property
|
| 105 |
def context_tokens(self) -> int:
|
| 106 |
return self.prompt_tokens + self.generated_tokens
|
| 107 |
|
| 108 |
+
@property
|
| 109 |
+
def uncached_prompt_tokens(self) -> int:
|
| 110 |
+
return max(0, self.prompt_tokens - self.cached_prefix_tokens)
|
| 111 |
+
|
| 112 |
+
@property
|
| 113 |
+
def prefix_cache_hit(self) -> bool:
|
| 114 |
+
return self.cached_prefix_tokens > 0
|
| 115 |
+
|
| 116 |
@property
|
| 117 |
def complete(self) -> bool:
|
| 118 |
return self.generated_tokens >= self.output_tokens
|
|
|
|
| 142 |
completed: int
|
| 143 |
kv_used_gb: float
|
| 144 |
kv_capacity_gb: float
|
| 145 |
+
transfer_pending: int = 0
|
| 146 |
+
decode_ready: int = 0
|
| 147 |
+
prefill_active: int = 0
|
| 148 |
|
| 149 |
|
| 150 |
@dataclass
|
py/inferscale/optimizer.py
CHANGED
|
@@ -40,6 +40,7 @@ def evaluate_rate(base: SimulationConfig, rate: float, repetitions: int = 3) ->
|
|
| 40 |
"max_unfinished": max(unfinished_values),
|
| 41 |
"repetitions": repetitions,
|
| 42 |
"criterion": "all-repetitions-meet-target-and-drain",
|
|
|
|
| 43 |
}
|
| 44 |
|
| 45 |
|
|
@@ -110,6 +111,7 @@ def compare_schedulers(config: dict, schedulers: list[str] | None = None) -> dic
|
|
| 110 |
"chunked_slo",
|
| 111 |
]
|
| 112 |
base = SimulationConfig.from_dict(config)
|
|
|
|
| 113 |
rows = []
|
| 114 |
for scheduler in schedulers:
|
| 115 |
cfg = deepcopy(base)
|
|
@@ -130,3 +132,143 @@ def compare_schedulers(config: dict, schedulers: list[str] | None = None) -> dic
|
|
| 130 |
)
|
| 131 |
rows.sort(key=lambda row: (row["slo_attainment"], row["goodput_rps"]), reverse=True)
|
| 132 |
return {"rows": rows}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
"max_unfinished": max(unfinished_values),
|
| 41 |
"repetitions": repetitions,
|
| 42 |
"criterion": "all-repetitions-meet-target-and-drain",
|
| 43 |
+
"target": base.slo_attainment_target,
|
| 44 |
}
|
| 45 |
|
| 46 |
|
|
|
|
| 111 |
"chunked_slo",
|
| 112 |
]
|
| 113 |
base = SimulationConfig.from_dict(config)
|
| 114 |
+
base.topology = "colocated"
|
| 115 |
rows = []
|
| 116 |
for scheduler in schedulers:
|
| 117 |
cfg = deepcopy(base)
|
|
|
|
| 132 |
)
|
| 133 |
rows.sort(key=lambda row: (row["slo_attainment"], row["goodput_rps"]), reverse=True)
|
| 134 |
return {"rows": rows}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def compare_topologies(config: dict) -> dict:
|
| 138 |
+
"""Compare colocated and P/D-disaggregated serving on one deterministic trace."""
|
| 139 |
+
base = SimulationConfig.from_dict(config)
|
| 140 |
+
rows = []
|
| 141 |
+
scenarios = [
|
| 142 |
+
("colocated", False),
|
| 143 |
+
("colocated", True),
|
| 144 |
+
("disaggregated_pd", False),
|
| 145 |
+
("disaggregated_pd", True),
|
| 146 |
+
]
|
| 147 |
+
for topology, cache_enabled in scenarios:
|
| 148 |
+
cfg = deepcopy(base)
|
| 149 |
+
cfg.topology = topology
|
| 150 |
+
cfg.prefix_cache_enabled = cache_enabled
|
| 151 |
+
if topology == "disaggregated_pd" and cfg.scheduler == "static_fcfs":
|
| 152 |
+
cfg.scheduler = "continuous_fcfs"
|
| 153 |
+
result = run_simulation(cfg.to_dict())
|
| 154 |
+
resource = result["resource"]
|
| 155 |
+
accelerator_instances = int(resource.get("accelerator_instances", 1))
|
| 156 |
+
goodput_rps = result["summary"]["goodput_rps"]
|
| 157 |
+
rows.append(
|
| 158 |
+
{
|
| 159 |
+
"scenario": f"{topology}{'_cache' if cache_enabled else ''}",
|
| 160 |
+
"topology": topology,
|
| 161 |
+
"prefix_cache": cache_enabled,
|
| 162 |
+
"accelerator_instances": accelerator_instances,
|
| 163 |
+
"goodput_rps": goodput_rps,
|
| 164 |
+
"goodput_per_accelerator": goodput_rps / max(accelerator_instances, 1),
|
| 165 |
+
"request_throughput_rps": result["summary"]["request_throughput_rps"],
|
| 166 |
+
"slo_attainment": result["summary"]["slo_attainment"],
|
| 167 |
+
"p95_ttft_ms": result["latency"]["ttft_ms"]["p95"],
|
| 168 |
+
"p95_e2e_ms": result["latency"]["e2e_ms"]["p95"],
|
| 169 |
+
"peak_kv_utilization": resource.get("peak_kv_utilization", 0.0),
|
| 170 |
+
"prefix_hit_rate": resource.get("prefix_cache_hit_rate", 0.0),
|
| 171 |
+
"prefill_tokens_saved": resource.get("prefill_tokens_saved", 0),
|
| 172 |
+
"p95_transfer_ms": resource.get("p95_transfer_ms", 0.0),
|
| 173 |
+
"prefill_busy_fraction": resource.get("prefill_busy_fraction", 0.0),
|
| 174 |
+
"decode_busy_fraction": resource.get("decode_busy_fraction", result["summary"].get("busy_fraction", 0.0)),
|
| 175 |
+
"transfer_busy_fraction": resource.get("transfer_busy_fraction", 0.0),
|
| 176 |
+
"bottleneck": result["diagnostics"]["label"],
|
| 177 |
+
}
|
| 178 |
+
)
|
| 179 |
+
rows.sort(key=lambda row: (row["slo_attainment"], row["goodput_rps"]), reverse=True)
|
| 180 |
+
return {"rows": rows}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _is_dominated(candidate: dict, rows: list[dict], throughput_key: str = "goodput_rps") -> bool:
|
| 184 |
+
for other in rows:
|
| 185 |
+
if other is candidate:
|
| 186 |
+
continue
|
| 187 |
+
no_worse = (
|
| 188 |
+
other[throughput_key] >= candidate[throughput_key]
|
| 189 |
+
and other["p95_ttft_ms"] <= candidate["p95_ttft_ms"]
|
| 190 |
+
)
|
| 191 |
+
strictly_better = (
|
| 192 |
+
other[throughput_key] > candidate[throughput_key]
|
| 193 |
+
or other["p95_ttft_ms"] < candidate["p95_ttft_ms"]
|
| 194 |
+
)
|
| 195 |
+
if no_worse and strictly_better:
|
| 196 |
+
return True
|
| 197 |
+
return False
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def design_space_search(config: dict, include_disaggregated: bool = True) -> dict:
|
| 201 |
+
"""Small browser-safe design-space sweep with a goodput/TTFT Pareto frontier.
|
| 202 |
+
|
| 203 |
+
The sweep is intentionally bounded: its purpose is to expose configuration
|
| 204 |
+
interactions interactively, not to claim exhaustive optimization.
|
| 205 |
+
"""
|
| 206 |
+
base = SimulationConfig.from_dict(config)
|
| 207 |
+
rows: list[dict] = []
|
| 208 |
+
|
| 209 |
+
colocated_candidates = []
|
| 210 |
+
for scheduler in ["continuous_fcfs", "continuous_slo", "chunked_slo"]:
|
| 211 |
+
for batch in [8, 16, 32]:
|
| 212 |
+
colocated_candidates.append((scheduler, batch, False))
|
| 213 |
+
# Prefix reuse is added as a separate systems dimension for the SLO-aware
|
| 214 |
+
# scheduler at the user's current batch size.
|
| 215 |
+
colocated_candidates.append(("continuous_slo", base.max_batch_size, True))
|
| 216 |
+
|
| 217 |
+
for scheduler, batch, cache in colocated_candidates:
|
| 218 |
+
cfg = deepcopy(base)
|
| 219 |
+
cfg.topology = "colocated"
|
| 220 |
+
cfg.scheduler = scheduler
|
| 221 |
+
cfg.max_batch_size = batch
|
| 222 |
+
cfg.prefix_cache_enabled = cache
|
| 223 |
+
result = run_simulation(cfg.to_dict())
|
| 224 |
+
rows.append(_design_row(result, cfg, f"colocated / {scheduler} / batch {batch}{' / cache' if cache else ''}"))
|
| 225 |
+
|
| 226 |
+
if include_disaggregated:
|
| 227 |
+
for prefill_workers, decode_workers in [(1, 1), (1, 2), (2, 1)]:
|
| 228 |
+
for cache in [False, True]:
|
| 229 |
+
cfg = deepcopy(base)
|
| 230 |
+
cfg.topology = "disaggregated_pd"
|
| 231 |
+
cfg.scheduler = "continuous_slo"
|
| 232 |
+
cfg.prefill_workers = prefill_workers
|
| 233 |
+
cfg.decode_workers = decode_workers
|
| 234 |
+
cfg.prefix_cache_enabled = cache
|
| 235 |
+
result = run_simulation(cfg.to_dict())
|
| 236 |
+
label = f"P/D {prefill_workers}P:{decode_workers}D{' / cache' if cache else ''}"
|
| 237 |
+
rows.append(_design_row(result, cfg, label))
|
| 238 |
+
|
| 239 |
+
for row in rows:
|
| 240 |
+
row["pareto"] = not _is_dominated(row, rows, "goodput_rps")
|
| 241 |
+
row["efficiency_pareto"] = not _is_dominated(row, rows, "goodput_per_accelerator")
|
| 242 |
+
rows.sort(key=lambda r: (not r["pareto"], not r["slo_pass"], -r["goodput_rps"], r["p95_ttft_ms"]))
|
| 243 |
+
return {
|
| 244 |
+
"rows": rows,
|
| 245 |
+
"pareto_count": sum(1 for r in rows if r["pareto"]),
|
| 246 |
+
"efficiency_pareto_count": sum(1 for r in rows if r["efficiency_pareto"]),
|
| 247 |
+
"candidate_count": len(rows),
|
| 248 |
+
"objectives": [
|
| 249 |
+
"performance: maximize goodput / minimize p95 TTFT",
|
| 250 |
+
"efficiency: maximize goodput per accelerator / minimize p95 TTFT",
|
| 251 |
+
],
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _design_row(result: dict, cfg: SimulationConfig, label: str) -> dict:
|
| 256 |
+
resource = result["resource"]
|
| 257 |
+
accelerator_instances = int(resource.get("accelerator_instances", 1))
|
| 258 |
+
goodput_rps = result["summary"]["goodput_rps"]
|
| 259 |
+
return {
|
| 260 |
+
"label": label,
|
| 261 |
+
"topology": cfg.topology,
|
| 262 |
+
"scheduler": cfg.scheduler,
|
| 263 |
+
"max_batch_size": cfg.max_batch_size,
|
| 264 |
+
"prefix_cache": cfg.prefix_cache_enabled,
|
| 265 |
+
"accelerator_instances": accelerator_instances,
|
| 266 |
+
"goodput_rps": goodput_rps,
|
| 267 |
+
"goodput_per_accelerator": goodput_rps / max(accelerator_instances, 1),
|
| 268 |
+
"slo_attainment": result["summary"]["slo_attainment"],
|
| 269 |
+
"slo_pass": result["summary"]["slo_attainment"] >= cfg.slo_attainment_target and result["summary"]["requests_unfinished"] == 0,
|
| 270 |
+
"p95_ttft_ms": result["latency"]["ttft_ms"]["p95"],
|
| 271 |
+
"p95_e2e_ms": result["latency"]["e2e_ms"]["p95"],
|
| 272 |
+
"peak_kv_utilization": resource.get("peak_kv_utilization", 0.0),
|
| 273 |
+
"bottleneck": result["diagnostics"]["label"],
|
| 274 |
+
}
|
py/inferscale/simulator.py
CHANGED
|
@@ -121,12 +121,12 @@ class Simulator:
|
|
| 121 |
for req in self._waiting_sorted():
|
| 122 |
if len(selected) >= slots:
|
| 123 |
break
|
| 124 |
-
if selected and total_tokens + req.
|
| 125 |
continue
|
| 126 |
if not self.kv.can_admit(req, self.active, selected):
|
| 127 |
continue
|
| 128 |
selected.append(req)
|
| 129 |
-
total_tokens += req.
|
| 130 |
|
| 131 |
if not selected:
|
| 132 |
return False
|
|
@@ -135,7 +135,7 @@ class Simulator:
|
|
| 135 |
for req in selected:
|
| 136 |
if req.first_prefill_time is None:
|
| 137 |
req.first_prefill_time = self.now
|
| 138 |
-
self._advance(self.latency.prefill_seconds([r.
|
| 139 |
for req in selected:
|
| 140 |
req.remaining_prefill = 0
|
| 141 |
self.active.append(req)
|
|
@@ -177,7 +177,7 @@ class Simulator:
|
|
| 177 |
# One prefill chunk. Decode is serviced on the next loop iteration,
|
| 178 |
# producing the intended prefill/decode interleaving.
|
| 179 |
self._advance(self.latency.prefill_seconds(chunks))
|
| 180 |
-
for req, chunk in zip(selected, chunks):
|
| 181 |
req.remaining_prefill -= chunk
|
| 182 |
if req.remaining_prefill <= 0:
|
| 183 |
self.prefill_pending.remove(req)
|
|
@@ -217,7 +217,7 @@ class Simulator:
|
|
| 217 |
self.waiting = [r for r in self.waiting if r.request_id not in ids]
|
| 218 |
for req in admitted:
|
| 219 |
req.first_prefill_time = self.now
|
| 220 |
-
self._advance(self.latency.prefill_seconds([r.
|
| 221 |
for req in admitted:
|
| 222 |
req.remaining_prefill = 0
|
| 223 |
self.active.append(req)
|
|
@@ -282,6 +282,12 @@ class Simulator:
|
|
| 282 |
"peak_kv_gb": self.peak_kv_gb,
|
| 283 |
"peak_kv_utilization": self.peak_kv_gb / self.kv.capacity_gb if self.kv.capacity_gb > 0 else 0.0,
|
| 284 |
"accelerator_vram_gb": self.accelerator.vram_gb,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
}
|
| 286 |
|
| 287 |
request_rows = []
|
|
@@ -293,6 +299,7 @@ class Simulator:
|
|
| 293 |
"arrival_time": req.arrival_time,
|
| 294 |
"prompt_tokens": req.prompt_tokens,
|
| 295 |
"output_tokens": req.output_tokens,
|
|
|
|
| 296 |
"ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
|
| 297 |
"e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
|
| 298 |
})
|
|
@@ -301,11 +308,12 @@ class Simulator:
|
|
| 301 |
|
| 302 |
provenance = {
|
| 303 |
"simulator": "InferScale-Sim",
|
| 304 |
-
"version": "0.
|
| 305 |
"latency_profile_type": "analytical-reference",
|
| 306 |
"profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
|
| 307 |
"model_profile_source": self.model.source,
|
| 308 |
"accelerator_profile_source": self.accelerator.source,
|
|
|
|
| 309 |
}
|
| 310 |
|
| 311 |
return SimulationResult(
|
|
@@ -322,4 +330,11 @@ class Simulator:
|
|
| 322 |
|
| 323 |
|
| 324 |
def run_simulation(config: dict) -> dict:
|
| 325 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
for req in self._waiting_sorted():
|
| 122 |
if len(selected) >= slots:
|
| 123 |
break
|
| 124 |
+
if selected and total_tokens + req.remaining_prefill > self.cfg.max_batch_tokens:
|
| 125 |
continue
|
| 126 |
if not self.kv.can_admit(req, self.active, selected):
|
| 127 |
continue
|
| 128 |
selected.append(req)
|
| 129 |
+
total_tokens += req.remaining_prefill
|
| 130 |
|
| 131 |
if not selected:
|
| 132 |
return False
|
|
|
|
| 135 |
for req in selected:
|
| 136 |
if req.first_prefill_time is None:
|
| 137 |
req.first_prefill_time = self.now
|
| 138 |
+
self._advance(self.latency.prefill_seconds([r.remaining_prefill for r in selected]))
|
| 139 |
for req in selected:
|
| 140 |
req.remaining_prefill = 0
|
| 141 |
self.active.append(req)
|
|
|
|
| 177 |
# One prefill chunk. Decode is serviced on the next loop iteration,
|
| 178 |
# producing the intended prefill/decode interleaving.
|
| 179 |
self._advance(self.latency.prefill_seconds(chunks))
|
| 180 |
+
for req, chunk in zip(selected, chunks, strict=True):
|
| 181 |
req.remaining_prefill -= chunk
|
| 182 |
if req.remaining_prefill <= 0:
|
| 183 |
self.prefill_pending.remove(req)
|
|
|
|
| 217 |
self.waiting = [r for r in self.waiting if r.request_id not in ids]
|
| 218 |
for req in admitted:
|
| 219 |
req.first_prefill_time = self.now
|
| 220 |
+
self._advance(self.latency.prefill_seconds([r.remaining_prefill for r in admitted]))
|
| 221 |
for req in admitted:
|
| 222 |
req.remaining_prefill = 0
|
| 223 |
self.active.append(req)
|
|
|
|
| 282 |
"peak_kv_gb": self.peak_kv_gb,
|
| 283 |
"peak_kv_utilization": self.peak_kv_gb / self.kv.capacity_gb if self.kv.capacity_gb > 0 else 0.0,
|
| 284 |
"accelerator_vram_gb": self.accelerator.vram_gb,
|
| 285 |
+
"topology": "colocated",
|
| 286 |
+
"accelerator_instances": 1,
|
| 287 |
+
"prefix_cache_gb": self.kv.shared_prefix_gb,
|
| 288 |
+
"prefix_cache_hits": sum(1 for r in self.requests if r.prefix_cache_hit),
|
| 289 |
+
"prefix_cache_hit_rate": (sum(1 for r in self.requests if r.prefix_cache_hit) / len(self.requests)) if self.requests else 0.0,
|
| 290 |
+
"prefill_tokens_saved": sum(r.cached_prefix_tokens for r in self.requests),
|
| 291 |
}
|
| 292 |
|
| 293 |
request_rows = []
|
|
|
|
| 299 |
"arrival_time": req.arrival_time,
|
| 300 |
"prompt_tokens": req.prompt_tokens,
|
| 301 |
"output_tokens": req.output_tokens,
|
| 302 |
+
"cached_prefix_tokens": req.cached_prefix_tokens,
|
| 303 |
"ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
|
| 304 |
"e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
|
| 305 |
})
|
|
|
|
| 308 |
|
| 309 |
provenance = {
|
| 310 |
"simulator": "InferScale-Sim",
|
| 311 |
+
"version": "0.3.0",
|
| 312 |
"latency_profile_type": "analytical-reference",
|
| 313 |
"profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
|
| 314 |
"model_profile_source": self.model.source,
|
| 315 |
"accelerator_profile_source": self.accelerator.source,
|
| 316 |
+
"topology": "colocated",
|
| 317 |
}
|
| 318 |
|
| 319 |
return SimulationResult(
|
|
|
|
| 330 |
|
| 331 |
|
| 332 |
def run_simulation(config: dict) -> dict:
|
| 333 |
+
cfg = SimulationConfig.from_dict(config)
|
| 334 |
+
if cfg.topology == "disaggregated_pd":
|
| 335 |
+
from .disaggregated import DisaggregatedSimulator
|
| 336 |
+
|
| 337 |
+
return DisaggregatedSimulator(cfg).run().to_dict()
|
| 338 |
+
if cfg.topology != "colocated":
|
| 339 |
+
raise ValueError(f"Unsupported topology: {cfg.topology}")
|
| 340 |
+
return Simulator(cfg).run().to_dict()
|
py/inferscale/workloads.py
CHANGED
|
@@ -29,9 +29,6 @@ def _arrival_times(cfg: SimulationConfig, rng: random.Random) -> list[float]:
|
|
| 29 |
return arrivals
|
| 30 |
|
| 31 |
if cfg.arrival_process == "bursty":
|
| 32 |
-
# Alternating baseline/high-load windows. The mean offered load is not
|
| 33 |
-
# forced to equal request_rate_rps; the UI labels this clearly as a burst
|
| 34 |
-
# stress test rather than an average-rate generator.
|
| 35 |
while t < cfg.duration_s:
|
| 36 |
phase = int(t // max(cfg.burst_period_s, 0.1)) % 2
|
| 37 |
local_rate = rate * (cfg.burst_multiplier if phase else 0.55)
|
|
@@ -52,10 +49,15 @@ def _arrival_times(cfg: SimulationConfig, rng: random.Random) -> list[float]:
|
|
| 52 |
|
| 53 |
def generate_workload(cfg: SimulationConfig) -> list[Request]:
|
| 54 |
rng = random.Random(cfg.seed)
|
|
|
|
| 55 |
requests: list[Request] = []
|
| 56 |
for idx, arrival in enumerate(_arrival_times(cfg, rng)):
|
| 57 |
prompt = _sample_lognormal(cfg.prompt_tokens_mean, cfg.prompt_tokens_cv, rng)
|
| 58 |
output = _sample_lognormal(cfg.output_tokens_mean, cfg.output_tokens_cv, rng)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
requests.append(
|
| 60 |
Request(
|
| 61 |
request_id=idx,
|
|
@@ -63,7 +65,8 @@ def generate_workload(cfg: SimulationConfig) -> list[Request]:
|
|
| 63 |
prompt_tokens=prompt,
|
| 64 |
output_tokens=output,
|
| 65 |
deadline_time=arrival + cfg.slo_e2e_ms / 1000.0,
|
| 66 |
-
remaining_prefill=prompt,
|
|
|
|
| 67 |
)
|
| 68 |
)
|
| 69 |
return requests
|
|
|
|
| 29 |
return arrivals
|
| 30 |
|
| 31 |
if cfg.arrival_process == "bursty":
|
|
|
|
|
|
|
|
|
|
| 32 |
while t < cfg.duration_s:
|
| 33 |
phase = int(t // max(cfg.burst_period_s, 0.1)) % 2
|
| 34 |
local_rate = rate * (cfg.burst_multiplier if phase else 0.55)
|
|
|
|
| 49 |
|
| 50 |
def generate_workload(cfg: SimulationConfig) -> list[Request]:
|
| 51 |
rng = random.Random(cfg.seed)
|
| 52 |
+
cache_rng = random.Random(cfg.seed ^ 0x5A17CACE)
|
| 53 |
requests: list[Request] = []
|
| 54 |
for idx, arrival in enumerate(_arrival_times(cfg, rng)):
|
| 55 |
prompt = _sample_lognormal(cfg.prompt_tokens_mean, cfg.prompt_tokens_cv, rng)
|
| 56 |
output = _sample_lognormal(cfg.output_tokens_mean, cfg.output_tokens_cv, rng)
|
| 57 |
+
cached = 0
|
| 58 |
+
if cfg.prefix_cache_enabled and cfg.shared_prefix_tokens > 0 and cfg.prefix_reuse_fraction > 0:
|
| 59 |
+
if cache_rng.random() < min(max(cfg.prefix_reuse_fraction, 0.0), 1.0):
|
| 60 |
+
cached = min(cfg.shared_prefix_tokens, max(prompt - 1, 0))
|
| 61 |
requests.append(
|
| 62 |
Request(
|
| 63 |
request_id=idx,
|
|
|
|
| 65 |
prompt_tokens=prompt,
|
| 66 |
output_tokens=output,
|
| 67 |
deadline_time=arrival + cfg.slo_e2e_ms / 1000.0,
|
| 68 |
+
remaining_prefill=max(0, prompt - cached),
|
| 69 |
+
cached_prefix_tokens=cached,
|
| 70 |
)
|
| 71 |
)
|
| 72 |
return requests
|
pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "inferscale-sim"
|
| 7 |
-
version = "0.
|
| 8 |
description = "Interactive LLM serving simulator and SLO capacity planner"
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10"
|
|
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "inferscale-sim"
|
| 7 |
+
version = "0.3.0"
|
| 8 |
description = "Interactive LLM serving simulator and SLO capacity planner"
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10"
|
scripts/release_check.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
| 3 |
import re
|
| 4 |
import sys
|
| 5 |
from pathlib import Path
|
|
@@ -9,7 +10,10 @@ README = (ROOT / "README.md").read_text()
|
|
| 9 |
SRC = ROOT / "src"
|
| 10 |
sys.path.insert(0, str(SRC))
|
| 11 |
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
errors: list[str] = []
|
| 15 |
|
|
@@ -24,8 +28,8 @@ else:
|
|
| 24 |
|
| 25 |
if "sdk: static" not in README:
|
| 26 |
errors.append("README metadata must use sdk: static")
|
| 27 |
-
if __version__ != "0.
|
| 28 |
-
errors.append(f"package version is {__version__}; expected 0.
|
| 29 |
|
| 30 |
src_files = sorted((ROOT / "src" / "inferscale").glob("*.py"))
|
| 31 |
web_files = sorted((ROOT / "py" / "inferscale").glob("*.py"))
|
|
@@ -47,6 +51,15 @@ for ui_file in (ROOT / "index.html", ROOT / "app.js"):
|
|
| 47 |
except UnicodeEncodeError:
|
| 48 |
errors.append(f"{ui_file.name} contains non-ASCII UI glyphs; use text labels for reliable rendering")
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
smoke_cfg = {
|
| 51 |
"model": "Qwen2.5-3B",
|
| 52 |
"accelerator": "L4",
|
|
@@ -63,11 +76,38 @@ try:
|
|
| 63 |
if smoke["provenance"]["latency_profile_type"] != "analytical-reference":
|
| 64 |
errors.append("profile provenance guard is missing")
|
| 65 |
if smoke["diagnostics"].get("provenance") != "heuristic-simulator-diagnosis":
|
| 66 |
-
errors.append("
|
| 67 |
-
if "
|
| 68 |
-
errors.append("
|
| 69 |
-
except Exception as exc: # pragma: no cover
|
| 70 |
-
errors.append(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
|
| 72 |
if errors:
|
| 73 |
print("InferScale release check: FAIL")
|
|
@@ -79,6 +119,10 @@ print("InferScale release check: PASS")
|
|
| 79 |
print(f"Version: {__version__}")
|
| 80 |
print(f"HF short_description: {len(short)}/60 characters")
|
| 81 |
print(f"Python modules mirrored: {len(src_files)}")
|
| 82 |
-
print(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
print(f"Profile provenance: {smoke['provenance']['latency_profile_type']}")
|
| 84 |
-
print(f"Diagnosis: {smoke['diagnostics']['label']}")
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
import importlib
|
| 4 |
import re
|
| 5 |
import sys
|
| 6 |
from pathlib import Path
|
|
|
|
| 10 |
SRC = ROOT / "src"
|
| 11 |
sys.path.insert(0, str(SRC))
|
| 12 |
|
| 13 |
+
inferscale = importlib.import_module("inferscale")
|
| 14 |
+
__version__ = inferscale.__version__
|
| 15 |
+
design_space_search = inferscale.design_space_search
|
| 16 |
+
run_simulation = inferscale.run_simulation
|
| 17 |
|
| 18 |
errors: list[str] = []
|
| 19 |
|
|
|
|
| 28 |
|
| 29 |
if "sdk: static" not in README:
|
| 30 |
errors.append("README metadata must use sdk: static")
|
| 31 |
+
if __version__ != "0.3.0":
|
| 32 |
+
errors.append(f"package version is {__version__}; expected 0.3.0")
|
| 33 |
|
| 34 |
src_files = sorted((ROOT / "src" / "inferscale").glob("*.py"))
|
| 35 |
web_files = sorted((ROOT / "py" / "inferscale").glob("*.py"))
|
|
|
|
| 51 |
except UnicodeEncodeError:
|
| 52 |
errors.append(f"{ui_file.name} contains non-ASCII UI glyphs; use text labels for reliable rendering")
|
| 53 |
|
| 54 |
+
index = (ROOT / "index.html").read_text()
|
| 55 |
+
app = (ROOT / "app.js").read_text()
|
| 56 |
+
if "<footer" in index.lower():
|
| 57 |
+
errors.append("v0.3 UI should not include the old product-style footer")
|
| 58 |
+
if "Download PNG" not in index or ".chart-download" not in app:
|
| 59 |
+
errors.append("chart PNG export controls are missing")
|
| 60 |
+
if "Worst repetition" not in index or "Target" not in index:
|
| 61 |
+
errors.append("capacity evidence columns are missing")
|
| 62 |
+
|
| 63 |
smoke_cfg = {
|
| 64 |
"model": "Qwen2.5-3B",
|
| 65 |
"accelerator": "L4",
|
|
|
|
| 76 |
if smoke["provenance"]["latency_profile_type"] != "analytical-reference":
|
| 77 |
errors.append("profile provenance guard is missing")
|
| 78 |
if smoke["diagnostics"].get("provenance") != "heuristic-simulator-diagnosis":
|
| 79 |
+
errors.append("diagnosis provenance missing")
|
| 80 |
+
if "prefix_cache_hit_rate" not in smoke["resource"]:
|
| 81 |
+
errors.append("prefix-cache telemetry missing")
|
| 82 |
+
except Exception as exc: # pragma: no cover
|
| 83 |
+
errors.append(f"colocated smoke test raised: {exc}")
|
| 84 |
+
|
| 85 |
+
try:
|
| 86 |
+
pd = run_simulation(smoke_cfg | {
|
| 87 |
+
"topology": "disaggregated_pd",
|
| 88 |
+
"scheduler": "continuous_slo",
|
| 89 |
+
"prefill_accelerator": "L4",
|
| 90 |
+
"decode_accelerator": "L4",
|
| 91 |
+
"interconnect_gbps": 50,
|
| 92 |
+
})
|
| 93 |
+
if pd["provenance"].get("topology") != "disaggregated_pd":
|
| 94 |
+
errors.append("P/D provenance missing")
|
| 95 |
+
if pd["resource"].get("p95_transfer_ms", 0) <= 0:
|
| 96 |
+
errors.append("P/D transfer telemetry missing")
|
| 97 |
+
except Exception as exc: # pragma: no cover
|
| 98 |
+
errors.append(f"P/D smoke test raised: {exc}")
|
| 99 |
+
|
| 100 |
+
try:
|
| 101 |
+
design = design_space_search(smoke_cfg | {
|
| 102 |
+
"shared_prefix_tokens": 64,
|
| 103 |
+
"prefix_reuse_fraction": 0.5,
|
| 104 |
+
"prefill_accelerator": "L4",
|
| 105 |
+
"decode_accelerator": "L4",
|
| 106 |
+
}, include_disaggregated=False)
|
| 107 |
+
if design["candidate_count"] != 10 or design["pareto_count"] < 1 or design["efficiency_pareto_count"] < 1:
|
| 108 |
+
errors.append("design-space smoke test did not return both expected Pareto frontiers")
|
| 109 |
+
except Exception as exc: # pragma: no cover
|
| 110 |
+
errors.append(f"design-space smoke test raised: {exc}")
|
| 111 |
|
| 112 |
if errors:
|
| 113 |
print("InferScale release check: FAIL")
|
|
|
|
| 119 |
print(f"Version: {__version__}")
|
| 120 |
print(f"HF short_description: {len(short)}/60 characters")
|
| 121 |
print(f"Python modules mirrored: {len(src_files)}")
|
| 122 |
+
print(f"Colocated smoke requests: {smoke['summary']['requests_completed']}")
|
| 123 |
+
print(f"P/D transfer p95: {pd['resource']['p95_transfer_ms']:.3f} ms")
|
| 124 |
+
print(
|
| 125 |
+
f"Design candidates: {design['candidate_count']} "
|
| 126 |
+
f"({design['pareto_count']} performance Pareto, {design['efficiency_pareto_count']} efficiency Pareto)"
|
| 127 |
+
)
|
| 128 |
print(f"Profile provenance: {smoke['provenance']['latency_profile_type']}")
|
|
|
src/inferscale/__init__.py
CHANGED
|
@@ -1,15 +1,17 @@
|
|
| 1 |
from .api import execute, metadata
|
| 2 |
from .models import SimulationConfig
|
| 3 |
-
from .optimizer import capacity_search, compare_schedulers
|
| 4 |
from .simulator import run_simulation
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"SimulationConfig",
|
| 8 |
"capacity_search",
|
| 9 |
"compare_schedulers",
|
|
|
|
|
|
|
| 10 |
"execute",
|
| 11 |
"metadata",
|
| 12 |
"run_simulation",
|
| 13 |
]
|
| 14 |
|
| 15 |
-
__version__ = "0.
|
|
|
|
| 1 |
from .api import execute, metadata
|
| 2 |
from .models import SimulationConfig
|
| 3 |
+
from .optimizer import capacity_search, compare_schedulers, compare_topologies, design_space_search
|
| 4 |
from .simulator import run_simulation
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"SimulationConfig",
|
| 8 |
"capacity_search",
|
| 9 |
"compare_schedulers",
|
| 10 |
+
"compare_topologies",
|
| 11 |
+
"design_space_search",
|
| 12 |
"execute",
|
| 13 |
"metadata",
|
| 14 |
"run_simulation",
|
| 15 |
]
|
| 16 |
|
| 17 |
+
__version__ = "0.3.0"
|
src/inferscale/api.py
CHANGED
|
@@ -1,16 +1,17 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
from .optimizer import capacity_search, compare_schedulers
|
| 4 |
from .profiles import ACCELERATORS, MODELS
|
| 5 |
from .simulator import SCHEDULERS, run_simulation
|
| 6 |
|
| 7 |
|
| 8 |
def metadata() -> dict:
|
| 9 |
return {
|
| 10 |
-
"version": "0.
|
| 11 |
"models": list(MODELS.keys()),
|
| 12 |
"accelerators": list(ACCELERATORS.keys()),
|
| 13 |
"schedulers": sorted(SCHEDULERS),
|
|
|
|
| 14 |
"profile_type": "analytical-reference",
|
| 15 |
}
|
| 16 |
|
|
@@ -30,8 +31,13 @@ def execute(action: str, payload: dict) -> dict:
|
|
| 30 |
)
|
| 31 |
if action == "compare":
|
| 32 |
config = payload.get("config", payload)
|
| 33 |
-
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
if action == "metadata":
|
| 36 |
return metadata()
|
| 37 |
raise ValueError(f"Unknown action: {action}")
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
from .optimizer import capacity_search, compare_schedulers, compare_topologies, design_space_search
|
| 4 |
from .profiles import ACCELERATORS, MODELS
|
| 5 |
from .simulator import SCHEDULERS, run_simulation
|
| 6 |
|
| 7 |
|
| 8 |
def metadata() -> dict:
|
| 9 |
return {
|
| 10 |
+
"version": "0.3.0",
|
| 11 |
"models": list(MODELS.keys()),
|
| 12 |
"accelerators": list(ACCELERATORS.keys()),
|
| 13 |
"schedulers": sorted(SCHEDULERS),
|
| 14 |
+
"topologies": ["colocated", "disaggregated_pd"],
|
| 15 |
"profile_type": "analytical-reference",
|
| 16 |
}
|
| 17 |
|
|
|
|
| 31 |
)
|
| 32 |
if action == "compare":
|
| 33 |
config = payload.get("config", payload)
|
| 34 |
+
return compare_schedulers(config, payload.get("schedulers"))
|
| 35 |
+
if action == "topology_compare":
|
| 36 |
+
config = payload.get("config", payload)
|
| 37 |
+
return compare_topologies(config)
|
| 38 |
+
if action == "design_space":
|
| 39 |
+
config = payload.get("config", payload)
|
| 40 |
+
return design_space_search(config, bool(payload.get("include_disaggregated", True)))
|
| 41 |
if action == "metadata":
|
| 42 |
return metadata()
|
| 43 |
raise ValueError(f"Unknown action: {action}")
|
src/inferscale/diagnostics.py
CHANGED
|
@@ -6,7 +6,13 @@ from .models import SimulationConfig
|
|
| 6 |
|
| 7 |
|
| 8 |
def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dict[str, Any], cfg: SimulationConfig) -> dict[str, Any]:
|
| 9 |
-
"""Explain the simulator's limiting regime from its own state.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
target = cfg.slo_attainment_target
|
| 11 |
attainment = float(summary.get("slo_attainment", 0.0))
|
| 12 |
ttft_attainment = float(summary.get("ttft_slo_attainment", 0.0))
|
|
@@ -15,37 +21,53 @@ def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dic
|
|
| 15 |
busy = float(summary.get("busy_fraction", 0.0))
|
| 16 |
kv_util = float(resource.get("peak_kv_utilization", 0.0))
|
| 17 |
queue_p95 = float(latency.get("queue_ms", {}).get("p95", 0.0))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
code = "mixed_pressure"
|
| 20 |
label = "Mixed SLO pressure"
|
| 21 |
explanation = "Several constraints are active; inspect queueing, KV pressure, and tail latency together."
|
| 22 |
recommendation = "Compare schedulers and run the capacity planner to isolate the limiting regime."
|
| 23 |
|
| 24 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
code, label = "kv_pressure", "KV memory pressure"
|
| 26 |
explanation = "Requests remain unfinished while the simulated KV cache approaches its configured capacity."
|
| 27 |
recommendation = "Reduce sequence pressure, lower offered load, use a smaller KV block, or choose a larger-memory profile."
|
| 28 |
elif unfinished > 0 and busy >= 0.92:
|
| 29 |
code, label = "over_capacity", "Offered load above capacity"
|
| 30 |
-
explanation = "The virtual
|
| 31 |
recommendation = "Reduce offered load or use the capacity planner to find a sustainable operating point."
|
| 32 |
elif unfinished > 0:
|
| 33 |
code, label = "admission_stall", "Admission stall"
|
| 34 |
explanation = "Some requests cannot be admitted or completed under the current serving constraints."
|
| 35 |
-
recommendation = "Inspect batch-token and KV-cache limits, then compare a less restrictive
|
| 36 |
elif attainment >= target:
|
| 37 |
if busy >= 0.90:
|
| 38 |
code, label = "healthy_near_saturation", "Healthy, little headroom"
|
| 39 |
-
explanation = "The workload meets the configured SLO target, but simulated
|
| 40 |
recommendation = "Run the capacity planner before increasing traffic; the current point is close to saturation."
|
| 41 |
else:
|
| 42 |
code, label = "healthy", "Healthy"
|
| 43 |
explanation = "The workload drains and meets the configured SLO-attainment target with visible headroom."
|
| 44 |
-
recommendation = "Use the capacity planner
|
| 45 |
elif kv_util >= 0.90:
|
| 46 |
code, label = "kv_pressure", "KV memory pressure"
|
| 47 |
explanation = "Peak simulated KV utilization is high enough to constrain admission and batching flexibility."
|
| 48 |
-
recommendation = "Reduce sequence pressure, use smaller KV blocks, or select a larger-memory
|
| 49 |
elif ttft_attainment < target and queue_p95 >= max(25.0, cfg.slo_ttft_ms * 0.20):
|
| 50 |
code, label = "queue_prefill_pressure", "Queue / prefill pressure"
|
| 51 |
explanation = "TTFT misses coincide with substantial queueing before prefill begins."
|
|
@@ -53,14 +75,14 @@ def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dic
|
|
| 53 |
elif ttft_attainment < target:
|
| 54 |
code, label = "ttft_pressure", "TTFT pressure"
|
| 55 |
explanation = "First-token latency is the dominant SLO failure even though queueing is not the only contributor."
|
| 56 |
-
recommendation = "Compare prefill-oriented
|
| 57 |
elif e2e_attainment < target:
|
| 58 |
code, label = "decode_pressure", "Decode / E2E pressure"
|
| 59 |
explanation = "Most first tokens arrive within target, but end-to-end latency still misses the configured SLO."
|
| 60 |
-
recommendation = "Reduce output-length pressure or offered load; larger decode
|
| 61 |
elif busy >= 0.95:
|
| 62 |
code, label = "compute_saturation", "Compute saturation"
|
| 63 |
-
explanation = "The simulated
|
| 64 |
recommendation = "Reduce load or switch to a higher-throughput accelerator profile."
|
| 65 |
|
| 66 |
return {
|
|
@@ -76,6 +98,9 @@ def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dic
|
|
| 76 |
"peak_kv_utilization": kv_util,
|
| 77 |
"queue_p95_ms": queue_p95,
|
| 78 |
"unfinished_requests": unfinished,
|
|
|
|
|
|
|
|
|
|
| 79 |
},
|
| 80 |
"provenance": "heuristic-simulator-diagnosis",
|
| 81 |
}
|
|
|
|
| 6 |
|
| 7 |
|
| 8 |
def diagnose_run(summary: dict[str, Any], latency: dict[str, Any], resource: dict[str, Any], cfg: SimulationConfig) -> dict[str, Any]:
|
| 9 |
+
"""Explain the simulator's limiting regime from its own state.
|
| 10 |
+
|
| 11 |
+
Diagnostics are heuristic labels over simulated telemetry; they are not
|
| 12 |
+
hardware-counter measurements and must be interpreted with the profile
|
| 13 |
+
provenance attached to every run.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
target = cfg.slo_attainment_target
|
| 17 |
attainment = float(summary.get("slo_attainment", 0.0))
|
| 18 |
ttft_attainment = float(summary.get("ttft_slo_attainment", 0.0))
|
|
|
|
| 21 |
busy = float(summary.get("busy_fraction", 0.0))
|
| 22 |
kv_util = float(resource.get("peak_kv_utilization", 0.0))
|
| 23 |
queue_p95 = float(latency.get("queue_ms", {}).get("p95", 0.0))
|
| 24 |
+
prefill_busy = float(resource.get("prefill_busy_fraction", 0.0))
|
| 25 |
+
decode_busy = float(resource.get("decode_busy_fraction", 0.0))
|
| 26 |
+
transfer_busy = float(resource.get("transfer_busy_fraction", 0.0))
|
| 27 |
+
topology = str(resource.get("topology", cfg.topology))
|
| 28 |
|
| 29 |
code = "mixed_pressure"
|
| 30 |
label = "Mixed SLO pressure"
|
| 31 |
explanation = "Several constraints are active; inspect queueing, KV pressure, and tail latency together."
|
| 32 |
recommendation = "Compare schedulers and run the capacity planner to isolate the limiting regime."
|
| 33 |
|
| 34 |
+
if topology == "disaggregated_pd" and transfer_busy >= 0.85 and attainment < target:
|
| 35 |
+
code, label = "kv_transfer_pressure", "KV transfer pressure"
|
| 36 |
+
explanation = "The simulated P/D interconnect is highly utilized and transfer time is contributing to first-token latency."
|
| 37 |
+
recommendation = "Increase link bandwidth, reduce prompt pressure, or compare against colocated serving."
|
| 38 |
+
elif topology == "disaggregated_pd" and prefill_busy >= 0.92 and prefill_busy > decode_busy + 0.08:
|
| 39 |
+
code, label = "prefill_pool_pressure", "Prefill pool pressure"
|
| 40 |
+
explanation = "Prefill workers are the hottest role in the disaggregated pipeline."
|
| 41 |
+
recommendation = "Add prefill capacity, use prefix reuse, or reduce prompt-side pressure."
|
| 42 |
+
elif topology == "disaggregated_pd" and decode_busy >= 0.92 and decode_busy > prefill_busy + 0.08:
|
| 43 |
+
code, label = "decode_pool_pressure", "Decode pool pressure"
|
| 44 |
+
explanation = "Decode workers are the hottest role in the disaggregated pipeline."
|
| 45 |
+
recommendation = "Add decode capacity or reduce output-length / offered-load pressure."
|
| 46 |
+
elif unfinished > 0 and kv_util >= 0.85:
|
| 47 |
code, label = "kv_pressure", "KV memory pressure"
|
| 48 |
explanation = "Requests remain unfinished while the simulated KV cache approaches its configured capacity."
|
| 49 |
recommendation = "Reduce sequence pressure, lower offered load, use a smaller KV block, or choose a larger-memory profile."
|
| 50 |
elif unfinished > 0 and busy >= 0.92:
|
| 51 |
code, label = "over_capacity", "Offered load above capacity"
|
| 52 |
+
explanation = "The virtual serving system remains busy while the workload fails to drain."
|
| 53 |
recommendation = "Reduce offered load or use the capacity planner to find a sustainable operating point."
|
| 54 |
elif unfinished > 0:
|
| 55 |
code, label = "admission_stall", "Admission stall"
|
| 56 |
explanation = "Some requests cannot be admitted or completed under the current serving constraints."
|
| 57 |
+
recommendation = "Inspect batch-token and KV-cache limits, then compare a less restrictive configuration."
|
| 58 |
elif attainment >= target:
|
| 59 |
if busy >= 0.90:
|
| 60 |
code, label = "healthy_near_saturation", "Healthy, little headroom"
|
| 61 |
+
explanation = "The workload meets the configured SLO target, but simulated resource utilization is already high."
|
| 62 |
recommendation = "Run the capacity planner before increasing traffic; the current point is close to saturation."
|
| 63 |
else:
|
| 64 |
code, label = "healthy", "Healthy"
|
| 65 |
explanation = "The workload drains and meets the configured SLO-attainment target with visible headroom."
|
| 66 |
+
recommendation = "Use the capacity planner or design explorer to quantify additional headroom."
|
| 67 |
elif kv_util >= 0.90:
|
| 68 |
code, label = "kv_pressure", "KV memory pressure"
|
| 69 |
explanation = "Peak simulated KV utilization is high enough to constrain admission and batching flexibility."
|
| 70 |
+
recommendation = "Reduce sequence pressure, use smaller KV blocks, enable reusable prefixes, or select a larger-memory profile."
|
| 71 |
elif ttft_attainment < target and queue_p95 >= max(25.0, cfg.slo_ttft_ms * 0.20):
|
| 72 |
code, label = "queue_prefill_pressure", "Queue / prefill pressure"
|
| 73 |
explanation = "TTFT misses coincide with substantial queueing before prefill begins."
|
|
|
|
| 75 |
elif ttft_attainment < target:
|
| 76 |
code, label = "ttft_pressure", "TTFT pressure"
|
| 77 |
explanation = "First-token latency is the dominant SLO failure even though queueing is not the only contributor."
|
| 78 |
+
recommendation = "Compare prefill-oriented scheduling, prefix reuse, and P/D disaggregation scenarios."
|
| 79 |
elif e2e_attainment < target:
|
| 80 |
code, label = "decode_pressure", "Decode / E2E pressure"
|
| 81 |
explanation = "Most first tokens arrive within target, but end-to-end latency still misses the configured SLO."
|
| 82 |
+
recommendation = "Reduce output-length pressure or offered load; larger decode capacity may help when memory allows."
|
| 83 |
elif busy >= 0.95:
|
| 84 |
code, label = "compute_saturation", "Compute saturation"
|
| 85 |
+
explanation = "The simulated serving resources are effectively saturated even though no single SLO component dominates."
|
| 86 |
recommendation = "Reduce load or switch to a higher-throughput accelerator profile."
|
| 87 |
|
| 88 |
return {
|
|
|
|
| 98 |
"peak_kv_utilization": kv_util,
|
| 99 |
"queue_p95_ms": queue_p95,
|
| 100 |
"unfinished_requests": unfinished,
|
| 101 |
+
"prefill_busy_fraction": prefill_busy,
|
| 102 |
+
"decode_busy_fraction": decode_busy,
|
| 103 |
+
"transfer_busy_fraction": transfer_busy,
|
| 104 |
},
|
| 105 |
"provenance": "heuristic-simulator-diagnosis",
|
| 106 |
}
|
src/inferscale/disaggregated.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import heapq
|
| 4 |
+
from dataclasses import asdict, dataclass, field
|
| 5 |
+
from itertools import count
|
| 6 |
+
|
| 7 |
+
from .diagnostics import diagnose_run
|
| 8 |
+
from .kv_cache import KVCacheModel
|
| 9 |
+
from .latency import AnalyticalLatencyModel
|
| 10 |
+
from .metrics import percentile, summarize
|
| 11 |
+
from .models import Request, SimulationConfig, SimulationResult, TimelinePoint
|
| 12 |
+
from .profiles import get_accelerator, get_model
|
| 13 |
+
from .workloads import generate_workload
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass
|
| 17 |
+
class PrefillWorker:
|
| 18 |
+
worker_id: int
|
| 19 |
+
latency: AnalyticalLatencyModel
|
| 20 |
+
kv: KVCacheModel
|
| 21 |
+
busy: bool = False
|
| 22 |
+
busy_time_s: float = 0.0
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@dataclass
|
| 26 |
+
class DecodeWorker:
|
| 27 |
+
worker_id: int
|
| 28 |
+
latency: AnalyticalLatencyModel
|
| 29 |
+
kv: KVCacheModel
|
| 30 |
+
active: list[Request] = field(default_factory=list)
|
| 31 |
+
busy: bool = False
|
| 32 |
+
busy_time_s: float = 0.0
|
| 33 |
+
peak_kv_gb: float = 0.0
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class DisaggregatedSimulator:
|
| 37 |
+
"""Two-stage prefill/decode discrete-event simulator.
|
| 38 |
+
|
| 39 |
+
v0.3 models role-specific worker pools plus a serialized analytical KV-transfer
|
| 40 |
+
link. It is intentionally a systems abstraction, not a distributed-runtime
|
| 41 |
+
emulator: transport, compute and memory timings remain reference-model
|
| 42 |
+
predictions and carry explicit provenance in every result.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
def __init__(self, cfg: SimulationConfig):
|
| 46 |
+
if cfg.prefill_workers < 1 or cfg.decode_workers < 1:
|
| 47 |
+
raise ValueError("prefill_workers and decode_workers must be >= 1")
|
| 48 |
+
if cfg.interconnect_gbps <= 0:
|
| 49 |
+
raise ValueError("interconnect_gbps must be > 0")
|
| 50 |
+
|
| 51 |
+
self.cfg = cfg
|
| 52 |
+
self.model = get_model(cfg.model)
|
| 53 |
+
self.prefill_accelerator = get_accelerator(cfg.prefill_accelerator)
|
| 54 |
+
self.decode_accelerator = get_accelerator(cfg.decode_accelerator)
|
| 55 |
+
self.prefill_latency = AnalyticalLatencyModel(self.model, self.prefill_accelerator, cfg.quantization)
|
| 56 |
+
self.decode_latency = AnalyticalLatencyModel(self.model, self.decode_accelerator, cfg.quantization)
|
| 57 |
+
|
| 58 |
+
self.prefill_workers = [
|
| 59 |
+
PrefillWorker(i, self.prefill_latency, KVCacheModel(self.prefill_latency, cfg))
|
| 60 |
+
for i in range(cfg.prefill_workers)
|
| 61 |
+
]
|
| 62 |
+
self.decode_workers = [
|
| 63 |
+
DecodeWorker(i, self.decode_latency, KVCacheModel(self.decode_latency, cfg))
|
| 64 |
+
for i in range(cfg.decode_workers)
|
| 65 |
+
]
|
| 66 |
+
|
| 67 |
+
self.requests = generate_workload(cfg)
|
| 68 |
+
self.prefill_waiting: list[Request] = []
|
| 69 |
+
self.decode_waiting: list[Request] = []
|
| 70 |
+
self.completed: list[Request] = []
|
| 71 |
+
self.events: list[tuple[float, int, str, object]] = []
|
| 72 |
+
self.seq = count()
|
| 73 |
+
self.now = 0.0
|
| 74 |
+
self.transfer_busy_until = 0.0
|
| 75 |
+
self.transfer_busy_time_s = 0.0
|
| 76 |
+
self.transfer_gb = 0.0
|
| 77 |
+
self.transfer_latencies_ms: list[float] = []
|
| 78 |
+
self.transfer_pending = 0
|
| 79 |
+
self.timeline: list[TimelinePoint] = []
|
| 80 |
+
self.peak_total_kv_gb = 0.0
|
| 81 |
+
self.warnings: list[str] = []
|
| 82 |
+
|
| 83 |
+
for req in self.requests:
|
| 84 |
+
self._push(req.arrival_time, "arrival", req)
|
| 85 |
+
|
| 86 |
+
def _push(self, time_s: float, kind: str, payload: object) -> None:
|
| 87 |
+
heapq.heappush(self.events, (max(time_s, self.now), next(self.seq), kind, payload))
|
| 88 |
+
|
| 89 |
+
def _waiting_sorted(self, requests: list[Request]) -> list[Request]:
|
| 90 |
+
if self.cfg.scheduler == "continuous_sjf":
|
| 91 |
+
return sorted(requests, key=lambda r: (r.remaining_prefill + r.output_tokens, r.arrival_time))
|
| 92 |
+
if self.cfg.scheduler in {"continuous_slo", "chunked_slo"}:
|
| 93 |
+
def slack(req: Request) -> tuple[float, float]:
|
| 94 |
+
prefill = self.prefill_latency.prefill_seconds([max(req.remaining_prefill, 1)])
|
| 95 |
+
midpoint_context = req.prompt_tokens + max(req.output_tokens // 2, 1)
|
| 96 |
+
decode = req.output_tokens * self.decode_latency.decode_step_seconds([midpoint_context])
|
| 97 |
+
return (req.deadline_time - self.now - prefill - decode, req.arrival_time)
|
| 98 |
+
|
| 99 |
+
return sorted(requests, key=slack)
|
| 100 |
+
return sorted(requests, key=lambda r: r.arrival_time)
|
| 101 |
+
|
| 102 |
+
def _record_timeline(self, force: bool = False) -> None:
|
| 103 |
+
target = max(self.cfg.timeline_points, 20)
|
| 104 |
+
total_kv = sum(w.kv.used_gb(w.active) for w in self.decode_workers)
|
| 105 |
+
total_capacity = sum(w.kv.capacity_gb for w in self.decode_workers)
|
| 106 |
+
self.peak_total_kv_gb = max(self.peak_total_kv_gb, total_kv)
|
| 107 |
+
for worker in self.decode_workers:
|
| 108 |
+
worker.peak_kv_gb = max(worker.peak_kv_gb, worker.kv.used_gb(worker.active))
|
| 109 |
+
point = TimelinePoint(
|
| 110 |
+
time_s=self.now,
|
| 111 |
+
waiting=len(self.prefill_waiting),
|
| 112 |
+
prefill_pending=sum(1 for w in self.prefill_workers if w.busy),
|
| 113 |
+
decoding=sum(len(w.active) for w in self.decode_workers),
|
| 114 |
+
completed=len(self.completed),
|
| 115 |
+
kv_used_gb=total_kv,
|
| 116 |
+
kv_capacity_gb=total_capacity,
|
| 117 |
+
transfer_pending=self.transfer_pending,
|
| 118 |
+
decode_ready=len(self.decode_waiting),
|
| 119 |
+
prefill_active=sum(1 for w in self.prefill_workers if w.busy),
|
| 120 |
+
)
|
| 121 |
+
if not self.timeline or force or self.now - self.timeline[-1].time_s >= max(self.cfg.duration_s / target, 0.05):
|
| 122 |
+
self.timeline.append(point)
|
| 123 |
+
if len(self.timeline) > target * 2:
|
| 124 |
+
self.timeline = self.timeline[::2]
|
| 125 |
+
|
| 126 |
+
def _prefill_batch_for(self, worker: PrefillWorker) -> list[tuple[Request, int]]:
|
| 127 |
+
if not self.prefill_waiting:
|
| 128 |
+
return []
|
| 129 |
+
selected: list[tuple[Request, int]] = []
|
| 130 |
+
budget = self.cfg.max_batch_tokens
|
| 131 |
+
virtual_selected: list[Request] = []
|
| 132 |
+
for req in self._waiting_sorted(self.prefill_waiting):
|
| 133 |
+
if len(selected) >= self.cfg.max_batch_size or budget <= 0:
|
| 134 |
+
break
|
| 135 |
+
chunk = req.remaining_prefill
|
| 136 |
+
if self.cfg.scheduler == "chunked_slo":
|
| 137 |
+
chunk = min(chunk, self.cfg.chunk_size)
|
| 138 |
+
chunk = min(chunk, budget)
|
| 139 |
+
if chunk <= 0:
|
| 140 |
+
continue
|
| 141 |
+
if not worker.kv.can_admit(req, [], virtual_selected):
|
| 142 |
+
continue
|
| 143 |
+
selected.append((req, chunk))
|
| 144 |
+
virtual_selected.append(req)
|
| 145 |
+
budget -= chunk
|
| 146 |
+
return selected
|
| 147 |
+
|
| 148 |
+
def _try_start_prefill(self) -> None:
|
| 149 |
+
for worker in self.prefill_workers:
|
| 150 |
+
if worker.busy:
|
| 151 |
+
continue
|
| 152 |
+
batch = self._prefill_batch_for(worker)
|
| 153 |
+
if not batch:
|
| 154 |
+
continue
|
| 155 |
+
selected_ids = {r.request_id for r, _ in batch}
|
| 156 |
+
self.prefill_waiting = [r for r in self.prefill_waiting if r.request_id not in selected_ids]
|
| 157 |
+
for req, _ in batch:
|
| 158 |
+
if req.first_prefill_time is None:
|
| 159 |
+
req.first_prefill_time = self.now
|
| 160 |
+
delta = worker.latency.prefill_seconds([chunk for _, chunk in batch])
|
| 161 |
+
worker.busy = True
|
| 162 |
+
worker.busy_time_s += delta
|
| 163 |
+
self._push(self.now + delta, "prefill_done", (worker.worker_id, batch))
|
| 164 |
+
|
| 165 |
+
def _schedule_transfer(self, req: Request) -> None:
|
| 166 |
+
# Cached shared-prefix KV is assumed resident in both role pools. Only
|
| 167 |
+
# newly computed prompt state must cross the P/D boundary.
|
| 168 |
+
bytes_to_transfer = req.uncached_prompt_tokens * self.prefill_latency.kv_bytes_per_token()
|
| 169 |
+
gb = bytes_to_transfer / 1e9
|
| 170 |
+
start = max(self.now, self.transfer_busy_until)
|
| 171 |
+
duration = self.cfg.transfer_base_ms / 1000.0 + bytes_to_transfer / (self.cfg.interconnect_gbps * 1e9)
|
| 172 |
+
end = start + duration
|
| 173 |
+
self.transfer_busy_until = end
|
| 174 |
+
self.transfer_busy_time_s += duration
|
| 175 |
+
self.transfer_gb += gb
|
| 176 |
+
self.transfer_latencies_ms.append(duration * 1000.0)
|
| 177 |
+
self.transfer_pending += 1
|
| 178 |
+
req.transfer_start_time = start
|
| 179 |
+
req.transfer_end_time = end
|
| 180 |
+
self._push(end, "transfer_done", req)
|
| 181 |
+
|
| 182 |
+
def _decode_order(self) -> list[Request]:
|
| 183 |
+
return self._waiting_sorted(self.decode_waiting)
|
| 184 |
+
|
| 185 |
+
def _try_schedule_decode(self) -> None:
|
| 186 |
+
# Admissions occur only between iterations. A worker marked busy has a
|
| 187 |
+
# decode step already in flight and cannot accept work until it completes.
|
| 188 |
+
for worker in self.decode_workers:
|
| 189 |
+
if worker.busy:
|
| 190 |
+
continue
|
| 191 |
+
slots = self.cfg.max_batch_size - len(worker.active)
|
| 192 |
+
if slots > 0 and self.decode_waiting:
|
| 193 |
+
for req in list(self._decode_order()):
|
| 194 |
+
if slots <= 0:
|
| 195 |
+
break
|
| 196 |
+
if worker.kv.can_admit(req, worker.active):
|
| 197 |
+
self.decode_waiting.remove(req)
|
| 198 |
+
req.decode_worker_id = worker.worker_id
|
| 199 |
+
worker.active.append(req)
|
| 200 |
+
slots -= 1
|
| 201 |
+
if not worker.active:
|
| 202 |
+
continue
|
| 203 |
+
contexts = [r.context_tokens for r in worker.active]
|
| 204 |
+
delta = worker.latency.decode_step_seconds(contexts)
|
| 205 |
+
worker.busy = True
|
| 206 |
+
worker.busy_time_s += delta
|
| 207 |
+
self._push(self.now + delta, "decode_done", worker.worker_id)
|
| 208 |
+
|
| 209 |
+
def _handle_event(self, kind: str, payload: object) -> None:
|
| 210 |
+
if kind == "arrival":
|
| 211 |
+
self.prefill_waiting.append(payload) # type: ignore[arg-type]
|
| 212 |
+
self._try_start_prefill()
|
| 213 |
+
return
|
| 214 |
+
|
| 215 |
+
if kind == "prefill_done":
|
| 216 |
+
worker_id, batch = payload # type: ignore[misc]
|
| 217 |
+
worker = self.prefill_workers[worker_id]
|
| 218 |
+
worker.busy = False
|
| 219 |
+
for req, chunk in batch:
|
| 220 |
+
req.remaining_prefill = max(0, req.remaining_prefill - chunk)
|
| 221 |
+
if req.remaining_prefill > 0:
|
| 222 |
+
self.prefill_waiting.append(req)
|
| 223 |
+
else:
|
| 224 |
+
req.prefill_complete_time = self.now
|
| 225 |
+
self._schedule_transfer(req)
|
| 226 |
+
self._try_start_prefill()
|
| 227 |
+
return
|
| 228 |
+
|
| 229 |
+
if kind == "transfer_done":
|
| 230 |
+
req = payload # type: ignore[assignment]
|
| 231 |
+
self.transfer_pending = max(0, self.transfer_pending - 1)
|
| 232 |
+
self.decode_waiting.append(req)
|
| 233 |
+
self._try_schedule_decode()
|
| 234 |
+
return
|
| 235 |
+
|
| 236 |
+
if kind == "decode_done":
|
| 237 |
+
worker = self.decode_workers[int(payload)]
|
| 238 |
+
worker.busy = False
|
| 239 |
+
for req in worker.active:
|
| 240 |
+
req.generated_tokens += 1
|
| 241 |
+
if req.first_token_time is None:
|
| 242 |
+
req.first_token_time = self.now
|
| 243 |
+
done = [r for r in worker.active if r.complete]
|
| 244 |
+
for req in done:
|
| 245 |
+
req.completion_time = self.now
|
| 246 |
+
self.completed.append(req)
|
| 247 |
+
if done:
|
| 248 |
+
done_ids = {r.request_id for r in done}
|
| 249 |
+
worker.active = [r for r in worker.active if r.request_id not in done_ids]
|
| 250 |
+
self._try_schedule_decode()
|
| 251 |
+
return
|
| 252 |
+
|
| 253 |
+
raise RuntimeError(f"Unknown event kind: {kind}")
|
| 254 |
+
|
| 255 |
+
def run(self) -> SimulationResult:
|
| 256 |
+
if self.cfg.scheduler == "static_fcfs":
|
| 257 |
+
self.warnings.append("Static FCFS is not defined for P/D disaggregation; using continuous FCFS semantics.")
|
| 258 |
+
self._record_timeline(force=True)
|
| 259 |
+
processed = 0
|
| 260 |
+
max_events = max(10000, len(self.requests) * max(self.cfg.output_tokens_mean, 1) * 20)
|
| 261 |
+
|
| 262 |
+
while self.events and len(self.completed) < len(self.requests):
|
| 263 |
+
time_s, _, kind, payload = heapq.heappop(self.events)
|
| 264 |
+
self.now = max(self.now, time_s)
|
| 265 |
+
self._handle_event(kind, payload)
|
| 266 |
+
self._record_timeline()
|
| 267 |
+
processed += 1
|
| 268 |
+
if processed > max_events:
|
| 269 |
+
self.warnings.append("Simulation stopped at the event safety limit.")
|
| 270 |
+
break
|
| 271 |
+
|
| 272 |
+
# If queued work remains with no events, the configuration is memory- or
|
| 273 |
+
# admission-constrained rather than silently considered complete.
|
| 274 |
+
if len(self.completed) < len(self.requests) and not self.events:
|
| 275 |
+
self.warnings.append("Disaggregated pipeline stalled before all requests completed.")
|
| 276 |
+
|
| 277 |
+
self._record_timeline(force=True)
|
| 278 |
+
makespan = max(self.now, self.cfg.duration_s if self.requests else 0.0)
|
| 279 |
+
prefill_util = sum(w.busy_time_s for w in self.prefill_workers) / max(makespan * len(self.prefill_workers), 1e-9)
|
| 280 |
+
decode_util = sum(w.busy_time_s for w in self.decode_workers) / max(makespan * len(self.decode_workers), 1e-9)
|
| 281 |
+
transfer_util = self.transfer_busy_time_s / max(makespan, 1e-9)
|
| 282 |
+
hottest = min(1.0, max(prefill_util, decode_util, transfer_util))
|
| 283 |
+
summary, latency = summarize(self.completed, self.cfg, makespan, hottest * makespan)
|
| 284 |
+
summary["requests_generated"] = len(self.requests)
|
| 285 |
+
summary["requests_unfinished"] = len(self.requests) - len(self.completed)
|
| 286 |
+
|
| 287 |
+
total_kv_capacity = sum(w.kv.capacity_gb for w in self.decode_workers)
|
| 288 |
+
peak_kv_gb = self.peak_total_kv_gb
|
| 289 |
+
peak_worker_util = max(
|
| 290 |
+
(w.peak_kv_gb / w.kv.capacity_gb if w.kv.capacity_gb > 0 else 0.0 for w in self.decode_workers),
|
| 291 |
+
default=0.0,
|
| 292 |
+
)
|
| 293 |
+
resource = {
|
| 294 |
+
"topology": "disaggregated_pd",
|
| 295 |
+
"model_weight_gb": self.decode_latency.model_weight_gb,
|
| 296 |
+
"kv_capacity_gb": total_kv_capacity,
|
| 297 |
+
"peak_kv_gb": peak_kv_gb,
|
| 298 |
+
"peak_kv_utilization": peak_worker_util,
|
| 299 |
+
"prefill_accelerator_vram_gb": self.prefill_accelerator.vram_gb,
|
| 300 |
+
"decode_accelerator_vram_gb": self.decode_accelerator.vram_gb,
|
| 301 |
+
"prefill_workers": len(self.prefill_workers),
|
| 302 |
+
"decode_workers": len(self.decode_workers),
|
| 303 |
+
"accelerator_instances": len(self.prefill_workers) + len(self.decode_workers),
|
| 304 |
+
"prefill_busy_fraction": min(1.0, prefill_util),
|
| 305 |
+
"decode_busy_fraction": min(1.0, decode_util),
|
| 306 |
+
"transfer_busy_fraction": min(1.0, transfer_util),
|
| 307 |
+
"kv_transfer_gb": self.transfer_gb,
|
| 308 |
+
"mean_transfer_ms": sum(self.transfer_latencies_ms) / len(self.transfer_latencies_ms) if self.transfer_latencies_ms else 0.0,
|
| 309 |
+
"p95_transfer_ms": percentile(self.transfer_latencies_ms, 0.95),
|
| 310 |
+
"interconnect_gbps": self.cfg.interconnect_gbps,
|
| 311 |
+
"prefix_cache_gb_per_decode_worker": self.decode_workers[0].kv.shared_prefix_gb if self.decode_workers else 0.0,
|
| 312 |
+
"prefix_cache_hits": sum(1 for r in self.requests if r.prefix_cache_hit),
|
| 313 |
+
"prefix_cache_hit_rate": (sum(1 for r in self.requests if r.prefix_cache_hit) / len(self.requests)) if self.requests else 0.0,
|
| 314 |
+
"prefill_tokens_saved": sum(r.cached_prefix_tokens for r in self.requests),
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
request_rows = []
|
| 318 |
+
for req in self.completed[:2000]:
|
| 319 |
+
transfer_ms = 0.0
|
| 320 |
+
if req.transfer_start_time is not None and req.transfer_end_time is not None:
|
| 321 |
+
transfer_ms = (req.transfer_end_time - req.transfer_start_time) * 1000.0
|
| 322 |
+
request_rows.append(
|
| 323 |
+
{
|
| 324 |
+
"request_id": req.request_id,
|
| 325 |
+
"arrival_time": req.arrival_time,
|
| 326 |
+
"prompt_tokens": req.prompt_tokens,
|
| 327 |
+
"output_tokens": req.output_tokens,
|
| 328 |
+
"cached_prefix_tokens": req.cached_prefix_tokens,
|
| 329 |
+
"decode_worker_id": req.decode_worker_id,
|
| 330 |
+
"transfer_ms": transfer_ms,
|
| 331 |
+
"ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
|
| 332 |
+
"e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
|
| 333 |
+
}
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
diagnostics = diagnose_run(summary, latency, resource, self.cfg)
|
| 337 |
+
provenance = {
|
| 338 |
+
"simulator": "InferScale-Sim",
|
| 339 |
+
"version": "0.3.0",
|
| 340 |
+
"latency_profile_type": "analytical-reference",
|
| 341 |
+
"profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
|
| 342 |
+
"model_profile_source": self.model.source,
|
| 343 |
+
"prefill_accelerator_profile_source": self.prefill_accelerator.source,
|
| 344 |
+
"decode_accelerator_profile_source": self.decode_accelerator.source,
|
| 345 |
+
"topology": "disaggregated_pd",
|
| 346 |
+
"transfer_model": "serialized-reference-link",
|
| 347 |
+
}
|
| 348 |
+
return SimulationResult(
|
| 349 |
+
config=self.cfg.to_dict(),
|
| 350 |
+
provenance=provenance,
|
| 351 |
+
summary=summary,
|
| 352 |
+
latency=latency,
|
| 353 |
+
resource=resource,
|
| 354 |
+
diagnostics=diagnostics,
|
| 355 |
+
requests=request_rows,
|
| 356 |
+
timeline=[asdict(p) for p in self.timeline],
|
| 357 |
+
warnings=self.warnings,
|
| 358 |
+
)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def run_disaggregated(config: dict) -> dict:
|
| 362 |
+
return DisaggregatedSimulator(SimulationConfig.from_dict(config)).run().to_dict()
|
src/inferscale/kv_cache.py
CHANGED
|
@@ -12,18 +12,20 @@ class KVCacheModel:
|
|
| 12 |
self.cfg = cfg
|
| 13 |
self.model_weight_gb = latency_model.model_weight_gb
|
| 14 |
total_vram = latency_model.accelerator.vram_gb
|
| 15 |
-
# Reserve non-KV space for runtime workspace and weights.
|
| 16 |
remaining = max(0.0, total_vram - self.model_weight_gb - 1.2)
|
| 17 |
self.capacity_gb = remaining * cfg.kv_memory_fraction
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
def _allocated_tokens(self, req: Request, include_output_reservation: bool = False) -> int:
|
| 20 |
-
|
|
|
|
|
|
|
| 21 |
if self.cfg.scheduler == "static_fcfs" or include_output_reservation:
|
| 22 |
-
|
| 23 |
-
# fragmentation / over-reservation versus paged allocation.
|
| 24 |
-
return req.prompt_tokens + req.output_tokens
|
| 25 |
block = max(self.cfg.kv_block_tokens, 1)
|
| 26 |
-
return int(math.ceil(live_tokens / block) * block)
|
| 27 |
|
| 28 |
def request_gb(self, req: Request, include_output_reservation: bool = False) -> float:
|
| 29 |
tokens = self._allocated_tokens(req, include_output_reservation)
|
|
@@ -33,7 +35,7 @@ class KVCacheModel:
|
|
| 33 |
requests = list(active)
|
| 34 |
if prefill_pending:
|
| 35 |
requests.extend(prefill_pending)
|
| 36 |
-
return sum(self.request_gb(r) for r in requests)
|
| 37 |
|
| 38 |
def can_admit(self, req: Request, active: list[Request], prefill_pending: list[Request] | None = None) -> bool:
|
| 39 |
return self.used_gb(active, prefill_pending) + self.request_gb(req) <= self.capacity_gb + 1e-12
|
|
|
|
| 12 |
self.cfg = cfg
|
| 13 |
self.model_weight_gb = latency_model.model_weight_gb
|
| 14 |
total_vram = latency_model.accelerator.vram_gb
|
|
|
|
| 15 |
remaining = max(0.0, total_vram - self.model_weight_gb - 1.2)
|
| 16 |
self.capacity_gb = remaining * cfg.kv_memory_fraction
|
| 17 |
+
self.shared_prefix_gb = 0.0
|
| 18 |
+
if cfg.prefix_cache_enabled and cfg.shared_prefix_tokens > 0 and cfg.prefix_reuse_fraction > 0:
|
| 19 |
+
self.shared_prefix_gb = cfg.shared_prefix_tokens * latency_model.kv_bytes_per_token() / 1e9
|
| 20 |
|
| 21 |
def _allocated_tokens(self, req: Request, include_output_reservation: bool = False) -> int:
|
| 22 |
+
# Cached prefix state is represented once by shared_prefix_gb. Per-request
|
| 23 |
+
# allocation therefore contains only the uncached suffix + generated state.
|
| 24 |
+
live_tokens = req.uncached_prompt_tokens + req.generated_tokens
|
| 25 |
if self.cfg.scheduler == "static_fcfs" or include_output_reservation:
|
| 26 |
+
return req.uncached_prompt_tokens + req.output_tokens
|
|
|
|
|
|
|
| 27 |
block = max(self.cfg.kv_block_tokens, 1)
|
| 28 |
+
return int(math.ceil(max(live_tokens, 0) / block) * block)
|
| 29 |
|
| 30 |
def request_gb(self, req: Request, include_output_reservation: bool = False) -> float:
|
| 31 |
tokens = self._allocated_tokens(req, include_output_reservation)
|
|
|
|
| 35 |
requests = list(active)
|
| 36 |
if prefill_pending:
|
| 37 |
requests.extend(prefill_pending)
|
| 38 |
+
return self.shared_prefix_gb + sum(self.request_gb(r) for r in requests)
|
| 39 |
|
| 40 |
def can_admit(self, req: Request, active: list[Request], prefill_pending: list[Request] | None = None) -> bool:
|
| 41 |
return self.used_gb(active, prefill_pending) + self.request_gb(req) <= self.capacity_gb + 1e-12
|
src/inferscale/latency.py
CHANGED
|
@@ -29,7 +29,7 @@ class AnalyticalLatencyModel:
|
|
| 29 |
return self.model.params_b * self.weight_bytes_per_param
|
| 30 |
|
| 31 |
def kv_bytes_per_token(self) -> float:
|
| 32 |
-
# K + V, all layers, KV heads only. KV state is assumed fp16 in v0.
|
| 33 |
return (
|
| 34 |
2
|
| 35 |
* self.model.layers
|
|
|
|
| 29 |
return self.model.params_b * self.weight_bytes_per_param
|
| 30 |
|
| 31 |
def kv_bytes_per_token(self) -> float:
|
| 32 |
+
# K + V, all layers, KV heads only. KV state is assumed fp16 in v0.3.
|
| 33 |
return (
|
| 34 |
2
|
| 35 |
* self.model.layers
|
src/inferscale/models.py
CHANGED
|
@@ -36,6 +36,7 @@ class SimulationConfig:
|
|
| 36 |
model: str = "Llama-3.1-8B"
|
| 37 |
accelerator: str = "L4"
|
| 38 |
scheduler: str = "continuous_fcfs"
|
|
|
|
| 39 |
arrival_process: str = "poisson"
|
| 40 |
request_rate_rps: float = 4.0
|
| 41 |
duration_s: float = 60.0
|
|
@@ -57,6 +58,21 @@ class SimulationConfig:
|
|
| 57 |
burst_period_s: float = 10.0
|
| 58 |
timeline_points: int = 300
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
@classmethod
|
| 61 |
def from_dict(cls, data: dict[str, Any]) -> "SimulationConfig":
|
| 62 |
allowed = cls.__dataclass_fields__.keys()
|
|
@@ -74,16 +90,29 @@ class Request:
|
|
| 74 |
output_tokens: int
|
| 75 |
deadline_time: float
|
| 76 |
remaining_prefill: int
|
|
|
|
| 77 |
generated_tokens: int = 0
|
| 78 |
first_prefill_time: float | None = None
|
|
|
|
|
|
|
|
|
|
| 79 |
first_token_time: float | None = None
|
| 80 |
completion_time: float | None = None
|
|
|
|
| 81 |
priority: int = 0
|
| 82 |
|
| 83 |
@property
|
| 84 |
def context_tokens(self) -> int:
|
| 85 |
return self.prompt_tokens + self.generated_tokens
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
@property
|
| 88 |
def complete(self) -> bool:
|
| 89 |
return self.generated_tokens >= self.output_tokens
|
|
@@ -113,6 +142,9 @@ class TimelinePoint:
|
|
| 113 |
completed: int
|
| 114 |
kv_used_gb: float
|
| 115 |
kv_capacity_gb: float
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
@dataclass
|
|
|
|
| 36 |
model: str = "Llama-3.1-8B"
|
| 37 |
accelerator: str = "L4"
|
| 38 |
scheduler: str = "continuous_fcfs"
|
| 39 |
+
topology: str = "colocated"
|
| 40 |
arrival_process: str = "poisson"
|
| 41 |
request_rate_rps: float = 4.0
|
| 42 |
duration_s: float = 60.0
|
|
|
|
| 58 |
burst_period_s: float = 10.0
|
| 59 |
timeline_points: int = 300
|
| 60 |
|
| 61 |
+
# Prefix-cache scenario. v0.3 intentionally models one reusable shared
|
| 62 |
+
# prefix rather than a full radix tree. Hits share one persistent KV entry.
|
| 63 |
+
prefix_cache_enabled: bool = False
|
| 64 |
+
shared_prefix_tokens: int = 256
|
| 65 |
+
prefix_reuse_fraction: float = 0.0
|
| 66 |
+
|
| 67 |
+
# Prefill/decode disaggregation scenario. These fields are ignored for the
|
| 68 |
+
# colocated topology. Interconnect bandwidth is expressed in GB/s.
|
| 69 |
+
prefill_accelerator: str = "L4"
|
| 70 |
+
decode_accelerator: str = "L4"
|
| 71 |
+
prefill_workers: int = 1
|
| 72 |
+
decode_workers: int = 1
|
| 73 |
+
interconnect_gbps: float = 50.0
|
| 74 |
+
transfer_base_ms: float = 0.20
|
| 75 |
+
|
| 76 |
@classmethod
|
| 77 |
def from_dict(cls, data: dict[str, Any]) -> "SimulationConfig":
|
| 78 |
allowed = cls.__dataclass_fields__.keys()
|
|
|
|
| 90 |
output_tokens: int
|
| 91 |
deadline_time: float
|
| 92 |
remaining_prefill: int
|
| 93 |
+
cached_prefix_tokens: int = 0
|
| 94 |
generated_tokens: int = 0
|
| 95 |
first_prefill_time: float | None = None
|
| 96 |
+
prefill_complete_time: float | None = None
|
| 97 |
+
transfer_start_time: float | None = None
|
| 98 |
+
transfer_end_time: float | None = None
|
| 99 |
first_token_time: float | None = None
|
| 100 |
completion_time: float | None = None
|
| 101 |
+
decode_worker_id: int | None = None
|
| 102 |
priority: int = 0
|
| 103 |
|
| 104 |
@property
|
| 105 |
def context_tokens(self) -> int:
|
| 106 |
return self.prompt_tokens + self.generated_tokens
|
| 107 |
|
| 108 |
+
@property
|
| 109 |
+
def uncached_prompt_tokens(self) -> int:
|
| 110 |
+
return max(0, self.prompt_tokens - self.cached_prefix_tokens)
|
| 111 |
+
|
| 112 |
+
@property
|
| 113 |
+
def prefix_cache_hit(self) -> bool:
|
| 114 |
+
return self.cached_prefix_tokens > 0
|
| 115 |
+
|
| 116 |
@property
|
| 117 |
def complete(self) -> bool:
|
| 118 |
return self.generated_tokens >= self.output_tokens
|
|
|
|
| 142 |
completed: int
|
| 143 |
kv_used_gb: float
|
| 144 |
kv_capacity_gb: float
|
| 145 |
+
transfer_pending: int = 0
|
| 146 |
+
decode_ready: int = 0
|
| 147 |
+
prefill_active: int = 0
|
| 148 |
|
| 149 |
|
| 150 |
@dataclass
|
src/inferscale/optimizer.py
CHANGED
|
@@ -40,6 +40,7 @@ def evaluate_rate(base: SimulationConfig, rate: float, repetitions: int = 3) ->
|
|
| 40 |
"max_unfinished": max(unfinished_values),
|
| 41 |
"repetitions": repetitions,
|
| 42 |
"criterion": "all-repetitions-meet-target-and-drain",
|
|
|
|
| 43 |
}
|
| 44 |
|
| 45 |
|
|
@@ -110,6 +111,7 @@ def compare_schedulers(config: dict, schedulers: list[str] | None = None) -> dic
|
|
| 110 |
"chunked_slo",
|
| 111 |
]
|
| 112 |
base = SimulationConfig.from_dict(config)
|
|
|
|
| 113 |
rows = []
|
| 114 |
for scheduler in schedulers:
|
| 115 |
cfg = deepcopy(base)
|
|
@@ -130,3 +132,143 @@ def compare_schedulers(config: dict, schedulers: list[str] | None = None) -> dic
|
|
| 130 |
)
|
| 131 |
rows.sort(key=lambda row: (row["slo_attainment"], row["goodput_rps"]), reverse=True)
|
| 132 |
return {"rows": rows}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
"max_unfinished": max(unfinished_values),
|
| 41 |
"repetitions": repetitions,
|
| 42 |
"criterion": "all-repetitions-meet-target-and-drain",
|
| 43 |
+
"target": base.slo_attainment_target,
|
| 44 |
}
|
| 45 |
|
| 46 |
|
|
|
|
| 111 |
"chunked_slo",
|
| 112 |
]
|
| 113 |
base = SimulationConfig.from_dict(config)
|
| 114 |
+
base.topology = "colocated"
|
| 115 |
rows = []
|
| 116 |
for scheduler in schedulers:
|
| 117 |
cfg = deepcopy(base)
|
|
|
|
| 132 |
)
|
| 133 |
rows.sort(key=lambda row: (row["slo_attainment"], row["goodput_rps"]), reverse=True)
|
| 134 |
return {"rows": rows}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def compare_topologies(config: dict) -> dict:
|
| 138 |
+
"""Compare colocated and P/D-disaggregated serving on one deterministic trace."""
|
| 139 |
+
base = SimulationConfig.from_dict(config)
|
| 140 |
+
rows = []
|
| 141 |
+
scenarios = [
|
| 142 |
+
("colocated", False),
|
| 143 |
+
("colocated", True),
|
| 144 |
+
("disaggregated_pd", False),
|
| 145 |
+
("disaggregated_pd", True),
|
| 146 |
+
]
|
| 147 |
+
for topology, cache_enabled in scenarios:
|
| 148 |
+
cfg = deepcopy(base)
|
| 149 |
+
cfg.topology = topology
|
| 150 |
+
cfg.prefix_cache_enabled = cache_enabled
|
| 151 |
+
if topology == "disaggregated_pd" and cfg.scheduler == "static_fcfs":
|
| 152 |
+
cfg.scheduler = "continuous_fcfs"
|
| 153 |
+
result = run_simulation(cfg.to_dict())
|
| 154 |
+
resource = result["resource"]
|
| 155 |
+
accelerator_instances = int(resource.get("accelerator_instances", 1))
|
| 156 |
+
goodput_rps = result["summary"]["goodput_rps"]
|
| 157 |
+
rows.append(
|
| 158 |
+
{
|
| 159 |
+
"scenario": f"{topology}{'_cache' if cache_enabled else ''}",
|
| 160 |
+
"topology": topology,
|
| 161 |
+
"prefix_cache": cache_enabled,
|
| 162 |
+
"accelerator_instances": accelerator_instances,
|
| 163 |
+
"goodput_rps": goodput_rps,
|
| 164 |
+
"goodput_per_accelerator": goodput_rps / max(accelerator_instances, 1),
|
| 165 |
+
"request_throughput_rps": result["summary"]["request_throughput_rps"],
|
| 166 |
+
"slo_attainment": result["summary"]["slo_attainment"],
|
| 167 |
+
"p95_ttft_ms": result["latency"]["ttft_ms"]["p95"],
|
| 168 |
+
"p95_e2e_ms": result["latency"]["e2e_ms"]["p95"],
|
| 169 |
+
"peak_kv_utilization": resource.get("peak_kv_utilization", 0.0),
|
| 170 |
+
"prefix_hit_rate": resource.get("prefix_cache_hit_rate", 0.0),
|
| 171 |
+
"prefill_tokens_saved": resource.get("prefill_tokens_saved", 0),
|
| 172 |
+
"p95_transfer_ms": resource.get("p95_transfer_ms", 0.0),
|
| 173 |
+
"prefill_busy_fraction": resource.get("prefill_busy_fraction", 0.0),
|
| 174 |
+
"decode_busy_fraction": resource.get("decode_busy_fraction", result["summary"].get("busy_fraction", 0.0)),
|
| 175 |
+
"transfer_busy_fraction": resource.get("transfer_busy_fraction", 0.0),
|
| 176 |
+
"bottleneck": result["diagnostics"]["label"],
|
| 177 |
+
}
|
| 178 |
+
)
|
| 179 |
+
rows.sort(key=lambda row: (row["slo_attainment"], row["goodput_rps"]), reverse=True)
|
| 180 |
+
return {"rows": rows}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _is_dominated(candidate: dict, rows: list[dict], throughput_key: str = "goodput_rps") -> bool:
|
| 184 |
+
for other in rows:
|
| 185 |
+
if other is candidate:
|
| 186 |
+
continue
|
| 187 |
+
no_worse = (
|
| 188 |
+
other[throughput_key] >= candidate[throughput_key]
|
| 189 |
+
and other["p95_ttft_ms"] <= candidate["p95_ttft_ms"]
|
| 190 |
+
)
|
| 191 |
+
strictly_better = (
|
| 192 |
+
other[throughput_key] > candidate[throughput_key]
|
| 193 |
+
or other["p95_ttft_ms"] < candidate["p95_ttft_ms"]
|
| 194 |
+
)
|
| 195 |
+
if no_worse and strictly_better:
|
| 196 |
+
return True
|
| 197 |
+
return False
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def design_space_search(config: dict, include_disaggregated: bool = True) -> dict:
|
| 201 |
+
"""Small browser-safe design-space sweep with a goodput/TTFT Pareto frontier.
|
| 202 |
+
|
| 203 |
+
The sweep is intentionally bounded: its purpose is to expose configuration
|
| 204 |
+
interactions interactively, not to claim exhaustive optimization.
|
| 205 |
+
"""
|
| 206 |
+
base = SimulationConfig.from_dict(config)
|
| 207 |
+
rows: list[dict] = []
|
| 208 |
+
|
| 209 |
+
colocated_candidates = []
|
| 210 |
+
for scheduler in ["continuous_fcfs", "continuous_slo", "chunked_slo"]:
|
| 211 |
+
for batch in [8, 16, 32]:
|
| 212 |
+
colocated_candidates.append((scheduler, batch, False))
|
| 213 |
+
# Prefix reuse is added as a separate systems dimension for the SLO-aware
|
| 214 |
+
# scheduler at the user's current batch size.
|
| 215 |
+
colocated_candidates.append(("continuous_slo", base.max_batch_size, True))
|
| 216 |
+
|
| 217 |
+
for scheduler, batch, cache in colocated_candidates:
|
| 218 |
+
cfg = deepcopy(base)
|
| 219 |
+
cfg.topology = "colocated"
|
| 220 |
+
cfg.scheduler = scheduler
|
| 221 |
+
cfg.max_batch_size = batch
|
| 222 |
+
cfg.prefix_cache_enabled = cache
|
| 223 |
+
result = run_simulation(cfg.to_dict())
|
| 224 |
+
rows.append(_design_row(result, cfg, f"colocated / {scheduler} / batch {batch}{' / cache' if cache else ''}"))
|
| 225 |
+
|
| 226 |
+
if include_disaggregated:
|
| 227 |
+
for prefill_workers, decode_workers in [(1, 1), (1, 2), (2, 1)]:
|
| 228 |
+
for cache in [False, True]:
|
| 229 |
+
cfg = deepcopy(base)
|
| 230 |
+
cfg.topology = "disaggregated_pd"
|
| 231 |
+
cfg.scheduler = "continuous_slo"
|
| 232 |
+
cfg.prefill_workers = prefill_workers
|
| 233 |
+
cfg.decode_workers = decode_workers
|
| 234 |
+
cfg.prefix_cache_enabled = cache
|
| 235 |
+
result = run_simulation(cfg.to_dict())
|
| 236 |
+
label = f"P/D {prefill_workers}P:{decode_workers}D{' / cache' if cache else ''}"
|
| 237 |
+
rows.append(_design_row(result, cfg, label))
|
| 238 |
+
|
| 239 |
+
for row in rows:
|
| 240 |
+
row["pareto"] = not _is_dominated(row, rows, "goodput_rps")
|
| 241 |
+
row["efficiency_pareto"] = not _is_dominated(row, rows, "goodput_per_accelerator")
|
| 242 |
+
rows.sort(key=lambda r: (not r["pareto"], not r["slo_pass"], -r["goodput_rps"], r["p95_ttft_ms"]))
|
| 243 |
+
return {
|
| 244 |
+
"rows": rows,
|
| 245 |
+
"pareto_count": sum(1 for r in rows if r["pareto"]),
|
| 246 |
+
"efficiency_pareto_count": sum(1 for r in rows if r["efficiency_pareto"]),
|
| 247 |
+
"candidate_count": len(rows),
|
| 248 |
+
"objectives": [
|
| 249 |
+
"performance: maximize goodput / minimize p95 TTFT",
|
| 250 |
+
"efficiency: maximize goodput per accelerator / minimize p95 TTFT",
|
| 251 |
+
],
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _design_row(result: dict, cfg: SimulationConfig, label: str) -> dict:
|
| 256 |
+
resource = result["resource"]
|
| 257 |
+
accelerator_instances = int(resource.get("accelerator_instances", 1))
|
| 258 |
+
goodput_rps = result["summary"]["goodput_rps"]
|
| 259 |
+
return {
|
| 260 |
+
"label": label,
|
| 261 |
+
"topology": cfg.topology,
|
| 262 |
+
"scheduler": cfg.scheduler,
|
| 263 |
+
"max_batch_size": cfg.max_batch_size,
|
| 264 |
+
"prefix_cache": cfg.prefix_cache_enabled,
|
| 265 |
+
"accelerator_instances": accelerator_instances,
|
| 266 |
+
"goodput_rps": goodput_rps,
|
| 267 |
+
"goodput_per_accelerator": goodput_rps / max(accelerator_instances, 1),
|
| 268 |
+
"slo_attainment": result["summary"]["slo_attainment"],
|
| 269 |
+
"slo_pass": result["summary"]["slo_attainment"] >= cfg.slo_attainment_target and result["summary"]["requests_unfinished"] == 0,
|
| 270 |
+
"p95_ttft_ms": result["latency"]["ttft_ms"]["p95"],
|
| 271 |
+
"p95_e2e_ms": result["latency"]["e2e_ms"]["p95"],
|
| 272 |
+
"peak_kv_utilization": resource.get("peak_kv_utilization", 0.0),
|
| 273 |
+
"bottleneck": result["diagnostics"]["label"],
|
| 274 |
+
}
|
src/inferscale/simulator.py
CHANGED
|
@@ -121,12 +121,12 @@ class Simulator:
|
|
| 121 |
for req in self._waiting_sorted():
|
| 122 |
if len(selected) >= slots:
|
| 123 |
break
|
| 124 |
-
if selected and total_tokens + req.
|
| 125 |
continue
|
| 126 |
if not self.kv.can_admit(req, self.active, selected):
|
| 127 |
continue
|
| 128 |
selected.append(req)
|
| 129 |
-
total_tokens += req.
|
| 130 |
|
| 131 |
if not selected:
|
| 132 |
return False
|
|
@@ -135,7 +135,7 @@ class Simulator:
|
|
| 135 |
for req in selected:
|
| 136 |
if req.first_prefill_time is None:
|
| 137 |
req.first_prefill_time = self.now
|
| 138 |
-
self._advance(self.latency.prefill_seconds([r.
|
| 139 |
for req in selected:
|
| 140 |
req.remaining_prefill = 0
|
| 141 |
self.active.append(req)
|
|
@@ -177,7 +177,7 @@ class Simulator:
|
|
| 177 |
# One prefill chunk. Decode is serviced on the next loop iteration,
|
| 178 |
# producing the intended prefill/decode interleaving.
|
| 179 |
self._advance(self.latency.prefill_seconds(chunks))
|
| 180 |
-
for req, chunk in zip(selected, chunks):
|
| 181 |
req.remaining_prefill -= chunk
|
| 182 |
if req.remaining_prefill <= 0:
|
| 183 |
self.prefill_pending.remove(req)
|
|
@@ -217,7 +217,7 @@ class Simulator:
|
|
| 217 |
self.waiting = [r for r in self.waiting if r.request_id not in ids]
|
| 218 |
for req in admitted:
|
| 219 |
req.first_prefill_time = self.now
|
| 220 |
-
self._advance(self.latency.prefill_seconds([r.
|
| 221 |
for req in admitted:
|
| 222 |
req.remaining_prefill = 0
|
| 223 |
self.active.append(req)
|
|
@@ -282,6 +282,12 @@ class Simulator:
|
|
| 282 |
"peak_kv_gb": self.peak_kv_gb,
|
| 283 |
"peak_kv_utilization": self.peak_kv_gb / self.kv.capacity_gb if self.kv.capacity_gb > 0 else 0.0,
|
| 284 |
"accelerator_vram_gb": self.accelerator.vram_gb,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
}
|
| 286 |
|
| 287 |
request_rows = []
|
|
@@ -293,6 +299,7 @@ class Simulator:
|
|
| 293 |
"arrival_time": req.arrival_time,
|
| 294 |
"prompt_tokens": req.prompt_tokens,
|
| 295 |
"output_tokens": req.output_tokens,
|
|
|
|
| 296 |
"ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
|
| 297 |
"e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
|
| 298 |
})
|
|
@@ -301,11 +308,12 @@ class Simulator:
|
|
| 301 |
|
| 302 |
provenance = {
|
| 303 |
"simulator": "InferScale-Sim",
|
| 304 |
-
"version": "0.
|
| 305 |
"latency_profile_type": "analytical-reference",
|
| 306 |
"profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
|
| 307 |
"model_profile_source": self.model.source,
|
| 308 |
"accelerator_profile_source": self.accelerator.source,
|
|
|
|
| 309 |
}
|
| 310 |
|
| 311 |
return SimulationResult(
|
|
@@ -322,4 +330,11 @@ class Simulator:
|
|
| 322 |
|
| 323 |
|
| 324 |
def run_simulation(config: dict) -> dict:
|
| 325 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
for req in self._waiting_sorted():
|
| 122 |
if len(selected) >= slots:
|
| 123 |
break
|
| 124 |
+
if selected and total_tokens + req.remaining_prefill > self.cfg.max_batch_tokens:
|
| 125 |
continue
|
| 126 |
if not self.kv.can_admit(req, self.active, selected):
|
| 127 |
continue
|
| 128 |
selected.append(req)
|
| 129 |
+
total_tokens += req.remaining_prefill
|
| 130 |
|
| 131 |
if not selected:
|
| 132 |
return False
|
|
|
|
| 135 |
for req in selected:
|
| 136 |
if req.first_prefill_time is None:
|
| 137 |
req.first_prefill_time = self.now
|
| 138 |
+
self._advance(self.latency.prefill_seconds([r.remaining_prefill for r in selected]))
|
| 139 |
for req in selected:
|
| 140 |
req.remaining_prefill = 0
|
| 141 |
self.active.append(req)
|
|
|
|
| 177 |
# One prefill chunk. Decode is serviced on the next loop iteration,
|
| 178 |
# producing the intended prefill/decode interleaving.
|
| 179 |
self._advance(self.latency.prefill_seconds(chunks))
|
| 180 |
+
for req, chunk in zip(selected, chunks, strict=True):
|
| 181 |
req.remaining_prefill -= chunk
|
| 182 |
if req.remaining_prefill <= 0:
|
| 183 |
self.prefill_pending.remove(req)
|
|
|
|
| 217 |
self.waiting = [r for r in self.waiting if r.request_id not in ids]
|
| 218 |
for req in admitted:
|
| 219 |
req.first_prefill_time = self.now
|
| 220 |
+
self._advance(self.latency.prefill_seconds([r.remaining_prefill for r in admitted]))
|
| 221 |
for req in admitted:
|
| 222 |
req.remaining_prefill = 0
|
| 223 |
self.active.append(req)
|
|
|
|
| 282 |
"peak_kv_gb": self.peak_kv_gb,
|
| 283 |
"peak_kv_utilization": self.peak_kv_gb / self.kv.capacity_gb if self.kv.capacity_gb > 0 else 0.0,
|
| 284 |
"accelerator_vram_gb": self.accelerator.vram_gb,
|
| 285 |
+
"topology": "colocated",
|
| 286 |
+
"accelerator_instances": 1,
|
| 287 |
+
"prefix_cache_gb": self.kv.shared_prefix_gb,
|
| 288 |
+
"prefix_cache_hits": sum(1 for r in self.requests if r.prefix_cache_hit),
|
| 289 |
+
"prefix_cache_hit_rate": (sum(1 for r in self.requests if r.prefix_cache_hit) / len(self.requests)) if self.requests else 0.0,
|
| 290 |
+
"prefill_tokens_saved": sum(r.cached_prefix_tokens for r in self.requests),
|
| 291 |
}
|
| 292 |
|
| 293 |
request_rows = []
|
|
|
|
| 299 |
"arrival_time": req.arrival_time,
|
| 300 |
"prompt_tokens": req.prompt_tokens,
|
| 301 |
"output_tokens": req.output_tokens,
|
| 302 |
+
"cached_prefix_tokens": req.cached_prefix_tokens,
|
| 303 |
"ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
|
| 304 |
"e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
|
| 305 |
})
|
|
|
|
| 308 |
|
| 309 |
provenance = {
|
| 310 |
"simulator": "InferScale-Sim",
|
| 311 |
+
"version": "0.3.0",
|
| 312 |
"latency_profile_type": "analytical-reference",
|
| 313 |
"profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
|
| 314 |
"model_profile_source": self.model.source,
|
| 315 |
"accelerator_profile_source": self.accelerator.source,
|
| 316 |
+
"topology": "colocated",
|
| 317 |
}
|
| 318 |
|
| 319 |
return SimulationResult(
|
|
|
|
| 330 |
|
| 331 |
|
| 332 |
def run_simulation(config: dict) -> dict:
|
| 333 |
+
cfg = SimulationConfig.from_dict(config)
|
| 334 |
+
if cfg.topology == "disaggregated_pd":
|
| 335 |
+
from .disaggregated import DisaggregatedSimulator
|
| 336 |
+
|
| 337 |
+
return DisaggregatedSimulator(cfg).run().to_dict()
|
| 338 |
+
if cfg.topology != "colocated":
|
| 339 |
+
raise ValueError(f"Unsupported topology: {cfg.topology}")
|
| 340 |
+
return Simulator(cfg).run().to_dict()
|
src/inferscale/workloads.py
CHANGED
|
@@ -29,9 +29,6 @@ def _arrival_times(cfg: SimulationConfig, rng: random.Random) -> list[float]:
|
|
| 29 |
return arrivals
|
| 30 |
|
| 31 |
if cfg.arrival_process == "bursty":
|
| 32 |
-
# Alternating baseline/high-load windows. The mean offered load is not
|
| 33 |
-
# forced to equal request_rate_rps; the UI labels this clearly as a burst
|
| 34 |
-
# stress test rather than an average-rate generator.
|
| 35 |
while t < cfg.duration_s:
|
| 36 |
phase = int(t // max(cfg.burst_period_s, 0.1)) % 2
|
| 37 |
local_rate = rate * (cfg.burst_multiplier if phase else 0.55)
|
|
@@ -52,10 +49,15 @@ def _arrival_times(cfg: SimulationConfig, rng: random.Random) -> list[float]:
|
|
| 52 |
|
| 53 |
def generate_workload(cfg: SimulationConfig) -> list[Request]:
|
| 54 |
rng = random.Random(cfg.seed)
|
|
|
|
| 55 |
requests: list[Request] = []
|
| 56 |
for idx, arrival in enumerate(_arrival_times(cfg, rng)):
|
| 57 |
prompt = _sample_lognormal(cfg.prompt_tokens_mean, cfg.prompt_tokens_cv, rng)
|
| 58 |
output = _sample_lognormal(cfg.output_tokens_mean, cfg.output_tokens_cv, rng)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
requests.append(
|
| 60 |
Request(
|
| 61 |
request_id=idx,
|
|
@@ -63,7 +65,8 @@ def generate_workload(cfg: SimulationConfig) -> list[Request]:
|
|
| 63 |
prompt_tokens=prompt,
|
| 64 |
output_tokens=output,
|
| 65 |
deadline_time=arrival + cfg.slo_e2e_ms / 1000.0,
|
| 66 |
-
remaining_prefill=prompt,
|
|
|
|
| 67 |
)
|
| 68 |
)
|
| 69 |
return requests
|
|
|
|
| 29 |
return arrivals
|
| 30 |
|
| 31 |
if cfg.arrival_process == "bursty":
|
|
|
|
|
|
|
|
|
|
| 32 |
while t < cfg.duration_s:
|
| 33 |
phase = int(t // max(cfg.burst_period_s, 0.1)) % 2
|
| 34 |
local_rate = rate * (cfg.burst_multiplier if phase else 0.55)
|
|
|
|
| 49 |
|
| 50 |
def generate_workload(cfg: SimulationConfig) -> list[Request]:
|
| 51 |
rng = random.Random(cfg.seed)
|
| 52 |
+
cache_rng = random.Random(cfg.seed ^ 0x5A17CACE)
|
| 53 |
requests: list[Request] = []
|
| 54 |
for idx, arrival in enumerate(_arrival_times(cfg, rng)):
|
| 55 |
prompt = _sample_lognormal(cfg.prompt_tokens_mean, cfg.prompt_tokens_cv, rng)
|
| 56 |
output = _sample_lognormal(cfg.output_tokens_mean, cfg.output_tokens_cv, rng)
|
| 57 |
+
cached = 0
|
| 58 |
+
if cfg.prefix_cache_enabled and cfg.shared_prefix_tokens > 0 and cfg.prefix_reuse_fraction > 0:
|
| 59 |
+
if cache_rng.random() < min(max(cfg.prefix_reuse_fraction, 0.0), 1.0):
|
| 60 |
+
cached = min(cfg.shared_prefix_tokens, max(prompt - 1, 0))
|
| 61 |
requests.append(
|
| 62 |
Request(
|
| 63 |
request_id=idx,
|
|
|
|
| 65 |
prompt_tokens=prompt,
|
| 66 |
output_tokens=output,
|
| 67 |
deadline_time=arrival + cfg.slo_e2e_ms / 1000.0,
|
| 68 |
+
remaining_prefill=max(0, prompt - cached),
|
| 69 |
+
cached_prefix_tokens=cached,
|
| 70 |
)
|
| 71 |
)
|
| 72 |
return requests
|
styles.css
CHANGED
|
@@ -1,12 +1,288 @@
|
|
| 1 |
-
:root{
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
:root {
|
| 2 |
+
--bg: #090d12;
|
| 3 |
+
--surface: #0d1219;
|
| 4 |
+
--surface-2: #0a0f15;
|
| 5 |
+
--surface-3: #111822;
|
| 6 |
+
--line: #26313f;
|
| 7 |
+
--line-soft: #1b2430;
|
| 8 |
+
--text: #e4e9ef;
|
| 9 |
+
--muted: #929dab;
|
| 10 |
+
--faint: #697586;
|
| 11 |
+
--accent: #6f9fe8;
|
| 12 |
+
--accent-2: #9ab8df;
|
| 13 |
+
--good: #6fd0a0;
|
| 14 |
+
--danger: #ee7f8d;
|
| 15 |
+
--amber: #d8b96e;
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
* { box-sizing: border-box; }
|
| 19 |
+
html { color-scheme: dark; }
|
| 20 |
+
body {
|
| 21 |
+
margin: 0;
|
| 22 |
+
background: var(--bg);
|
| 23 |
+
color: var(--text);
|
| 24 |
+
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
| 25 |
+
font-size: 14px;
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
.topbar {
|
| 29 |
+
position: fixed;
|
| 30 |
+
inset: 0 0 auto 0;
|
| 31 |
+
z-index: 80;
|
| 32 |
+
height: 64px;
|
| 33 |
+
padding: 0 24px;
|
| 34 |
+
display: flex;
|
| 35 |
+
align-items: center;
|
| 36 |
+
justify-content: space-between;
|
| 37 |
+
gap: 16px;
|
| 38 |
+
background: #090d12;
|
| 39 |
+
border-bottom: 1px solid var(--line);
|
| 40 |
+
transition: transform .18s ease;
|
| 41 |
+
}
|
| 42 |
+
body.runtime-ready .topbar { transform: translateY(-58px); }
|
| 43 |
+
body.runtime-ready .topbar:hover,
|
| 44 |
+
body.runtime-ready .topbar:focus-within { transform: translateY(0); }
|
| 45 |
+
.brand-wrap { display: flex; align-items: center; min-width: 0; }
|
| 46 |
+
.brand { font-size: 14px; font-weight: 760; letter-spacing: -.01em; }
|
| 47 |
+
.subtitle { margin-top: 2px; color: var(--faint); font-size: 10px; }
|
| 48 |
+
.runtime-pill {
|
| 49 |
+
display: inline-flex;
|
| 50 |
+
align-items: center;
|
| 51 |
+
gap: 7px;
|
| 52 |
+
color: var(--muted);
|
| 53 |
+
border: 1px solid var(--line);
|
| 54 |
+
background: var(--surface-2);
|
| 55 |
+
border-radius: 4px;
|
| 56 |
+
padding: 6px 9px;
|
| 57 |
+
font-size: 10px;
|
| 58 |
+
white-space: nowrap;
|
| 59 |
+
}
|
| 60 |
+
.runtime-pill .dot { width: 6px; height: 6px; border-radius: 50%; background: var(--amber); }
|
| 61 |
+
.runtime-pill.ready .dot { background: var(--good); }
|
| 62 |
+
.runtime-pill.error .dot { background: var(--danger); }
|
| 63 |
+
|
| 64 |
+
.shell {
|
| 65 |
+
width: min(1460px, calc(100% - 40px));
|
| 66 |
+
margin: 0 auto;
|
| 67 |
+
padding: 94px 0 44px;
|
| 68 |
+
transition: padding-top .18s ease;
|
| 69 |
+
}
|
| 70 |
+
body.runtime-ready .shell { padding-top: 34px; }
|
| 71 |
+
|
| 72 |
+
.intro {
|
| 73 |
+
display: grid;
|
| 74 |
+
grid-template-columns: minmax(0, 1.4fr) minmax(320px, .6fr);
|
| 75 |
+
gap: 46px;
|
| 76 |
+
align-items: end;
|
| 77 |
+
padding: 16px 0 18px;
|
| 78 |
+
}
|
| 79 |
+
.eyebrow, .section-kicker {
|
| 80 |
+
color: var(--accent-2);
|
| 81 |
+
font-size: 9px;
|
| 82 |
+
line-height: 1.2;
|
| 83 |
+
text-transform: uppercase;
|
| 84 |
+
letter-spacing: .1em;
|
| 85 |
+
font-weight: 760;
|
| 86 |
+
}
|
| 87 |
+
.intro h1 {
|
| 88 |
+
max-width: 900px;
|
| 89 |
+
margin: 8px 0 9px;
|
| 90 |
+
color: #edf1f5;
|
| 91 |
+
font-size: clamp(28px, 3vw, 43px);
|
| 92 |
+
line-height: 1.06;
|
| 93 |
+
letter-spacing: -.038em;
|
| 94 |
+
font-weight: 720;
|
| 95 |
+
}
|
| 96 |
+
.intro p { max-width: 920px; margin: 0; color: var(--muted); font-size: 13px; line-height: 1.65; }
|
| 97 |
+
.project-facts { margin: 0; border-top: 1px solid var(--line); }
|
| 98 |
+
.project-facts div { display: grid; grid-template-columns: 120px 1fr; gap: 14px; padding: 9px 0; border-bottom: 1px solid var(--line-soft); }
|
| 99 |
+
.project-facts dt { color: var(--faint); font-size: 10px; }
|
| 100 |
+
.project-facts dd { margin: 0; color: #bdc7d2; font-size: 11px; }
|
| 101 |
+
.reference-note {
|
| 102 |
+
border-left: 2px solid #6c6041;
|
| 103 |
+
background: #11120f;
|
| 104 |
+
color: #bcb5a5;
|
| 105 |
+
padding: 10px 12px;
|
| 106 |
+
font-size: 11px;
|
| 107 |
+
line-height: 1.55;
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
.tabs {
|
| 111 |
+
display: flex;
|
| 112 |
+
gap: 2px;
|
| 113 |
+
padding: 22px 0 10px;
|
| 114 |
+
overflow-x: auto;
|
| 115 |
+
border-bottom: 1px solid var(--line-soft);
|
| 116 |
+
margin-bottom: 14px;
|
| 117 |
+
}
|
| 118 |
+
.tab {
|
| 119 |
+
border: 0;
|
| 120 |
+
border-bottom: 2px solid transparent;
|
| 121 |
+
color: var(--muted);
|
| 122 |
+
background: transparent;
|
| 123 |
+
padding: 9px 11px;
|
| 124 |
+
cursor: pointer;
|
| 125 |
+
font-weight: 650;
|
| 126 |
+
white-space: nowrap;
|
| 127 |
+
}
|
| 128 |
+
.tab:hover { color: var(--text); }
|
| 129 |
+
.tab.active { color: var(--text); border-bottom-color: var(--accent); }
|
| 130 |
+
.tab-panel { display: none; }
|
| 131 |
+
.tab-panel.active { display: block; }
|
| 132 |
+
|
| 133 |
+
.workspace { display: grid; grid-template-columns: 405px minmax(0, 1fr); gap: 14px; align-items: start; }
|
| 134 |
+
.planner-grid { grid-template-columns: 360px minmax(0, 1fr); }
|
| 135 |
+
.panel { background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
|
| 136 |
+
.controls-panel { padding: 18px; position: sticky; top: 16px; }
|
| 137 |
+
.result-panel, .wide-panel { padding: 20px; }
|
| 138 |
+
.panel-title-row { display: flex; align-items: flex-start; justify-content: space-between; gap: 14px; margin-bottom: 16px; }
|
| 139 |
+
.panel-title-row h2 { margin: 0; font-size: 17px; letter-spacing: -.02em; }
|
| 140 |
+
.panel-title-row p { margin: 6px 0 0; }
|
| 141 |
+
.tag {
|
| 142 |
+
display: inline-flex;
|
| 143 |
+
align-items: center;
|
| 144 |
+
border: 1px solid var(--line);
|
| 145 |
+
background: var(--surface-2);
|
| 146 |
+
color: var(--muted);
|
| 147 |
+
border-radius: 4px;
|
| 148 |
+
padding: 5px 7px;
|
| 149 |
+
font-size: 10px;
|
| 150 |
+
white-space: nowrap;
|
| 151 |
+
}
|
| 152 |
+
.tag.good { color: var(--good); border-color: #315e4a; }
|
| 153 |
+
.tag.bad { color: var(--danger); border-color: #643943; }
|
| 154 |
+
|
| 155 |
+
label { display: block; color: #b5bfcb; font-size: 11px; font-weight: 650; position: relative; }
|
| 156 |
+
select, input[type="number"] {
|
| 157 |
+
width: 100%;
|
| 158 |
+
margin-top: 7px;
|
| 159 |
+
height: 37px;
|
| 160 |
+
border: 1px solid var(--line);
|
| 161 |
+
background: #0a0f15;
|
| 162 |
+
color: var(--text);
|
| 163 |
+
border-radius: 4px;
|
| 164 |
+
padding: 0 9px;
|
| 165 |
+
outline: none;
|
| 166 |
+
}
|
| 167 |
+
select:focus, input[type="number"]:focus { border-color: #5475a8; box-shadow: 0 0 0 2px rgba(84,117,168,.16); }
|
| 168 |
+
.unit { position: absolute; right: 9px; bottom: 11px; color: #697687; font-size: 9px; pointer-events: none; }
|
| 169 |
+
.field-grid { display: grid; gap: 10px; margin-bottom: 10px; }
|
| 170 |
+
.field-grid.two { grid-template-columns: 1fr 1fr; }
|
| 171 |
+
.subcontrols { margin: 12px 0 4px; padding: 12px; border: 1px solid var(--line-soft); background: #0a0f15; }
|
| 172 |
+
.subcontrols .section-kicker { margin-bottom: 10px; }
|
| 173 |
+
hr { border: 0; border-top: 1px solid var(--line-soft); margin: 17px 0; }
|
| 174 |
+
.section-kicker { margin-bottom: 10px; }
|
| 175 |
+
.hidden { display: none !important; }
|
| 176 |
+
|
| 177 |
+
button.primary, button.secondary {
|
| 178 |
+
height: 40px;
|
| 179 |
+
border-radius: 4px;
|
| 180 |
+
font-weight: 700;
|
| 181 |
+
cursor: pointer;
|
| 182 |
+
transition: .12s ease;
|
| 183 |
+
}
|
| 184 |
+
button.primary { width: 100%; border: 1px solid #668fd5; background: #4f7fce; color: white; margin-top: 11px; }
|
| 185 |
+
button.primary:hover:not(:disabled) { background: #5b8bd8; }
|
| 186 |
+
button.secondary { border: 1px solid var(--line); background: #0b1016; color: #c4ccd7; }
|
| 187 |
+
button.secondary:hover:not(:disabled),
|
| 188 |
+
.mini-button:hover:not(:disabled),
|
| 189 |
+
.chart-expand:hover,
|
| 190 |
+
.chart-download:hover { border-color: #536174; background: #151b24; color: #d9e0e8; }
|
| 191 |
+
button:disabled { opacity: .42; cursor: not-allowed; }
|
| 192 |
+
.compact { width: auto !important; min-width: 178px; padding: 0 17px; margin-top: 0 !important; }
|
| 193 |
+
.button-row { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; margin-top: 8px; }
|
| 194 |
+
.action-stack { display: flex; flex-direction: column; align-items: flex-end; gap: 8px; }
|
| 195 |
+
.checkline { display: flex; align-items: center; gap: 7px; color: var(--muted); font-size: 10px; font-weight: 600; }
|
| 196 |
+
.checkline input { accent-color: var(--accent); }
|
| 197 |
+
|
| 198 |
+
.empty-state { min-height: 380px; display: grid; place-content: center; text-align: center; color: var(--muted); padding: 30px; }
|
| 199 |
+
.empty-state.small { min-height: 240px; }
|
| 200 |
+
.empty-state h3 { color: #ccd4de; margin: 0 0 6px; font-size: 15px; }
|
| 201 |
+
.empty-state p { max-width: 560px; margin: 0; line-height: 1.55; font-size: 12px; }
|
| 202 |
+
.metric-grid { display: grid; grid-template-columns: repeat(6, 1fr); gap: 8px; margin-bottom: 12px; }
|
| 203 |
+
.metric-grid.four { grid-template-columns: repeat(4, 1fr); }
|
| 204 |
+
.metric { min-height: 82px; padding: 13px; border: 1px solid var(--line-soft); background: var(--surface-2); }
|
| 205 |
+
.metric span { display: block; color: var(--muted); font-size: 10px; margin-bottom: 9px; }
|
| 206 |
+
.metric strong { font-size: 18px; letter-spacing: -.025em; }
|
| 207 |
+
.metric strong.metric-small { font-size: 13px; line-height: 1.35; }
|
| 208 |
+
.metric.emphasis { border-color: #486da4; background: #101824; }
|
| 209 |
+
|
| 210 |
+
.diagnostic-card { margin-bottom: 12px; border: 1px solid var(--line); background: #0d131b; padding: 14px; }
|
| 211 |
+
.diagnostic-head { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; }
|
| 212 |
+
.diagnostic-head span { color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: .06em; }
|
| 213 |
+
.diagnostic-head strong { font-size: 14px; }
|
| 214 |
+
.diagnostic-card p { margin: 8px 0 0; color: #b8c1cd; font-size: 12px; line-height: 1.55; }
|
| 215 |
+
.diagnostic-card .diagnostic-action { color: var(--muted); }
|
| 216 |
+
.evidence-row { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 10px; }
|
| 217 |
+
.evidence-row span { border: 1px solid var(--line-soft); background: #090e14; padding: 4px 6px; color: #8390a0; font: 10px/1.2 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
|
| 218 |
+
|
| 219 |
+
.chart-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
|
| 220 |
+
.chart-card { border: 1px solid var(--line); background: var(--surface-2); min-height: 300px; position: relative; }
|
| 221 |
+
.chart-card.full { margin-top: 10px; }
|
| 222 |
+
.chart-head { height: 42px; display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 0 12px; border-bottom: 1px solid var(--line-soft); }
|
| 223 |
+
.chart-title { color: #c7d0db; font-size: 11px; font-weight: 700; }
|
| 224 |
+
.chart-actions { display: flex; gap: 6px; }
|
| 225 |
+
.chart-expand, .chart-download, .mini-button { border: 1px solid var(--line); background: #0a0f15; color: var(--muted); border-radius: 3px; cursor: pointer; height: 28px; padding: 0 9px; font-size: 10px; }
|
| 226 |
+
.chart-body { height: 260px; padding: 10px 12px 12px; }
|
| 227 |
+
.chart-body.large { height: 350px; }
|
| 228 |
+
.chart-card.chart-expanded { position: fixed; inset: 16px; z-index: 100; background: #0a0f15; border-color: #4c596b; box-shadow: 0 0 0 9999px rgba(0,0,0,.76); min-height: 0; margin: 0; }
|
| 229 |
+
.chart-card.chart-expanded .chart-body { height: calc(100vh - 80px); }
|
| 230 |
+
body.chart-open { overflow: hidden; }
|
| 231 |
+
canvas { width: 100% !important; height: 100% !important; max-height: none; }
|
| 232 |
+
|
| 233 |
+
.warnings { margin-top: 10px; border-left: 3px solid #7b6840; padding: 9px 11px; background: #15140f; color: #c4bca9; font-size: 11px; line-height: 1.5; }
|
| 234 |
+
.muted { color: var(--muted); font-size: 12px; line-height: 1.55; }
|
| 235 |
+
.table-toolbar { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 14px; color: #aab4c1; font-size: 11px; }
|
| 236 |
+
.table-toolbar > div { display: flex; gap: 6px; }
|
| 237 |
+
.table-wrap { overflow: auto; margin-top: 7px; border: 1px solid var(--line); }
|
| 238 |
+
table { width: 100%; border-collapse: collapse; font-size: 12px; min-width: 900px; }
|
| 239 |
+
th, td { padding: 10px 11px; text-align: right; border-bottom: 1px solid var(--line-soft); white-space: nowrap; }
|
| 240 |
+
th:first-child, td:first-child { text-align: left; }
|
| 241 |
+
th { color: #7f8b9b; font-size: 9px; text-transform: uppercase; letter-spacing: .06em; background: #090e14; position: sticky; top: 0; }
|
| 242 |
+
td { color: #c7d0db; }
|
| 243 |
+
tbody tr:last-child td { border-bottom: 0; }
|
| 244 |
+
tbody tr:hover td { background: #111821; }
|
| 245 |
+
.pass { color: var(--good); }
|
| 246 |
+
.fail { color: var(--danger); }
|
| 247 |
+
.best-label, .pareto-label { display: inline-block; margin-left: 7px; border: 1px solid #4a6386; color: #9ab8df; padding: 2px 5px; border-radius: 2px; font-size: 9px; text-transform: uppercase; }
|
| 248 |
+
.planner-note { color: var(--muted); border-left: 2px solid #445a77; padding: 7px 9px; margin-bottom: 2px; font-size: 11px; line-height: 1.5; }
|
| 249 |
+
|
| 250 |
+
.method-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
|
| 251 |
+
.prose { padding: 22px; }
|
| 252 |
+
.prose h2 { font-size: 20px; margin: 8px 0 11px; }
|
| 253 |
+
.prose p { color: var(--muted); line-height: 1.7; font-size: 13px; }
|
| 254 |
+
.prose code { color: #a8bfe4; }
|
| 255 |
+
.formula { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; color: #b7c3d2; padding: 10px; border: 1px solid var(--line); background: #0a0f15; font-size: 11px; }
|
| 256 |
+
.wide-method { grid-column: 1 / -1; }
|
| 257 |
+
.paper-grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 8px; margin-top: 18px; }
|
| 258 |
+
.paper-grid > div { padding: 12px; border: 1px solid var(--line-soft); background: #0c1219; }
|
| 259 |
+
.paper-grid strong { display: block; font-size: 11px; margin-bottom: 7px; }
|
| 260 |
+
.paper-grid span { color: var(--muted); font-size: 10px; line-height: 1.55; display: block; }
|
| 261 |
+
.toast { position: fixed; right: 20px; bottom: 20px; z-index: 130; background: #17202b; border: 1px solid #465467; color: #d5dce5; padding: 9px 12px; border-radius: 4px; font-size: 11px; opacity: 0; transform: translateY(8px); pointer-events: none; transition: .16s ease; }
|
| 262 |
+
.toast.show { opacity: 1; transform: translateY(0); }
|
| 263 |
+
|
| 264 |
+
@media (max-width: 1100px) {
|
| 265 |
+
.intro { grid-template-columns: 1fr; gap: 24px; }
|
| 266 |
+
.workspace, .planner-grid { grid-template-columns: 1fr; }
|
| 267 |
+
.controls-panel { position: static; }
|
| 268 |
+
.metric-grid { grid-template-columns: repeat(3, 1fr); }
|
| 269 |
+
.paper-grid { grid-template-columns: 1fr 1fr; }
|
| 270 |
+
}
|
| 271 |
+
|
| 272 |
+
@media (max-width: 720px) {
|
| 273 |
+
.topbar { padding: 0 15px; }
|
| 274 |
+
.subtitle { display: none; }
|
| 275 |
+
.runtime-pill { max-width: 170px; }
|
| 276 |
+
.shell { width: calc(100% - 20px); padding-top: 88px; }
|
| 277 |
+
body.runtime-ready .shell { padding-top: 20px; }
|
| 278 |
+
.intro { padding-top: 10px; }
|
| 279 |
+
.project-facts div { grid-template-columns: 100px 1fr; }
|
| 280 |
+
.field-grid.two, .chart-grid, .method-grid { grid-template-columns: 1fr; }
|
| 281 |
+
.metric-grid, .metric-grid.four { grid-template-columns: 1fr 1fr; }
|
| 282 |
+
.paper-grid { grid-template-columns: 1fr; }
|
| 283 |
+
.panel-title-row { flex-direction: column; }
|
| 284 |
+
.arena-title-row .compact { width: 100% !important; }
|
| 285 |
+
.action-stack { width: 100%; align-items: stretch; }
|
| 286 |
+
.wide-method { grid-column: auto; }
|
| 287 |
+
.chart-card.chart-expanded { inset: 6px; }
|
| 288 |
+
}
|
tests/test_design_space.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from inferscale import compare_topologies, design_space_search
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def test_topology_comparison_has_four_scenarios():
|
| 5 |
+
result = compare_topologies({
|
| 6 |
+
"model": "Qwen2.5-3B",
|
| 7 |
+
"accelerator": "L4",
|
| 8 |
+
"prefill_accelerator": "L4",
|
| 9 |
+
"decode_accelerator": "L4",
|
| 10 |
+
"quantization": "int8",
|
| 11 |
+
"duration_s": 3,
|
| 12 |
+
"request_rate_rps": 1,
|
| 13 |
+
"prompt_tokens_mean": 64,
|
| 14 |
+
"output_tokens_mean": 4,
|
| 15 |
+
"prefix_reuse_fraction": 0.5,
|
| 16 |
+
"shared_prefix_tokens": 32,
|
| 17 |
+
})
|
| 18 |
+
assert len(result["rows"]) == 4
|
| 19 |
+
assert {row["topology"] for row in result["rows"]} == {"colocated", "disaggregated_pd"}
|
| 20 |
+
assert all(row["goodput_per_accelerator"] >= 0 for row in result["rows"])
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_design_space_marks_pareto_candidates():
|
| 24 |
+
result = design_space_search({
|
| 25 |
+
"model": "Qwen2.5-3B",
|
| 26 |
+
"accelerator": "L4",
|
| 27 |
+
"prefill_accelerator": "L4",
|
| 28 |
+
"decode_accelerator": "L4",
|
| 29 |
+
"quantization": "int8",
|
| 30 |
+
"duration_s": 2,
|
| 31 |
+
"request_rate_rps": 0.8,
|
| 32 |
+
"prompt_tokens_mean": 64,
|
| 33 |
+
"output_tokens_mean": 4,
|
| 34 |
+
"shared_prefix_tokens": 32,
|
| 35 |
+
"prefix_reuse_fraction": 0.5,
|
| 36 |
+
}, include_disaggregated=False)
|
| 37 |
+
assert result["candidate_count"] == 10
|
| 38 |
+
assert result["pareto_count"] >= 1
|
| 39 |
+
assert result["efficiency_pareto_count"] >= 1
|
| 40 |
+
assert any(row["pareto"] for row in result["rows"])
|
| 41 |
+
assert any(row["efficiency_pareto"] for row in result["rows"])
|
tests/test_disaggregated.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from inferscale import run_simulation
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def base_config():
|
| 5 |
+
return {
|
| 6 |
+
"model": "Qwen2.5-3B",
|
| 7 |
+
"accelerator": "L4",
|
| 8 |
+
"prefill_accelerator": "L4",
|
| 9 |
+
"decode_accelerator": "L4",
|
| 10 |
+
"quantization": "int8",
|
| 11 |
+
"scheduler": "continuous_slo",
|
| 12 |
+
"topology": "disaggregated_pd",
|
| 13 |
+
"duration_s": 5,
|
| 14 |
+
"request_rate_rps": 1.5,
|
| 15 |
+
"prompt_tokens_mean": 128,
|
| 16 |
+
"prompt_tokens_cv": 0.1,
|
| 17 |
+
"output_tokens_mean": 8,
|
| 18 |
+
"output_tokens_cv": 0.1,
|
| 19 |
+
"seed": 4,
|
| 20 |
+
"slo_ttft_ms": 2000,
|
| 21 |
+
"slo_e2e_ms": 10000,
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def test_disaggregated_pipeline_completes_and_reports_transfer():
|
| 26 |
+
result = run_simulation(base_config())
|
| 27 |
+
assert result["provenance"]["topology"] == "disaggregated_pd"
|
| 28 |
+
assert result["summary"]["requests_completed"] > 0
|
| 29 |
+
assert result["resource"]["kv_transfer_gb"] > 0
|
| 30 |
+
assert result["resource"]["accelerator_instances"] == 2
|
| 31 |
+
assert result["resource"]["p95_transfer_ms"] > 0
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def test_more_decode_workers_are_modelled():
|
| 35 |
+
cfg = base_config()
|
| 36 |
+
cfg["decode_workers"] = 2
|
| 37 |
+
result = run_simulation(cfg)
|
| 38 |
+
assert result["resource"]["decode_workers"] == 2
|
| 39 |
+
assert result["resource"]["accelerator_instances"] == 3
|
tests/test_prefix_cache.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from inferscale import run_simulation
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def cfg(cache: bool):
|
| 5 |
+
return {
|
| 6 |
+
"model": "Qwen2.5-3B",
|
| 7 |
+
"accelerator": "L4",
|
| 8 |
+
"quantization": "int8",
|
| 9 |
+
"scheduler": "continuous_fcfs",
|
| 10 |
+
"duration_s": 6,
|
| 11 |
+
"request_rate_rps": 2,
|
| 12 |
+
"prompt_tokens_mean": 512,
|
| 13 |
+
"prompt_tokens_cv": 0,
|
| 14 |
+
"output_tokens_mean": 8,
|
| 15 |
+
"output_tokens_cv": 0,
|
| 16 |
+
"seed": 2,
|
| 17 |
+
"prefix_cache_enabled": cache,
|
| 18 |
+
"shared_prefix_tokens": 256,
|
| 19 |
+
"prefix_reuse_fraction": 1.0,
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_prefix_cache_reduces_prefill_work_and_is_reported():
|
| 24 |
+
uncached = run_simulation(cfg(False))
|
| 25 |
+
cached = run_simulation(cfg(True))
|
| 26 |
+
assert cached["resource"]["prefix_cache_hit_rate"] == 1.0
|
| 27 |
+
assert cached["resource"]["prefill_tokens_saved"] > 0
|
| 28 |
+
assert cached["latency"]["ttft_ms"]["p50"] < uncached["latency"]["ttft_ms"]["p50"]
|
worker.mjs
CHANGED
|
@@ -4,6 +4,7 @@ const MODULES = [
|
|
| 4 |
"__init__.py",
|
| 5 |
"api.py",
|
| 6 |
"diagnostics.py",
|
|
|
|
| 7 |
"kv_cache.py",
|
| 8 |
"latency.py",
|
| 9 |
"metrics.py",
|
|
|
|
| 4 |
"__init__.py",
|
| 5 |
"api.py",
|
| 6 |
"diagnostics.py",
|
| 7 |
+
"disaggregated.py",
|
| 8 |
"kv_cache.py",
|
| 9 |
"latency.py",
|
| 10 |
"metrics.py",
|