Speed predictions now come from a model trained on 6.6k real measurements (17.5% median error on unseen hardware), physics formula beyond its measured range
Browse files- .gitattributes +1 -0
- engine/speed.py +43 -5
- model/README.md +53 -0
- model/metrics.json +11 -0
- model/speed_model.skops +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
model/speed_model.skops filter=lfs diff=lfs merge=lfs -text
|
engine/speed.py
CHANGED
|
@@ -116,7 +116,11 @@ def _trained_model():
|
|
| 116 |
return None
|
| 117 |
try:
|
| 118 |
from skops.io import load as skops_load
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
print(f"[FitCheck] speed predictor loaded from {_MODEL_PATH.name}", flush=True)
|
| 121 |
return model
|
| 122 |
except Exception as e: # noqa: BLE001
|
|
@@ -129,6 +133,37 @@ def _trained_model():
|
|
| 129 |
return None
|
| 130 |
|
| 131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
# --------------------------------------------------------------------------
|
| 133 |
# Prediction
|
| 134 |
# --------------------------------------------------------------------------
|
|
@@ -161,7 +196,7 @@ def predict_decode_tps(
|
|
| 161 |
eff_bw = 1.0 / ((1.0 - f) / bandwidth_gbs + f / _RAM_BW_GBS)
|
| 162 |
|
| 163 |
model = _trained_model()
|
| 164 |
-
if model is not None:
|
| 165 |
try:
|
| 166 |
import numpy as np
|
| 167 |
x = np.array([[eff_bw, bytes_gb, weights_gb, kv_gb,
|
|
@@ -178,12 +213,15 @@ def predict_decode_tps(
|
|
| 178 |
pass
|
| 179 |
|
| 180 |
base = eff_bw / bytes_gb
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
return {"tps": round(base * _EFF_MID, 1),
|
| 182 |
"lo": round(base * _EFF_LO, 1), "hi": round(base * _EFF_HI, 1),
|
| 183 |
"bytes_gb": round(bytes_gb, 2), "eff_bw": round(eff_bw, 1),
|
| 184 |
-
"method": "roofline",
|
| 185 |
-
"note": ("analytical estimate: decode speed is memory-bandwidth-bound "
|
| 186 |
-
"(bandwidth divided by bytes read per token)")}
|
| 187 |
|
| 188 |
|
| 189 |
def feel_text(pred: dict) -> str:
|
|
|
|
| 116 |
return None
|
| 117 |
try:
|
| 118 |
from skops.io import load as skops_load
|
| 119 |
+
# skops only loads explicitly-trusted types — exactly these two, which
|
| 120 |
+
# scripts/train_speed_model.py produces. Anything else is refused.
|
| 121 |
+
model = skops_load(_MODEL_PATH, trusted=[
|
| 122 |
+
"xgboost.core.Booster", "xgboost.sklearn.XGBRegressor",
|
| 123 |
+
])
|
| 124 |
print(f"[FitCheck] speed predictor loaded from {_MODEL_PATH.name}", flush=True)
|
| 125 |
return model
|
| 126 |
except Exception as e: # noqa: BLE001
|
|
|
|
| 133 |
return None
|
| 134 |
|
| 135 |
|
| 136 |
+
_METRICS_PATH = _ROOT / "model" / "metrics.json"
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@lru_cache(maxsize=1)
|
| 140 |
+
def _envelope() -> dict:
|
| 141 |
+
"""The region of feature space the training data actually covered.
|
| 142 |
+
|
| 143 |
+
Decision trees cannot extrapolate: outside what they saw, they clamp to
|
| 144 |
+
the nearest seen value and quietly give wrong answers (e.g. a 32B model
|
| 145 |
+
gets a 14B's speed). The roofline DOES extrapolate — it's physics. So the
|
| 146 |
+
trained model only answers inside its measured envelope; outside it, the
|
| 147 |
+
labelled analytical estimate takes over. Bounds come from metrics.json
|
| 148 |
+
when the training script recorded them, else conservative defaults
|
| 149 |
+
matching the LocalScore grid (<=14B Q4 models, consumer hardware).
|
| 150 |
+
"""
|
| 151 |
+
env = {"bytes_gb": (0.8, 10.0), "eff_bw": (30.0, 1900.0)}
|
| 152 |
+
try:
|
| 153 |
+
rec = json.loads(_METRICS_PATH.read_text(encoding="utf-8")).get("envelope")
|
| 154 |
+
if rec:
|
| 155 |
+
env.update({k: tuple(v) for k, v in rec.items()})
|
| 156 |
+
except OSError:
|
| 157 |
+
pass
|
| 158 |
+
return env
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def _in_envelope(eff_bw: float, bytes_gb: float) -> bool:
|
| 162 |
+
env = _envelope()
|
| 163 |
+
return (env["bytes_gb"][0] <= bytes_gb <= env["bytes_gb"][1]
|
| 164 |
+
and env["eff_bw"][0] <= eff_bw <= env["eff_bw"][1])
|
| 165 |
+
|
| 166 |
+
|
| 167 |
# --------------------------------------------------------------------------
|
| 168 |
# Prediction
|
| 169 |
# --------------------------------------------------------------------------
|
|
|
|
| 196 |
eff_bw = 1.0 / ((1.0 - f) / bandwidth_gbs + f / _RAM_BW_GBS)
|
| 197 |
|
| 198 |
model = _trained_model()
|
| 199 |
+
if model is not None and _in_envelope(eff_bw, bytes_gb):
|
| 200 |
try:
|
| 201 |
import numpy as np
|
| 202 |
x = np.array([[eff_bw, bytes_gb, weights_gb, kv_gb,
|
|
|
|
| 213 |
pass
|
| 214 |
|
| 215 |
base = eff_bw / bytes_gb
|
| 216 |
+
note = ("analytical estimate: decode speed is memory-bandwidth-bound "
|
| 217 |
+
"(bandwidth divided by bytes read per token)")
|
| 218 |
+
if model is not None:
|
| 219 |
+
note += (" — this configuration is outside the measured data's range, "
|
| 220 |
+
"so the physics formula answers instead of the trained model")
|
| 221 |
return {"tps": round(base * _EFF_MID, 1),
|
| 222 |
"lo": round(base * _EFF_LO, 1), "hi": round(base * _EFF_HI, 1),
|
| 223 |
"bytes_gb": round(bytes_gb, 2), "eff_bw": round(eff_bw, 1),
|
| 224 |
+
"method": "roofline", "note": note}
|
|
|
|
|
|
|
| 225 |
|
| 226 |
|
| 227 |
def feel_text(pred: dict) -> str:
|
model/README.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
library_name: skops
|
| 4 |
+
tags: [tabular-regression, xgboost, llm-inference, performance-prediction]
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
# FitCheck speed predictor
|
| 8 |
+
|
| 9 |
+
Predicts local-LLM **decode tokens/sec** from hardware + model features.
|
| 10 |
+
Part of [FitCheck](https://huggingface.co/spaces/build-small-hackathon/FitCheck),
|
| 11 |
+
the honest "what AI can your computer run" advisor.
|
| 12 |
+
|
| 13 |
+
## Method
|
| 14 |
+
|
| 15 |
+
Gradient-boosted regression (XGBoost) following the methodology of
|
| 16 |
+
**LLM-Pilot** (IBM, SC'24): [arXiv:2410.02425](https://arxiv.org/abs/2410.02425) —
|
| 17 |
+
performance prediction for LLM inference on unseen hardware, validated
|
| 18 |
+
**leave-one-accelerator-out** so the error below is measured on hardware the
|
| 19 |
+
model never saw in training.
|
| 20 |
+
|
| 21 |
+
Features: effective memory bandwidth, bytes read per token (weights + KV),
|
| 22 |
+
weights size, KV size, MoE active fraction, offload fraction, and the
|
| 23 |
+
analytical roofline prior (bandwidth / bytes). Decode is memory-bandwidth-bound;
|
| 24 |
+
the model learns the residual between the roofline ideal and reality.
|
| 25 |
+
|
| 26 |
+
## Training data
|
| 27 |
+
|
| 28 |
+
6,633 real measurements across 595 distinct
|
| 29 |
+
accelerators (consumer CPUs, Apple Silicon, NVIDIA/AMD GPUs), from the
|
| 30 |
+
[LocalScore](https://www.localscore.ai) community benchmark (Mozilla Builders /
|
| 31 |
+
cjpais — thank you; data attributed, not owned, takedown requests honoured).
|
| 32 |
+
Trained 2026-06-10.
|
| 33 |
+
|
| 34 |
+
## Honest holdout results (leave-one-accelerator-out)
|
| 35 |
+
|
| 36 |
+
| metric | roofline baseline | this model |
|
| 37 |
+
|---|---|---|
|
| 38 |
+
| median APE (bandwidth-known hardware) | 28.1% | 17.5% |
|
| 39 |
+
| median abs error (tok/s) | 11.63 | 9.55 |
|
| 40 |
+
| all hardware incl. CPUs (no baseline possible) | — | 23.6% median APE |
|
| 41 |
+
|
| 42 |
+
**Shipping rule:** this model is only deployed because it beat the analytical
|
| 43 |
+
baseline on held-out hardware. If a retrain ever fails that gate, FitCheck
|
| 44 |
+
falls back to the labelled roofline estimate.
|
| 45 |
+
|
| 46 |
+
## Limits (read this)
|
| 47 |
+
|
| 48 |
+
- Trained on **dense LLMs running fully on-device** (LocalScore's fixed grid:
|
| 49 |
+
1B / 8B / 14B at Q4_K_M, varied context). The model axis generalises through
|
| 50 |
+
the bytes-per-token feature, not data diversity.
|
| 51 |
+
- MoE and GPU->RAM offload are corrected analytically upstream, then fed
|
| 52 |
+
through — those corrections are engineering estimates, labelled as such.
|
| 53 |
+
- Does NOT cover vision/diffusion models (compute-bound, different physics).
|
model/metrics.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"rows": 6633,
|
| 3 |
+
"accelerators": 595,
|
| 4 |
+
"model_mape_pct": 23.6,
|
| 5 |
+
"model_mae_tps": 7.63,
|
| 6 |
+
"bw_known_rows": 3732,
|
| 7 |
+
"baseline_mape_pct": 28.1,
|
| 8 |
+
"baseline_mae_tps": 11.63,
|
| 9 |
+
"model_mape_on_bw_known_pct": 17.5,
|
| 10 |
+
"model_mae_on_bw_known_tps": 9.55
|
| 11 |
+
}
|
model/speed_model.skops
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:70249e8241e24de5202af7ba82e911b9dbfaed06c19be920d6b6fd2873bef58b
|
| 3 |
+
size 1658817
|