cn0303 commited on
Commit
0935028
·
verified ·
1 Parent(s): ee8ca43

Speed predictions now come from a model trained on 6.6k real measurements (17.5% median error on unseen hardware), physics formula beyond its measured range

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ model/speed_model.skops filter=lfs diff=lfs merge=lfs -text
engine/speed.py CHANGED
@@ -116,7 +116,11 @@ def _trained_model():
116
  return None
117
  try:
118
  from skops.io import load as skops_load
119
- model = skops_load(_MODEL_PATH, trusted=None)
 
 
 
 
120
  print(f"[FitCheck] speed predictor loaded from {_MODEL_PATH.name}", flush=True)
121
  return model
122
  except Exception as e: # noqa: BLE001
@@ -129,6 +133,37 @@ def _trained_model():
129
  return None
130
 
131
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  # --------------------------------------------------------------------------
133
  # Prediction
134
  # --------------------------------------------------------------------------
@@ -161,7 +196,7 @@ def predict_decode_tps(
161
  eff_bw = 1.0 / ((1.0 - f) / bandwidth_gbs + f / _RAM_BW_GBS)
162
 
163
  model = _trained_model()
164
- if model is not None:
165
  try:
166
  import numpy as np
167
  x = np.array([[eff_bw, bytes_gb, weights_gb, kv_gb,
@@ -178,12 +213,15 @@ def predict_decode_tps(
178
  pass
179
 
180
  base = eff_bw / bytes_gb
 
 
 
 
 
181
  return {"tps": round(base * _EFF_MID, 1),
182
  "lo": round(base * _EFF_LO, 1), "hi": round(base * _EFF_HI, 1),
183
  "bytes_gb": round(bytes_gb, 2), "eff_bw": round(eff_bw, 1),
184
- "method": "roofline",
185
- "note": ("analytical estimate: decode speed is memory-bandwidth-bound "
186
- "(bandwidth divided by bytes read per token)")}
187
 
188
 
189
  def feel_text(pred: dict) -> str:
 
116
  return None
117
  try:
118
  from skops.io import load as skops_load
119
+ # skops only loads explicitly-trusted types — exactly these two, which
120
+ # scripts/train_speed_model.py produces. Anything else is refused.
121
+ model = skops_load(_MODEL_PATH, trusted=[
122
+ "xgboost.core.Booster", "xgboost.sklearn.XGBRegressor",
123
+ ])
124
  print(f"[FitCheck] speed predictor loaded from {_MODEL_PATH.name}", flush=True)
125
  return model
126
  except Exception as e: # noqa: BLE001
 
133
  return None
134
 
135
 
136
+ _METRICS_PATH = _ROOT / "model" / "metrics.json"
137
+
138
+
139
+ @lru_cache(maxsize=1)
140
+ def _envelope() -> dict:
141
+ """The region of feature space the training data actually covered.
142
+
143
+ Decision trees cannot extrapolate: outside what they saw, they clamp to
144
+ the nearest seen value and quietly give wrong answers (e.g. a 32B model
145
+ gets a 14B's speed). The roofline DOES extrapolate — it's physics. So the
146
+ trained model only answers inside its measured envelope; outside it, the
147
+ labelled analytical estimate takes over. Bounds come from metrics.json
148
+ when the training script recorded them, else conservative defaults
149
+ matching the LocalScore grid (<=14B Q4 models, consumer hardware).
150
+ """
151
+ env = {"bytes_gb": (0.8, 10.0), "eff_bw": (30.0, 1900.0)}
152
+ try:
153
+ rec = json.loads(_METRICS_PATH.read_text(encoding="utf-8")).get("envelope")
154
+ if rec:
155
+ env.update({k: tuple(v) for k, v in rec.items()})
156
+ except OSError:
157
+ pass
158
+ return env
159
+
160
+
161
+ def _in_envelope(eff_bw: float, bytes_gb: float) -> bool:
162
+ env = _envelope()
163
+ return (env["bytes_gb"][0] <= bytes_gb <= env["bytes_gb"][1]
164
+ and env["eff_bw"][0] <= eff_bw <= env["eff_bw"][1])
165
+
166
+
167
  # --------------------------------------------------------------------------
168
  # Prediction
169
  # --------------------------------------------------------------------------
 
196
  eff_bw = 1.0 / ((1.0 - f) / bandwidth_gbs + f / _RAM_BW_GBS)
197
 
198
  model = _trained_model()
199
+ if model is not None and _in_envelope(eff_bw, bytes_gb):
200
  try:
201
  import numpy as np
202
  x = np.array([[eff_bw, bytes_gb, weights_gb, kv_gb,
 
213
  pass
214
 
215
  base = eff_bw / bytes_gb
216
+ note = ("analytical estimate: decode speed is memory-bandwidth-bound "
217
+ "(bandwidth divided by bytes read per token)")
218
+ if model is not None:
219
+ note += (" — this configuration is outside the measured data's range, "
220
+ "so the physics formula answers instead of the trained model")
221
  return {"tps": round(base * _EFF_MID, 1),
222
  "lo": round(base * _EFF_LO, 1), "hi": round(base * _EFF_HI, 1),
223
  "bytes_gb": round(bytes_gb, 2), "eff_bw": round(eff_bw, 1),
224
+ "method": "roofline", "note": note}
 
 
225
 
226
 
227
  def feel_text(pred: dict) -> str:
model/README.md ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ library_name: skops
4
+ tags: [tabular-regression, xgboost, llm-inference, performance-prediction]
5
+ ---
6
+
7
+ # FitCheck speed predictor
8
+
9
+ Predicts local-LLM **decode tokens/sec** from hardware + model features.
10
+ Part of [FitCheck](https://huggingface.co/spaces/build-small-hackathon/FitCheck),
11
+ the honest "what AI can your computer run" advisor.
12
+
13
+ ## Method
14
+
15
+ Gradient-boosted regression (XGBoost) following the methodology of
16
+ **LLM-Pilot** (IBM, SC'24): [arXiv:2410.02425](https://arxiv.org/abs/2410.02425) —
17
+ performance prediction for LLM inference on unseen hardware, validated
18
+ **leave-one-accelerator-out** so the error below is measured on hardware the
19
+ model never saw in training.
20
+
21
+ Features: effective memory bandwidth, bytes read per token (weights + KV),
22
+ weights size, KV size, MoE active fraction, offload fraction, and the
23
+ analytical roofline prior (bandwidth / bytes). Decode is memory-bandwidth-bound;
24
+ the model learns the residual between the roofline ideal and reality.
25
+
26
+ ## Training data
27
+
28
+ 6,633 real measurements across 595 distinct
29
+ accelerators (consumer CPUs, Apple Silicon, NVIDIA/AMD GPUs), from the
30
+ [LocalScore](https://www.localscore.ai) community benchmark (Mozilla Builders /
31
+ cjpais — thank you; data attributed, not owned, takedown requests honoured).
32
+ Trained 2026-06-10.
33
+
34
+ ## Honest holdout results (leave-one-accelerator-out)
35
+
36
+ | metric | roofline baseline | this model |
37
+ |---|---|---|
38
+ | median APE (bandwidth-known hardware) | 28.1% | 17.5% |
39
+ | median abs error (tok/s) | 11.63 | 9.55 |
40
+ | all hardware incl. CPUs (no baseline possible) | — | 23.6% median APE |
41
+
42
+ **Shipping rule:** this model is only deployed because it beat the analytical
43
+ baseline on held-out hardware. If a retrain ever fails that gate, FitCheck
44
+ falls back to the labelled roofline estimate.
45
+
46
+ ## Limits (read this)
47
+
48
+ - Trained on **dense LLMs running fully on-device** (LocalScore's fixed grid:
49
+ 1B / 8B / 14B at Q4_K_M, varied context). The model axis generalises through
50
+ the bytes-per-token feature, not data diversity.
51
+ - MoE and GPU->RAM offload are corrected analytically upstream, then fed
52
+ through — those corrections are engineering estimates, labelled as such.
53
+ - Does NOT cover vision/diffusion models (compute-bound, different physics).
model/metrics.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "rows": 6633,
3
+ "accelerators": 595,
4
+ "model_mape_pct": 23.6,
5
+ "model_mae_tps": 7.63,
6
+ "bw_known_rows": 3732,
7
+ "baseline_mape_pct": 28.1,
8
+ "baseline_mae_tps": 11.63,
9
+ "model_mape_on_bw_known_pct": 17.5,
10
+ "model_mae_on_bw_known_tps": 9.55
11
+ }
model/speed_model.skops ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70249e8241e24de5202af7ba82e911b9dbfaed06c19be920d6b6fd2873bef58b
3
+ size 1658817