Spaces:
Sleeping
Sleeping
DSSD demo from DSSD_demo 4a3ca5b (dssd 73b3f27)
Browse files- .gitignore +41 -0
- README.md +48 -8
- app.py +194 -0
- demo/__init__.py +0 -0
- demo/backend.py +91 -0
- demo/examples.py +33 -0
- demo/render.py +102 -0
- demo/tiny.py +52 -0
- dssd/VENDORED.txt +1 -0
- dssd/__init__.py +11 -0
- dssd/api.py +287 -0
- dssd/calibration/__init__.py +1 -0
- dssd/calibration/calib.py +283 -0
- dssd/calibration/online.py +791 -0
- dssd/decode.py +495 -0
- dssd/fast.py +1327 -0
- dssd/heads.py +143 -0
- dssd/hub.py +111 -0
- dssd/kernels.py +79 -0
- dssd/layered.py +212 -0
- dssd/pld.py +39 -0
- dssd/stops.py +33 -0
- requirements.txt +6 -0
- scripts/vendor_dssd.sh +12 -0
.gitignore
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Python
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*.class
|
| 5 |
+
*.so
|
| 6 |
+
.Python
|
| 7 |
+
env/
|
| 8 |
+
build/
|
| 9 |
+
develop-eggs/
|
| 10 |
+
dist/
|
| 11 |
+
downloads/
|
| 12 |
+
eggs/
|
| 13 |
+
.eggs/
|
| 14 |
+
lib/
|
| 15 |
+
lib64/
|
| 16 |
+
parts/
|
| 17 |
+
sdist/
|
| 18 |
+
var/
|
| 19 |
+
wheels/
|
| 20 |
+
*.egg-info/
|
| 21 |
+
.installed.cfg
|
| 22 |
+
*.egg
|
| 23 |
+
|
| 24 |
+
# Virtual Environment
|
| 25 |
+
venv/
|
| 26 |
+
ENV/
|
| 27 |
+
|
| 28 |
+
# IDEs
|
| 29 |
+
.vscode/
|
| 30 |
+
.idea/
|
| 31 |
+
|
| 32 |
+
# Models and large files
|
| 33 |
+
*.pt
|
| 34 |
+
*.bin
|
| 35 |
+
*.pth
|
| 36 |
+
checkpoints/
|
| 37 |
+
|
| 38 |
+
# OS
|
| 39 |
+
.DS_Store
|
| 40 |
+
|
| 41 |
+
.venv/
|
README.md
CHANGED
|
@@ -1,13 +1,53 @@
|
|
| 1 |
---
|
| 2 |
-
title: DSSD
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version:
|
| 9 |
app_file: app.py
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: DSSD exact self-speculative decoding
|
| 3 |
+
emoji: ⚡
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: pink
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.29.1
|
| 8 |
+
python_version: "3.12"
|
| 9 |
app_file: app.py
|
| 10 |
+
license: apache-2.0
|
| 11 |
+
short_description: Exact self-speculative decoding with exit heads + copying
|
| 12 |
+
models:
|
| 13 |
+
- FlorianVal/DSSD-Qwen3-8B-Base
|
| 14 |
+
- FlorianVal/Llama-3.1-8B-DSSD
|
| 15 |
+
- Qwen/Qwen3-8B-Base
|
| 16 |
+
- meta-llama/Llama-3.1-8B
|
| 17 |
---
|
| 18 |
|
| 19 |
+
# DSSD demo
|
| 20 |
+
|
| 21 |
+
Exact self-speculative decoding: a small exit head on layer 4 of the frozen model, plus n-gram copies from the
|
| 22 |
+
context, propose a tree of draft tokens. The full model verifies it in one forward pass. Outputs are identical to
|
| 23 |
+
greedy decoding.
|
| 24 |
+
|
| 25 |
+
The demo shows:
|
| 26 |
+
- each token coloured by its source: copy, exit head, head top-4 candidate, or full model;
|
| 27 |
+
- the same prompt decoded autoregressively for comparison;
|
| 28 |
+
- the measured speedup, tokens per verification, and a check that both outputs are identical;
|
| 29 |
+
- a side-by-side replay of both generations at the measured speed.
|
| 30 |
+
|
| 31 |
+
Models: Qwen3-8B-Base and Llama-3.1-8B (base models, plain-text completion). The heads are in
|
| 32 |
+
[FlorianVal/DSSD-Qwen3-8B-Base](https://huggingface.co/FlorianVal/DSSD-Qwen3-8B-Base) and
|
| 33 |
+
[FlorianVal/Llama-3.1-8B-DSSD](https://huggingface.co/FlorianVal/Llama-3.1-8B-DSSD). The Llama heads are built with Llama.
|
| 34 |
+
|
| 35 |
+
## Configuration
|
| 36 |
+
|
| 37 |
+
| Variable | Default | Effect |
|
| 38 |
+
|---|---|---|
|
| 39 |
+
| `HF_TOKEN` (secret) | — | needed for the gated Llama base model and for private head repositories |
|
| 40 |
+
| `DSSD_DEMO_MODELS` | both | comma-separated subset of `Qwen3-8B-Base,Llama-3.1-8B` |
|
| 41 |
+
| `DSSD_GRAPHS` | off on ZeroGPU, on otherwise | `1` / `0` forces CUDA graphs |
|
| 42 |
+
| `DSSD_DEMO_BACKEND` | `hub` | `tiny`: random tiny model on CPU, for local UI work |
|
| 43 |
+
|
| 44 |
+
## Local development
|
| 45 |
+
|
| 46 |
+
```bash
|
| 47 |
+
uv venv .venv && uv pip install --python .venv/bin/python -r requirements.txt pytest
|
| 48 |
+
DSSD_DEMO_BACKEND=tiny .venv/bin/python app.py # http://localhost:7860, no GPU needed
|
| 49 |
+
.venv/bin/python -m pytest -q tests
|
| 50 |
+
scripts/vendor_dssd.sh ../BranchyLLM-pub # refresh the vendored dssd package
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
The `dssd/` directory is a copy of the inference package from BranchyLLM (see `dssd/VENDORED.txt`); do not edit it here.
|
app.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DSSD demo: exact self-speculative decoding with pruned exit heads and context copying.
|
| 2 |
+
|
| 3 |
+
Local UI work without a GPU: DSSD_DEMO_BACKEND=tiny python app.py
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import gradio as gr
|
| 7 |
+
|
| 8 |
+
from demo.backend import MAX_NEW_TOKENS, MODELS, generate_updates, load_models
|
| 9 |
+
from demo.examples import EXAMPLES
|
| 10 |
+
from demo.render import legend_html
|
| 11 |
+
|
| 12 |
+
def gpu_seconds(model, domain_label, prompt, max_new, compare):
|
| 13 |
+
# ZeroGPU refuses a call whose requested duration exceeds the visitor's remaining daily quota
|
| 14 |
+
# (5 min on free accounts), so ask for what the request needs: engine setup + both decodes.
|
| 15 |
+
return int(30 + int(max_new) * (0.25 if compare else 0.12))
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
try: # ZeroGPU on Hugging Face Spaces
|
| 19 |
+
import spaces
|
| 20 |
+
GPU = spaces.GPU(duration=gpu_seconds)
|
| 21 |
+
except ImportError:
|
| 22 |
+
def GPU(f):
|
| 23 |
+
return f
|
| 24 |
+
|
| 25 |
+
ENGINES = load_models()
|
| 26 |
+
DOMAINS = {"Web text": "web", "Math (few-shot)": "gsm8k", "Python code": "code"}
|
| 27 |
+
|
| 28 |
+
HEADER = """
|
| 29 |
+
<div class="hero">
|
| 30 |
+
<h1>DSSD <span class="hero-sub">exact self-speculative decoding</span></h1>
|
| 31 |
+
<p>A tiny exit head on layer 4 of the frozen model, plus tokens copied from the context, propose a
|
| 32 |
+
small tree of draft tokens. The full model checks the whole tree in one forward pass and keeps the
|
| 33 |
+
longest correct branch. The text is <b>exactly</b> what greedy decoding would write, only faster.</p>
|
| 34 |
+
<div class="badges"><span>✓ identical to greedy decoding</span><span>no draft model</span>
|
| 35 |
+
<span>≈ 280 MB of heads</span></div>
|
| 36 |
+
</div>
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
HOW = """
|
| 40 |
+
<div class="how">
|
| 41 |
+
<div class="how-step"><b>1 · Draft</b><br>Copy n-grams found earlier in the context<br>+ chain of the
|
| 42 |
+
layer-4 head<br>+ its top-4 alternatives at the root</div>
|
| 43 |
+
<div class="how-arrow">→</div>
|
| 44 |
+
<div class="how-step"><b>2 · Verify</b><br>One forward pass of the full model over the whole tree
|
| 45 |
+
(16 rows, row-invariant kernels)</div>
|
| 46 |
+
<div class="how-arrow">→</div>
|
| 47 |
+
<div class="how-step"><b>3 · Commit</b><br>Longest branch that matches the model's own greedy choice,
|
| 48 |
+
plus one token from the model</div>
|
| 49 |
+
</div>
|
| 50 |
+
<p class="how-note">The exit heads predict over a small, layer-specific vocabulary (≈ 3k tokens at layer 4)
|
| 51 |
+
plus a <i>reject</i> class, about 50× cheaper than a full projection. Speedups depend on the GPU and on
|
| 52 |
+
whether CUDA graphs are used. The paper's measurements (AMD MI210, greedy, batch 1) are 2.50× / 1.72× / 1.46× on
|
| 53 |
+
web text, GSM8K and HumanEval for Qwen3-8B-Base, and 2.51× / 1.80× / 1.57× for Llama-3.1-8B.</p>
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
CSS = """
|
| 57 |
+
.hero h1 { margin: 0 0 .3rem 0; font-size: 1.9rem; }
|
| 58 |
+
.hero-sub { font-weight: 400; opacity: .65; font-size: 1.1rem; margin-left: .4rem; }
|
| 59 |
+
.hero p { margin: .2rem 0 .6rem 0; max-width: 60rem; }
|
| 60 |
+
.badges span { display: inline-block; padding: .15rem .6rem; margin: 0 .4rem .3rem 0; border-radius: 999px;
|
| 61 |
+
font-size: .82rem; background: var(--block-background-fill); border: 1px solid var(--border-color-primary); }
|
| 62 |
+
:root { --tok-copy: #cfe4fb; --tok-head: #ffe1b0; --tok-star: #f3d1ec; --tok-draft: #d9f2d0; }
|
| 63 |
+
.dark { --tok-copy: #1f4b78; --tok-head: #7a5413; --tok-star: #6b2f60; --tok-draft: #2f5a2a; }
|
| 64 |
+
.tok { border-radius: 3px; padding: 0 1px; }
|
| 65 |
+
.tok-copy { background: var(--tok-copy); }
|
| 66 |
+
.tok-head { background: var(--tok-head); }
|
| 67 |
+
.tok-star { background: var(--tok-star); box-shadow: inset 0 -2px 0 #b3479b; }
|
| 68 |
+
.tok-draft { background: var(--tok-draft); }
|
| 69 |
+
.tok-model { background: transparent; border: 1px dashed var(--border-color-primary); }
|
| 70 |
+
.legend { display: flex; flex-wrap: wrap; gap: .9rem; font-size: .88rem; margin: .2rem 0; }
|
| 71 |
+
.legend-item { display: inline-flex; align-items: center; gap: .35rem; }
|
| 72 |
+
.swatch { font-size: .8rem; padding: 0 .3rem; }
|
| 73 |
+
.dssd-text { white-space: pre-wrap; overflow-wrap: anywhere; font-family: var(--font-mono); font-size: .9rem; line-height: 1.55;
|
| 74 |
+
min-height: 14rem; max-height: 30rem; overflow-y: auto; padding: .8rem; border-radius: 8px;
|
| 75 |
+
background: var(--block-background-fill); border: 1px solid var(--border-color-primary); }
|
| 76 |
+
.cursor { display: inline-block; width: .5em; height: 1em; background: var(--body-text-color);
|
| 77 |
+
vertical-align: text-bottom; animation: blink 1s steps(1) infinite; opacity: .6; }
|
| 78 |
+
@keyframes blink { 50% { opacity: 0; } }
|
| 79 |
+
.panel-title { font-weight: 600; margin-bottom: .2rem; }
|
| 80 |
+
.stats { display: flex; flex-wrap: wrap; gap: .6rem; margin: .4rem 0 .6rem 0; }
|
| 81 |
+
.stat { flex: 1 1 8rem; padding: .6rem .8rem; border-radius: 8px; background: var(--block-background-fill);
|
| 82 |
+
border: 1px solid var(--border-color-primary); }
|
| 83 |
+
.stat-value { font-size: 1.35rem; font-weight: 700; }
|
| 84 |
+
.stat-label { font-size: .78rem; opacity: .7; }
|
| 85 |
+
.stat-main .stat-value { font-size: 1.9rem; color: var(--color-accent); }
|
| 86 |
+
.stat-ok .stat-value { color: #1a7f37; } .stat-bad .stat-value { color: #cf222e; }
|
| 87 |
+
.share-bar { display: flex; height: .7rem; border-radius: 999px; overflow: hidden;
|
| 88 |
+
border: 1px solid var(--border-color-primary); }
|
| 89 |
+
.share-bar .tok-model { background: var(--border-color-primary); border: 0; }
|
| 90 |
+
.share-labels { font-size: .8rem; opacity: .75; margin-top: .25rem; }
|
| 91 |
+
.progress { font-size: .9rem; opacity: .75; padding: .4rem 0; }
|
| 92 |
+
.replay-clock { font-size: .78rem; opacity: .7; margin-left: .4rem; }
|
| 93 |
+
.how { display: flex; flex-wrap: wrap; align-items: center; gap: .5rem; }
|
| 94 |
+
.how-step { flex: 1 1 12rem; padding: .7rem; border-radius: 8px; background: var(--block-background-fill);
|
| 95 |
+
border: 1px solid var(--border-color-primary); font-size: .88rem; }
|
| 96 |
+
.how-arrow { font-size: 1.4rem; opacity: .5; }
|
| 97 |
+
.how-note { font-size: .85rem; opacity: .8; margin-top: .6rem; }
|
| 98 |
+
@media (max-width: 700px) { .how-arrow { display: none; } }
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
# Client-side replay of both generations side by side, at the speed measured on the GPU.
|
| 102 |
+
HEAD = """
|
| 103 |
+
<script>
|
| 104 |
+
window.dssdReplay = function () {
|
| 105 |
+
const holder = document.querySelector('#replay-holder .replay-data');
|
| 106 |
+
if (!holder) return;
|
| 107 |
+
const data = JSON.parse(holder.dataset.replay);
|
| 108 |
+
const el = {dssd: document.querySelector('#dssd-out .dssd-text'), ar: document.querySelector('#ar-out .dssd-text')};
|
| 109 |
+
const esc = s => s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
| 110 |
+
for (const k in el) if (el[k]) el[k].innerHTML = '';
|
| 111 |
+
const idx = {dssd: 0, ar: 0}, start = performance.now();
|
| 112 |
+
function frame(now) {
|
| 113 |
+
const t = (now - start) / 1000; let more = false;
|
| 114 |
+
for (const k of ['dssd', 'ar']) {
|
| 115 |
+
const arr = data[k] || [], box = el[k]; if (!box) continue;
|
| 116 |
+
let html = '';
|
| 117 |
+
while (idx[k] < arr.length && arr[idx[k]][0] <= t) {
|
| 118 |
+
const [, txt, src] = arr[idx[k]++];
|
| 119 |
+
html += (k === 'dssd' && src !== 'model') ? `<span class="tok tok-${src}">${esc(txt)}</span>` : esc(txt);
|
| 120 |
+
}
|
| 121 |
+
if (html) box.insertAdjacentHTML('beforeend', html);
|
| 122 |
+
if (idx[k] < arr.length) more = true;
|
| 123 |
+
else if (arr.length && !box.dataset.done) {
|
| 124 |
+
box.dataset.done = 1;
|
| 125 |
+
box.insertAdjacentHTML('beforeend', `<span class="replay-clock">⏱ ${arr[arr.length - 1][0].toFixed(2)} s</span>`);
|
| 126 |
+
}
|
| 127 |
+
}
|
| 128 |
+
if (more) requestAnimationFrame(frame);
|
| 129 |
+
else for (const k in el) if (el[k]) delete el[k].dataset.done;
|
| 130 |
+
}
|
| 131 |
+
requestAnimationFrame(frame);
|
| 132 |
+
};
|
| 133 |
+
</script>
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@GPU
|
| 138 |
+
def run(model, domain_label, prompt, max_new, compare):
|
| 139 |
+
try:
|
| 140 |
+
for u in generate_updates(ENGINES[model], prompt, DOMAINS[domain_label], max_new, compare):
|
| 141 |
+
yield u.dssd, u.ar, u.stats, u.replay, gr.update(visible=u.done and compare)
|
| 142 |
+
except ValueError as e:
|
| 143 |
+
raise gr.Error(str(e))
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def example(domain_label, i):
|
| 147 |
+
ex = EXAMPLES[DOMAINS[domain_label]]
|
| 148 |
+
i = (i + 1) % len(ex)
|
| 149 |
+
return ex[i], i
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
with gr.Blocks(title="DSSD: exact self-speculative decoding") as demo:
|
| 153 |
+
gr.HTML(HEADER)
|
| 154 |
+
with gr.Row(equal_height=False):
|
| 155 |
+
with gr.Column(scale=2):
|
| 156 |
+
prompt = gr.Textbox(label="Prompt (plain-text completion: these are base models)", lines=6, max_lines=14,
|
| 157 |
+
value=EXAMPLES["web"][0])
|
| 158 |
+
with gr.Row():
|
| 159 |
+
ex_btn = gr.Button("Another example", size="sm", variant="secondary")
|
| 160 |
+
ex_idx = gr.State(0)
|
| 161 |
+
with gr.Column(scale=1):
|
| 162 |
+
model = gr.Radio(list(ENGINES), value=next(iter(ENGINES)), label="Model")
|
| 163 |
+
domain = gr.Radio(list(DOMAINS), value="Web text", label="Domain (calibrated settings and stop rule)")
|
| 164 |
+
max_new = gr.Slider(16, MAX_NEW_TOKENS, value=160, step=16, label="Max new tokens")
|
| 165 |
+
compare = gr.Checkbox(value=True, label="Also run plain autoregressive decoding and compare")
|
| 166 |
+
with gr.Row():
|
| 167 |
+
go = gr.Button("Generate", variant="primary")
|
| 168 |
+
stop = gr.Button("Stop", variant="stop")
|
| 169 |
+
gr.HTML(legend_html())
|
| 170 |
+
with gr.Row(equal_height=True):
|
| 171 |
+
with gr.Column():
|
| 172 |
+
gr.HTML('<div class="panel-title">DSSD</div>')
|
| 173 |
+
out_dssd = gr.HTML(elem_id="dssd-out")
|
| 174 |
+
with gr.Column():
|
| 175 |
+
gr.HTML('<div class="panel-title">Autoregressive (greedy)</div>')
|
| 176 |
+
out_ar = gr.HTML(elem_id="ar-out")
|
| 177 |
+
stats = gr.HTML()
|
| 178 |
+
replay_btn = gr.Button("▶ Replay both side by side at the measured speed", visible=False, size="sm")
|
| 179 |
+
replay = gr.HTML(elem_id="replay-holder", visible=True)
|
| 180 |
+
with gr.Accordion("How it works", open=False):
|
| 181 |
+
gr.HTML(HOW)
|
| 182 |
+
gr.Markdown(f"Heads: {', '.join(f'[{r}](https://huggingface.co/{r})' for r in MODELS.values())}. "
|
| 183 |
+
"Llama-3.1-8B heads: built with Llama.", elem_classes="footer")
|
| 184 |
+
|
| 185 |
+
gen = go.click(run, [model, domain, prompt, max_new, compare], [out_dssd, out_ar, stats, replay, replay_btn])
|
| 186 |
+
stop.click(None, cancels=[gen])
|
| 187 |
+
replay_btn.click(None, js="() => { window.dssdReplay(); }")
|
| 188 |
+
domain.change(lambda d: (EXAMPLES[DOMAINS[d]][0], 0), domain, [prompt, ex_idx])
|
| 189 |
+
ex_btn.click(example, [domain, ex_idx], [prompt, ex_idx])
|
| 190 |
+
|
| 191 |
+
demo.queue(default_concurrency_limit=1, max_size=16)
|
| 192 |
+
|
| 193 |
+
if __name__ == "__main__":
|
| 194 |
+
demo.launch(theme=gr.themes.Soft(primary_hue="indigo", neutral_hue="slate"), css=CSS, head=HEAD)
|
demo/__init__.py
ADDED
|
File without changes
|
demo/backend.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model loading and the generation loop of the demo (independent of Gradio)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from typing import Iterator
|
| 9 |
+
|
| 10 |
+
from .render import Detokenizer, replay_html, stats_html, tokens_html
|
| 11 |
+
|
| 12 |
+
# display name -> Hub repo with the heads (base model and tokenizer are pinned in its config)
|
| 13 |
+
MODELS = {
|
| 14 |
+
"Qwen3-8B-Base": "FlorianVal/DSSD-Qwen3-8B-Base",
|
| 15 |
+
"Llama-3.1-8B": "FlorianVal/Llama-3.1-8B-DSSD",
|
| 16 |
+
}
|
| 17 |
+
MAX_PROMPT_TOKENS = 1536
|
| 18 |
+
MAX_NEW_TOKENS = 256
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _graphs() -> bool | str:
|
| 22 |
+
# CUDA graphs are captured once per process; under ZeroGPU each call may run in a fresh GPU
|
| 23 |
+
# worker, so the eager engine is the safe default there. DSSD_GRAPHS=1 forces them on.
|
| 24 |
+
v = os.environ.get("DSSD_GRAPHS")
|
| 25 |
+
if v is not None:
|
| 26 |
+
return v == "1"
|
| 27 |
+
return False if os.environ.get("SPACES_ZERO_GPU") else "auto"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def load_models(backend: str | None = None) -> dict:
|
| 31 |
+
"""{display name: DSSD}. backend "hub" (default): the published heads and their base models
|
| 32 |
+
on CUDA; "tiny": a random tiny model on CPU for local UI work."""
|
| 33 |
+
backend = backend or os.environ.get("DSSD_DEMO_BACKEND", "hub")
|
| 34 |
+
if backend == "tiny":
|
| 35 |
+
from .tiny import tiny_engine
|
| 36 |
+
return {"Tiny random model (local test)": tiny_engine()}
|
| 37 |
+
import torch
|
| 38 |
+
from dssd import DSSD
|
| 39 |
+
names = [n.strip() for n in os.environ.get("DSSD_DEMO_MODELS", ",".join(MODELS)).split(",") if n.strip()]
|
| 40 |
+
device = "cuda" if (torch.cuda.is_available() or os.environ.get("SPACES_ZERO_GPU")) else "cpu"
|
| 41 |
+
token = os.environ.get("HF_TOKEN")
|
| 42 |
+
return {n: DSSD.from_pretrained(MODELS[n], device=device, graphs=_graphs(), token=token) for n in names}
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class Update:
|
| 47 |
+
dssd: str # HTML of the DSSD output (coloured by source)
|
| 48 |
+
ar: str # HTML of the autoregressive output ("" when not compared)
|
| 49 |
+
stats: str # HTML stat cards (final update) or a progress line
|
| 50 |
+
replay: str # hidden replay payload (final update)
|
| 51 |
+
done: bool = False
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def generate_updates(engine, prompt: str, domain: str, max_new: int, compare: bool,
|
| 55 |
+
min_interval: float = 0.05) -> Iterator[Update]:
|
| 56 |
+
"""Stream DSSD (then AR when compare) as HTML updates, at most one every min_interval seconds,
|
| 57 |
+
then a final update with stats and the replay timelines."""
|
| 58 |
+
if not prompt or not prompt.strip():
|
| 59 |
+
raise ValueError("Please enter a prompt.")
|
| 60 |
+
n = len(engine.tokenizer.encode(prompt, add_special_tokens=False))
|
| 61 |
+
if n > MAX_PROMPT_TOKENS:
|
| 62 |
+
raise ValueError(f"The prompt is too long ({n} tokens; at most {MAX_PROMPT_TOKENS}).")
|
| 63 |
+
max_new = max(1, min(int(max_new), MAX_NEW_TOKENS))
|
| 64 |
+
detok = {"dssd": Detokenizer(engine.tokenizer), "ar": Detokenizer(engine.tokenizer)}
|
| 65 |
+
pieces = {"dssd": [], "ar": []}
|
| 66 |
+
timeline = {"dssd": [], "ar": []}
|
| 67 |
+
last = 0.0
|
| 68 |
+
|
| 69 |
+
def view(done=False, stats="", replay=""):
|
| 70 |
+
return Update(tokens_html(pieces["dssd"], cursor=not done and not pieces["ar"]),
|
| 71 |
+
tokens_html(pieces["ar"], colored=False, cursor=not done and bool(pieces["ar"]))
|
| 72 |
+
if compare else "",
|
| 73 |
+
stats, replay, done)
|
| 74 |
+
|
| 75 |
+
for ev in engine.stream(prompt, max_new, domain=domain, ar=compare):
|
| 76 |
+
if ev.done:
|
| 77 |
+
for ph in ("dssd", "ar"):
|
| 78 |
+
new = detok[ph].flush()
|
| 79 |
+
pieces[ph] += new
|
| 80 |
+
if timeline[ph]:
|
| 81 |
+
timeline[ph] += [[timeline[ph][-1][0], t, s] for t, s in new]
|
| 82 |
+
yield view(True, stats_html(ev.result), replay_html(timeline))
|
| 83 |
+
return
|
| 84 |
+
new = detok[ev.phase].feed(ev.token_ids, ev.sources)
|
| 85 |
+
pieces[ev.phase] += new
|
| 86 |
+
timeline[ev.phase] += [[round(ev.t, 4), t, s] for t, s in new]
|
| 87 |
+
now = time.perf_counter()
|
| 88 |
+
if now - last >= min_interval:
|
| 89 |
+
last = now
|
| 90 |
+
phase = "Generating with DSSD…" if ev.phase == "dssd" else "Generating autoregressively for comparison…"
|
| 91 |
+
yield view(stats=f'<div class="progress">{phase}</div>')
|
demo/examples.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Example prompts per domain. The models are base (pre-trained) models: prompts are plain-text
|
| 2 |
+
completions in the format the heads were calibrated on (web text, GSM8K-style few-shot, Python)."""
|
| 3 |
+
|
| 4 |
+
WEB = [
|
| 5 |
+
"The history of the bicycle begins in the early nineteenth century. In 1817, the German inventor Karl Drais "
|
| 6 |
+
"built a two-wheeled \"running machine\" that riders pushed along with their feet. Over the following decades,",
|
| 7 |
+
"Photosynthesis is the process by which plants, algae and some bacteria convert light energy into chemical "
|
| 8 |
+
"energy. During photosynthesis, light energy is captured by chlorophyll and used to convert water and carbon "
|
| 9 |
+
"dioxide into",
|
| 10 |
+
]
|
| 11 |
+
|
| 12 |
+
_SHOTS = (
|
| 13 |
+
"Question: A baker makes 24 muffins in the morning and 18 in the afternoon. She sells 30 of them. "
|
| 14 |
+
"How many muffins are left?\n"
|
| 15 |
+
"Answer: She makes 24 + 18 = 42 muffins. After selling 30, she has 42 - 30 = 12 muffins left.\n#### 12\n\n"
|
| 16 |
+
"Question: Tom reads 15 pages a day. How many pages does he read in 2 weeks?\n"
|
| 17 |
+
"Answer: Two weeks have 2 * 7 = 14 days. He reads 15 * 14 = 210 pages.\n#### 210\n\n"
|
| 18 |
+
)
|
| 19 |
+
MATH = [
|
| 20 |
+
_SHOTS + "Question: A train travels at 60 miles per hour for 3 hours, then at 40 miles per hour for 2 hours. "
|
| 21 |
+
"How many miles does it travel in total?\nAnswer:",
|
| 22 |
+
_SHOTS + "Question: Lisa has 3 boxes with 12 pencils each. She gives 9 pencils to her brother and buys 5 more. "
|
| 23 |
+
"How many pencils does she have now?\nAnswer:",
|
| 24 |
+
]
|
| 25 |
+
|
| 26 |
+
CODE = [
|
| 27 |
+
'def count_vowels(text: str) -> int:\n """Return the number of vowels (a, e, i, o, u, case-insensitive) in text.\n\n'
|
| 28 |
+
' >>> count_vowels("Hello World")\n 3\n """\n',
|
| 29 |
+
'def merge_sorted(a: list[int], b: list[int]) -> list[int]:\n """Merge two sorted lists into one sorted '
|
| 30 |
+
'list.\n\n >>> merge_sorted([1, 4, 9], [2, 3, 10])\n [1, 2, 3, 4, 9, 10]\n """\n',
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
EXAMPLES = {"web": WEB, "gsm8k": MATH, "code": CODE}
|
demo/render.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""HTML rendering for the demo: coloured tokens by source, legend, stats, replay payload."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import html
|
| 6 |
+
import json
|
| 7 |
+
|
| 8 |
+
# token source -> (legend label, tooltip)
|
| 9 |
+
SOURCES = {
|
| 10 |
+
"copy": ("Copied from context", "copied from the prompt or earlier output by n-gram lookup, then verified"),
|
| 11 |
+
"head": ("Exit head (layer 4)", "drafted by the shallow exit head, then verified"),
|
| 12 |
+
"star": ("Head top-4 candidate", "an alternative from the head's top-4, then verified"),
|
| 13 |
+
"draft": ("Draft model", "drafted by a separate model, then verified"),
|
| 14 |
+
"model": ("Full model", "produced by the full model (correction or bonus token of a verification)"),
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class Detokenizer:
|
| 19 |
+
"""Incremental decoding into (text, source) pieces. Tokens that end inside a multi-byte
|
| 20 |
+
character are held back until the character is complete; a piece takes the source of its
|
| 21 |
+
first token."""
|
| 22 |
+
|
| 23 |
+
def __init__(self, tokenizer):
|
| 24 |
+
self.tok = tokenizer
|
| 25 |
+
self.ids: list[int] = []
|
| 26 |
+
self.text = ""
|
| 27 |
+
self.pending: list[str] = []
|
| 28 |
+
|
| 29 |
+
def _emit(self, final: bool) -> list[tuple[str, str]]:
|
| 30 |
+
cur = self.tok.decode(self.ids)
|
| 31 |
+
if cur.endswith("�") and not final:
|
| 32 |
+
return []
|
| 33 |
+
piece = cur[len(self.text):]
|
| 34 |
+
src = self.pending[0] if self.pending else "model"
|
| 35 |
+
self.text, self.pending = cur, []
|
| 36 |
+
return [(piece, src)] if piece else []
|
| 37 |
+
|
| 38 |
+
def feed(self, ids: list[int], sources: list[str]) -> list[tuple[str, str]]:
|
| 39 |
+
out = []
|
| 40 |
+
for t, s in zip(ids, sources):
|
| 41 |
+
self.ids.append(int(t))
|
| 42 |
+
self.pending.append(s)
|
| 43 |
+
out += self._emit(final=False)
|
| 44 |
+
return out
|
| 45 |
+
|
| 46 |
+
def flush(self) -> list[tuple[str, str]]:
|
| 47 |
+
return self._emit(final=True) if self.pending else []
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def tokens_html(pieces, colored: bool = True, cursor: bool = False) -> str:
|
| 51 |
+
out = []
|
| 52 |
+
for text, src in pieces:
|
| 53 |
+
t = html.escape(text)
|
| 54 |
+
if colored and src != "model" and src in SOURCES:
|
| 55 |
+
out.append(f'<span class="tok tok-{src}" title="{html.escape(SOURCES[src][1])}">{t}</span>')
|
| 56 |
+
else:
|
| 57 |
+
out.append(t)
|
| 58 |
+
if cursor:
|
| 59 |
+
out.append('<span class="cursor"></span>')
|
| 60 |
+
return f'<div class="dssd-text">{"".join(out)}</div>'
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def legend_html() -> str:
|
| 64 |
+
chips = "".join(
|
| 65 |
+
f'<span class="legend-item"><span class="tok tok-{s} swatch">Aa</span>'
|
| 66 |
+
f"{html.escape(SOURCES[s][0])}</span>"
|
| 67 |
+
for s in ("copy", "head", "star", "model"))
|
| 68 |
+
return f'<div class="legend">{chips}</div>'
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _card(value: str, label: str, cls: str = "") -> str:
|
| 72 |
+
return f'<div class="stat {cls}"><div class="stat-value">{value}</div><div class="stat-label">{label}</div></div>'
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def stats_html(g) -> str:
|
| 76 |
+
"""Stat cards and the source-share bar for a DSSD Generation (with g.ar when compared)."""
|
| 77 |
+
n = max(len(g.sources), 1)
|
| 78 |
+
counts = {s: g.sources.count(s) for s in ("copy", "head", "star", "draft", "model")}
|
| 79 |
+
cards = []
|
| 80 |
+
if g.ar is not None and g.speedup is not None:
|
| 81 |
+
cards.append(_card(f"{g.speedup:.2f}×", "decoding speedup", "stat-main"))
|
| 82 |
+
cards.append(_card(f"{g.decode_tokens_per_s:.1f}", "DSSD tokens/s"))
|
| 83 |
+
if g.ar is not None:
|
| 84 |
+
cards.append(_card(f"{g.ar.decode_tokens_per_s:.1f}", "autoregressive tokens/s"))
|
| 85 |
+
cards.append(_card(f"{g.stats['tau']:.2f}", "tokens per verification"))
|
| 86 |
+
if g.ar is not None:
|
| 87 |
+
ok = bool(g.identical)
|
| 88 |
+
cards.append(_card("✓ identical" if ok else "✗ outputs differ", "DSSD vs autoregressive",
|
| 89 |
+
"stat-ok" if ok else "stat-bad"))
|
| 90 |
+
bar = "".join(
|
| 91 |
+
f'<div class="share tok-{s}" style="width:{100 * c / n:.2f}%" title="{SOURCES[s][0]}: {c} tokens"></div>'
|
| 92 |
+
for s, c in counts.items() if c)
|
| 93 |
+
labels = " · ".join(f"{SOURCES[s][0]} {round(100 * c / n)}%" for s, c in counts.items() if c)
|
| 94 |
+
return (f'<div class="stats">{"".join(cards)}</div>'
|
| 95 |
+
f'<div class="share-bar">{bar}</div><div class="share-labels">{html.escape(labels)}</div>')
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def replay_html(timelines: dict) -> str:
|
| 99 |
+
"""Hidden element holding {"dssd": [[t, text, source], ...], "ar": [...]} for the client-side
|
| 100 |
+
side-by-side replay at the measured speed."""
|
| 101 |
+
data = html.escape(json.dumps(timelines, ensure_ascii=False), quote=True)
|
| 102 |
+
return f"<div class=\"replay-data\" data-replay='{data}'></div>"
|
demo/tiny.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tiny random model with logit-lens heads, for local UI development on CPU: the real dssd
|
| 2 |
+
engine runs end to end (exact verification, copies, head, star), on gibberish text."""
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from transformers import Qwen3Config, Qwen3ForCausalLM
|
| 6 |
+
|
| 7 |
+
from dssd import DSSD
|
| 8 |
+
from dssd.heads import ExitHead
|
| 9 |
+
|
| 10 |
+
LAYERS = [2, 4, 6]
|
| 11 |
+
_CHARS = [chr(c) for c in range(32, 127)] + ["\n", "\t"] # 97 ids
|
| 12 |
+
_TREE = {"copy": {"m": 11, "n_max": 4, "n_min": 1}, "head_budget": 4,
|
| 13 |
+
"star": {"head": 0, "C": 4, "tau": [0.0, 0.0, 0.0, 0.0], "gamma": 0.0}}
|
| 14 |
+
_POLICY = {"type": "threshold", "thresholds": {"0": 0.0, "2": 0.0}, "cum_gamma": {"0": 0.0}, "max_draft": 15,
|
| 15 |
+
"tree": _TREE}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class CharTokenizer:
|
| 19 |
+
eos_token_id = None
|
| 20 |
+
|
| 21 |
+
def encode(self, text, add_special_tokens=False):
|
| 22 |
+
return [_CHARS.index(c) if c in _CHARS else _CHARS.index("?") for c in text]
|
| 23 |
+
|
| 24 |
+
def decode(self, ids, skip_special_tokens=True):
|
| 25 |
+
return "".join(_CHARS[i % len(_CHARS)] for i in ids)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def tiny_engine() -> DSSD:
|
| 29 |
+
torch.manual_seed(0)
|
| 30 |
+
cfg = Qwen3Config(vocab_size=len(_CHARS), hidden_size=64, intermediate_size=128, num_hidden_layers=8,
|
| 31 |
+
num_attention_heads=4, num_key_value_heads=2, head_dim=16, max_position_embeddings=2048,
|
| 32 |
+
tie_word_embeddings=False)
|
| 33 |
+
m = Qwen3ForCausalLM(cfg).eval()
|
| 34 |
+
with torch.no_grad():
|
| 35 |
+
for p in m.parameters():
|
| 36 |
+
if p.dim() > 1:
|
| 37 |
+
p.mul_(4.0)
|
| 38 |
+
for layer in m.model.layers[5:]: # deep layers do nothing: the heads are often right
|
| 39 |
+
layer.self_attn.o_proj.weight.zero_()
|
| 40 |
+
layer.mlp.down_proj.weight.zero_()
|
| 41 |
+
heads = []
|
| 42 |
+
for _ in LAYERS:
|
| 43 |
+
full = ExitHead(64, cfg.vocab_size, cfg.rms_norm_eps)
|
| 44 |
+
with torch.no_grad():
|
| 45 |
+
full.norm.weight.copy_(m.model.norm.weight)
|
| 46 |
+
full.proj.weight.copy_(m.lm_head.weight)
|
| 47 |
+
heads.append(ExitHead.from_full(full, torch.arange(cfg.vocab_size), reject=True))
|
| 48 |
+
config = {"base_model": "tiny", "default_domain": "web",
|
| 49 |
+
"domains": {"web": {"policy": _POLICY, "stop": None},
|
| 50 |
+
"gsm8k": {"policy": _POLICY, "stop": "gsm8k"},
|
| 51 |
+
"code": {"policy": _POLICY, "stop": "humaneval"}}}
|
| 52 |
+
return DSSD.from_components(m, CharTokenizer(), heads, LAYERS, config, max_len=2048)
|
dssd/VENDORED.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Vendored from BranchyLLM 73b3f27 (dssd-pub); do not edit here.
|
dssd/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DSSD: exact self-speculative decoding with pruned exit heads and context copying."""
|
| 2 |
+
|
| 3 |
+
__version__ = "0.1.0"
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def __getattr__(name):
|
| 7 |
+
# lazy: `import dssd` must not import transformers (the engine modules do not need it)
|
| 8 |
+
if name in ("DSSD", "Generation", "StreamEvent"):
|
| 9 |
+
from . import api
|
| 10 |
+
return getattr(api, name)
|
| 11 |
+
raise AttributeError(f"module 'dssd' has no attribute {name!r}")
|
dssd/api.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""User API: load a published head set with its base model, generate with provenance.
|
| 2 |
+
|
| 3 |
+
from dssd import DSSD
|
| 4 |
+
m = DSSD.from_pretrained("<namespace>/DSSD-Qwen3-8B-Base")
|
| 5 |
+
g = m.generate("def fib(n):", max_new_tokens=256, domain="code", compare_ar=True)
|
| 6 |
+
g.text, g.sources, g.speedup, g.identical
|
| 7 |
+
|
| 8 |
+
Greedy decoding only; the output is identical to greedy autoregressive decoding of the same
|
| 9 |
+
engine (exact verification, row-invariant computation).
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import queue
|
| 15 |
+
import threading
|
| 16 |
+
import time
|
| 17 |
+
from collections import Counter
|
| 18 |
+
from dataclasses import dataclass, field
|
| 19 |
+
from typing import Iterator
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
from .decode import ThresholdPolicy
|
| 25 |
+
from .fast import FastDecoder
|
| 26 |
+
from .hub import load_heads, read_config
|
| 27 |
+
from .layered import LayeredDecoder, use_row_invariant_blas
|
| 28 |
+
from .stops import make_stop
|
| 29 |
+
|
| 30 |
+
ALIASES = {"math": "gsm8k", "humaneval": "code"}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@dataclass
|
| 34 |
+
class Generation:
|
| 35 |
+
text: str
|
| 36 |
+
token_ids: list[int]
|
| 37 |
+
sources: list[str]
|
| 38 |
+
stats: dict
|
| 39 |
+
decode_tokens_per_s: float # (new_tokens - 1) / (seconds - prefill_seconds)
|
| 40 |
+
ar: "Generation | None" = None
|
| 41 |
+
speedup: float | None = None # decode_tokens_per_s / ar.decode_tokens_per_s
|
| 42 |
+
identical: bool | None = None # token_ids == ar.token_ids
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class StreamEvent:
|
| 47 |
+
token_ids: list[int] # tokens appended by this event
|
| 48 |
+
sources: list[str]
|
| 49 |
+
text: str # text of the phase so far
|
| 50 |
+
done: bool
|
| 51 |
+
phase: str = "dssd" # "dssd" or "ar"
|
| 52 |
+
result: Generation | None = None # on the final event
|
| 53 |
+
t: float = 0.0 # seconds since the phase's first token (end of prefill), taken
|
| 54 |
+
# when the engine committed these tokens
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class _GammaOnly:
|
| 58 |
+
"""Inner controller of a gamma-only cost safety: never asks for AR, no lambda."""
|
| 59 |
+
lam, probe, K = 0.0, False, 4
|
| 60 |
+
|
| 61 |
+
def policy(self):
|
| 62 |
+
return np.zeros(self.K), 0.0, False
|
| 63 |
+
|
| 64 |
+
def update(self, c):
|
| 65 |
+
pass
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
class _Cancelled(Exception):
|
| 69 |
+
pass
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class DSSD:
|
| 73 |
+
MAX_ROWS = 16 # fixed engine width (row-invariant GEMMs)
|
| 74 |
+
|
| 75 |
+
def __init__(self, model, tokenizer, heads, layers, config: dict, graphs: bool | str = "auto",
|
| 76 |
+
max_len: int = 4096, prefill: str = "engine"):
|
| 77 |
+
"""No engine work here (ZeroGPU: CUDA is only emulated at module level): the engine is
|
| 78 |
+
built and its graphs captured on first use, or by warmup()."""
|
| 79 |
+
self.model, self.tokenizer, self.config = model.eval().requires_grad_(False), tokenizer, config
|
| 80 |
+
self.heads = [h.eval().requires_grad_(False) for h in heads]
|
| 81 |
+
self.layers, self.graphs, self.max_len, self.prefill = list(layers), graphs, max_len, prefill
|
| 82 |
+
self.domains = list(config["domains"])
|
| 83 |
+
eos = getattr(tokenizer, "eos_token_id", None)
|
| 84 |
+
self.eos_ids = frozenset() if eos is None else frozenset([eos])
|
| 85 |
+
self._lock = threading.Lock() # one generation at a time on the engine buffers
|
| 86 |
+
self._ar_step = None # seconds per AR step (cost safety), measured once
|
| 87 |
+
self._fast, self._ready = None, False
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def fast(self) -> FastDecoder:
|
| 91 |
+
if self._fast is None:
|
| 92 |
+
dev = next(self.model.parameters()).device
|
| 93 |
+
use_graphs = (dev.type == "cuda") if self.graphs == "auto" else bool(self.graphs)
|
| 94 |
+
self._fast = FastDecoder(LayeredDecoder(self.model, max_len=self.max_len), self.heads, self.layers,
|
| 95 |
+
max_rows=self.MAX_ROWS, use_graphs=use_graphs, prefill=self.prefill)
|
| 96 |
+
return self._fast
|
| 97 |
+
|
| 98 |
+
def _warmup(self) -> None:
|
| 99 |
+
if not self._ready:
|
| 100 |
+
with torch.no_grad():
|
| 101 |
+
self.fast.precapture([self._policy(d) for d in self.domains]
|
| 102 |
+
+ [ThresholdPolicy(self.heads, {k: 1.0 for k in range(len(self.heads))},
|
| 103 |
+
max_draft=15)])
|
| 104 |
+
self._ready = True
|
| 105 |
+
|
| 106 |
+
def warmup(self) -> None:
|
| 107 |
+
"""Build the engine and capture its graphs now (otherwise done by the first generation)."""
|
| 108 |
+
with self._lock:
|
| 109 |
+
self._warmup()
|
| 110 |
+
|
| 111 |
+
# ------------------------------------------------------------------ loading
|
| 112 |
+
@classmethod
|
| 113 |
+
def from_components(cls, model, tokenizer, heads, layers, config: dict, graphs: bool | str = "auto",
|
| 114 |
+
max_len: int = 4096, prefill: str = "engine") -> "DSSD":
|
| 115 |
+
use_row_invariant_blas() # rocBLAS on ROCm (row-invariant GEMMs); no-op elsewhere
|
| 116 |
+
return cls(model, tokenizer, heads, layers, config, graphs, max_len, prefill)
|
| 117 |
+
|
| 118 |
+
@classmethod
|
| 119 |
+
def from_pretrained(cls, repo_or_dir: str, device: str | None = None, dtype=torch.bfloat16,
|
| 120 |
+
graphs: bool | str = "auto", max_len: int = 4096, base_model: str | None = None,
|
| 121 |
+
revision: str | None = None, token: str | None = None,
|
| 122 |
+
prefill: str = "engine") -> "DSSD":
|
| 123 |
+
"""Heads from a Hub repo or directory; base model and tokenizer from the config's
|
| 124 |
+
base_model at its pinned revision (or `base_model`, unpinned)."""
|
| 125 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 126 |
+
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
| 127 |
+
if device == "cpu":
|
| 128 |
+
dtype = torch.float32
|
| 129 |
+
conf = read_config(repo_or_dir, revision, token)
|
| 130 |
+
base = base_model or conf["base_model"]
|
| 131 |
+
base_rev = None if base_model else conf.get("base_revision")
|
| 132 |
+
tok = AutoTokenizer.from_pretrained(base, revision=base_rev, token=token)
|
| 133 |
+
model = AutoModelForCausalLM.from_pretrained(base, revision=base_rev, dtype=dtype, token=token)
|
| 134 |
+
model = model.to(device).eval()
|
| 135 |
+
heads, layers, conf = load_heads(repo_or_dir, model.config, device, dtype, revision, token)
|
| 136 |
+
return cls.from_components(model, tok, heads, layers, conf, graphs, max_len, prefill)
|
| 137 |
+
|
| 138 |
+
# ------------------------------------------------------------------ helpers
|
| 139 |
+
def _domain(self, domain: str | None) -> str:
|
| 140 |
+
d = ALIASES.get(domain, domain) if domain is not None else self.config.get("default_domain", self.domains[0])
|
| 141 |
+
if d not in self.config["domains"]:
|
| 142 |
+
raise ValueError(f"unknown domain {domain!r}; available: {self.domains}")
|
| 143 |
+
return d
|
| 144 |
+
|
| 145 |
+
def _policy(self, domain: str) -> ThresholdPolicy:
|
| 146 |
+
spec = self.config["domains"][domain]["policy"]
|
| 147 |
+
if spec.get("type") != "threshold":
|
| 148 |
+
raise ValueError(f"unsupported policy type {spec.get('type')!r}")
|
| 149 |
+
g = spec.get("cum_gamma", 0.0)
|
| 150 |
+
pol = ThresholdPolicy(self.heads, {int(k): float(v) for k, v in spec["thresholds"].items()},
|
| 151 |
+
max_draft=spec.get("max_draft", 15),
|
| 152 |
+
cum_gamma={int(k): float(v) for k, v in g.items()} if isinstance(g, dict) else float(g),
|
| 153 |
+
acc_maps=spec.get("acc_maps"))
|
| 154 |
+
pol.tree = spec.get("tree")
|
| 155 |
+
pol.star = spec.get("star")
|
| 156 |
+
return pol
|
| 157 |
+
|
| 158 |
+
def _ids(self, prompt) -> list[int]:
|
| 159 |
+
if not isinstance(prompt, str):
|
| 160 |
+
return list(prompt)
|
| 161 |
+
# as for head training and the benchmark prompts: no BOS / special tokens
|
| 162 |
+
return list(self.tokenizer.encode(prompt, add_special_tokens=False))
|
| 163 |
+
|
| 164 |
+
def _stop(self, domain: str, stop):
|
| 165 |
+
kind = self.config["domains"][domain].get("stop") if stop == "auto" else stop
|
| 166 |
+
return make_stop(self.tokenizer, kind)
|
| 167 |
+
|
| 168 |
+
def _check(self, ids, max_new):
|
| 169 |
+
if len(ids) + max_new > self.max_len - self.MAX_ROWS:
|
| 170 |
+
raise ValueError(f"prompt ({len(ids)} tokens) + max_new_tokens ({max_new}) exceeds "
|
| 171 |
+
f"max_len={self.max_len} minus {self.MAX_ROWS} draft rows")
|
| 172 |
+
|
| 173 |
+
def _result(self, out, st) -> Generation:
|
| 174 |
+
d = st.as_dict()
|
| 175 |
+
d["source_counts"] = dict(Counter(st.sources))
|
| 176 |
+
dec_s = st.seconds - st.prefill_seconds
|
| 177 |
+
return Generation(self.tokenizer.decode(out), list(out), list(st.sources), d,
|
| 178 |
+
(st.new_tokens - 1) / dec_s if dec_s > 0 else float("nan"))
|
| 179 |
+
|
| 180 |
+
def _ar_step_seconds(self, ids) -> float:
|
| 181 |
+
if self._ar_step is None:
|
| 182 |
+
n = min(33, self.fast.dec.max_len - self.fast.max_rows - len(ids))
|
| 183 |
+
_, st = self.fast.generate_ar(ids, n)
|
| 184 |
+
self._ar_step = st.ar_step_median
|
| 185 |
+
return self._ar_step
|
| 186 |
+
|
| 187 |
+
def _run_dssd(self, ids, max_new, domain, stop, no_slowdown, on_commit=None):
|
| 188 |
+
from .calibration import online
|
| 189 |
+
pol = self._policy(domain)
|
| 190 |
+
kw = {}
|
| 191 |
+
if no_slowdown: # cost safety (chantier 3b, NoSlowdown, gamma only): never slower than AR (+xi)
|
| 192 |
+
ctl = online.NoSlowdown(0.02, 0.02, 0.0, _GammaOnly(), gamma_min=0.0)
|
| 193 |
+
ctl.K, ctl.gamma_only = len(self.heads), True
|
| 194 |
+
kw = {"controller": ctl, "ar_step_seconds": self._ar_step_seconds(ids)}
|
| 195 |
+
out, st = self.fast.generate_spec(pol, ids, max_new, eos_ids=self.eos_ids, stop=self._stop(domain, stop),
|
| 196 |
+
pld=self.config["domains"][domain]["policy"].get("pld"),
|
| 197 |
+
on_commit=on_commit, **kw)
|
| 198 |
+
return self._result(out, st)
|
| 199 |
+
|
| 200 |
+
def _run_ar(self, ids, max_new, domain, stop, on_commit=None):
|
| 201 |
+
out, st = self.fast.generate_ar(ids, max_new, eos_ids=self.eos_ids, stop=self._stop(domain, stop),
|
| 202 |
+
on_commit=on_commit)
|
| 203 |
+
return self._result(out, st)
|
| 204 |
+
|
| 205 |
+
@staticmethod
|
| 206 |
+
def _compare(g: Generation, ar: Generation) -> Generation:
|
| 207 |
+
g.ar, g.identical = ar, g.token_ids == ar.token_ids
|
| 208 |
+
g.speedup = g.decode_tokens_per_s / ar.decode_tokens_per_s if ar.decode_tokens_per_s > 0 else None
|
| 209 |
+
return g
|
| 210 |
+
|
| 211 |
+
# ------------------------------------------------------------------ API
|
| 212 |
+
@torch.no_grad()
|
| 213 |
+
def generate(self, prompt, max_new_tokens: int = 256, domain: str | None = None, stop: str | None = "auto",
|
| 214 |
+
compare_ar: bool = False, no_slowdown: bool = False) -> Generation:
|
| 215 |
+
"""Greedy DSSD generation. domain: "web", "gsm8k" (alias "math") or "code" selects the
|
| 216 |
+
calibrated policy and, with stop="auto", the stop rule. compare_ar: also run greedy AR
|
| 217 |
+
on the same engine and report speedup (decoding, prefill excluded) and identity."""
|
| 218 |
+
d, ids = self._domain(domain), self._ids(prompt)
|
| 219 |
+
self._check(ids, max_new_tokens)
|
| 220 |
+
with self._lock:
|
| 221 |
+
self._warmup()
|
| 222 |
+
g = self._run_dssd(ids, max_new_tokens, d, stop, no_slowdown)
|
| 223 |
+
if compare_ar:
|
| 224 |
+
g = self._compare(g, self._run_ar(ids, max_new_tokens, d, stop))
|
| 225 |
+
return g
|
| 226 |
+
|
| 227 |
+
@torch.no_grad()
|
| 228 |
+
def generate_ar(self, prompt, max_new_tokens: int = 256, domain: str | None = None,
|
| 229 |
+
stop: str | None = "auto") -> Generation:
|
| 230 |
+
d, ids = self._domain(domain), self._ids(prompt)
|
| 231 |
+
self._check(ids, max_new_tokens)
|
| 232 |
+
with self._lock:
|
| 233 |
+
self._warmup()
|
| 234 |
+
return self._run_ar(ids, max_new_tokens, d, stop)
|
| 235 |
+
|
| 236 |
+
def stream(self, prompt, max_new_tokens: int = 256, domain: str | None = None, stop: str | None = "auto",
|
| 237 |
+
no_slowdown: bool = False, ar: bool = False) -> Iterator[StreamEvent]:
|
| 238 |
+
"""One event per verification cycle (DSSD), then, if ar=True, one per AR step; the
|
| 239 |
+
final event (done=True) carries the Generation (with .ar, speedup, identical)."""
|
| 240 |
+
d, ids = self._domain(domain), self._ids(prompt)
|
| 241 |
+
self._check(ids, max_new_tokens)
|
| 242 |
+
q: queue.Queue = queue.Queue()
|
| 243 |
+
cancel = threading.Event()
|
| 244 |
+
|
| 245 |
+
def sink(phase):
|
| 246 |
+
t0 = []
|
| 247 |
+
|
| 248 |
+
def cb(toks, srcs): # runs inside the timed region: enqueue only, no decoding
|
| 249 |
+
if cancel.is_set():
|
| 250 |
+
raise _Cancelled
|
| 251 |
+
now = time.perf_counter()
|
| 252 |
+
if not t0:
|
| 253 |
+
t0.append(now)
|
| 254 |
+
q.put((phase, toks, srcs, now - t0[0]))
|
| 255 |
+
return cb
|
| 256 |
+
|
| 257 |
+
def work():
|
| 258 |
+
try:
|
| 259 |
+
with self._lock, torch.no_grad():
|
| 260 |
+
self._warmup()
|
| 261 |
+
g = self._run_dssd(ids, max_new_tokens, d, stop, no_slowdown, sink("dssd"))
|
| 262 |
+
if ar:
|
| 263 |
+
g = self._compare(g, self._run_ar(ids, max_new_tokens, d, stop, sink("ar")))
|
| 264 |
+
q.put(StreamEvent([], [], g.text, True, "ar" if ar else "dssd", g))
|
| 265 |
+
except _Cancelled:
|
| 266 |
+
pass
|
| 267 |
+
except BaseException as e: # re-raised in the consumer
|
| 268 |
+
q.put(e)
|
| 269 |
+
|
| 270 |
+
th = threading.Thread(target=work, daemon=True)
|
| 271 |
+
th.start()
|
| 272 |
+
acc = {"dssd": [], "ar": []}
|
| 273 |
+
try:
|
| 274 |
+
while True:
|
| 275 |
+
ev = q.get()
|
| 276 |
+
if isinstance(ev, BaseException):
|
| 277 |
+
raise ev
|
| 278 |
+
if isinstance(ev, tuple): # (phase, tokens, sources, t) from the worker
|
| 279 |
+
phase, toks, srcs, t = ev
|
| 280 |
+
acc[phase].extend(toks)
|
| 281 |
+
ev = StreamEvent(toks, srcs, self.tokenizer.decode(acc[phase]), False, phase, t=t)
|
| 282 |
+
yield ev
|
| 283 |
+
if ev.done:
|
| 284 |
+
return
|
| 285 |
+
finally:
|
| 286 |
+
cancel.set()
|
| 287 |
+
th.join()
|
dssd/calibration/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Certified (chantier 3) and online (chantier 3b) calibration, reference implementations."""
|
dssd/calibration/calib.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Calibration certifiée des seuils de sortie anticipée pour DSSD.
|
| 2 |
+
|
| 3 |
+
Copie du code de référence de chantiers/03-calibration/calib.py (13/10/2026).
|
| 4 |
+
|
| 5 |
+
Implémentation de référence de la procédure décrite dans ../03-calibration.md :
|
| 6 |
+
Learn-then-Test (Angelopoulos et al.) avec Fixed Sequence Testing le long d'un
|
| 7 |
+
chemin de configurations de seuils, construit sur une partie D1 des données de
|
| 8 |
+
calibration et testé sur la partie D2, avec des p-values au niveau SÉQUENCE
|
| 9 |
+
(robustes à la dépendance entre tokens d'une même séquence).
|
| 10 |
+
|
| 11 |
+
Conventions de données (toutes les fonctions publiques) :
|
| 12 |
+
H : (n, T, K) float — score de confiance de chaque tête (entropie : plus
|
| 13 |
+
petit = plus confiant). La tête k sort en t si H[i, t, k] <= tau[k].
|
| 14 |
+
A : (n, T, K) float dans [0, 1] — variable d'acceptation de la tête k :
|
| 15 |
+
binaire 1{argmax tête = argmax modèle} (vérification greedy) ou
|
| 16 |
+
1 - TV(p_tête, p_modèle) (échantillonnage spéculatif).
|
| 17 |
+
mask : (n, T) bool — positions valides (séquences de longueur variable).
|
| 18 |
+
tau : (K,) float — seuils ; -inf désactive une tête.
|
| 19 |
+
Chaque ligne i est une unité i.i.d. (un prompt et sa continuation générée par
|
| 20 |
+
le modèle cible dans le mode de décodage du déploiement).
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import math
|
| 26 |
+
from dataclasses import dataclass, field
|
| 27 |
+
|
| 28 |
+
import numpy as np
|
| 29 |
+
from scipy import stats
|
| 30 |
+
|
| 31 |
+
NO_DRAFT = None # configuration « jamais de brouillon » (toujours sûre)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# ---------------------------------------------------------------------------
|
| 35 |
+
# Évaluation de la cascade
|
| 36 |
+
# ---------------------------------------------------------------------------
|
| 37 |
+
|
| 38 |
+
def cascade_exit(H: np.ndarray, tau: np.ndarray) -> np.ndarray:
|
| 39 |
+
"""Indice de la première tête confiante en chaque position, -1 si aucune."""
|
| 40 |
+
ok = H <= np.asarray(tau)[None, None, :]
|
| 41 |
+
first = ok.argmax(axis=-1)
|
| 42 |
+
return np.where(ok.any(axis=-1), first, -1)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def seq_stats(H, A, mask, tau):
|
| 46 |
+
"""Statistiques par séquence d'une configuration de seuils.
|
| 47 |
+
|
| 48 |
+
Renvoie N (n,) nombre de tokens rédigés, S (n,) somme des acceptations,
|
| 49 |
+
et Nk, Sk (n, K) leur ventilation par tête de sortie.
|
| 50 |
+
"""
|
| 51 |
+
K = H.shape[-1]
|
| 52 |
+
ex = cascade_exit(H, tau)
|
| 53 |
+
ex = np.where(mask, ex, -1)
|
| 54 |
+
onehot = ex[..., None] == np.arange(K)[None, None, :] # (n, T, K)
|
| 55 |
+
Nk = onehot.sum(axis=1).astype(float)
|
| 56 |
+
Sk = (onehot * A).sum(axis=1)
|
| 57 |
+
return Nk.sum(1), Sk.sum(1), Nk, Sk
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def g_stat(N, S, eps, T):
|
| 61 |
+
"""G = (S - eps N) / T + eps, dans [0, 1].
|
| 62 |
+
|
| 63 |
+
E[G] >= eps <=> alpha = E[S] / E[N] >= eps (si E[N] > 0).
|
| 64 |
+
"""
|
| 65 |
+
return (S - eps * N) / T + eps
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
# ---------------------------------------------------------------------------
|
| 69 |
+
# p-values pour H0 : « le taux d'acceptation est < eps »
|
| 70 |
+
# ---------------------------------------------------------------------------
|
| 71 |
+
|
| 72 |
+
def pval_binom(n: int, n_rej: int, eps: float) -> float:
|
| 73 |
+
"""Test binomial exact (pertes binaires i.i.d.) : P(Bin(n, 1-eps) <= n_rej)."""
|
| 74 |
+
if n == 0:
|
| 75 |
+
return 1.0
|
| 76 |
+
return float(stats.binom.cdf(n_rej, n, 1.0 - eps))
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _h1(a, b):
|
| 80 |
+
a = min(max(a, 1e-12), 1 - 1e-12)
|
| 81 |
+
return a * math.log(a / b) + (1 - a) * math.log((1 - a) / (1 - b))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def pval_hb(losses: np.ndarray, alpha: float) -> float:
|
| 85 |
+
"""p-value de Hoeffding–Bentkus (Bates et al. 2021) pour H0 : E[perte] > alpha.
|
| 86 |
+
|
| 87 |
+
Valide pour des pertes i.i.d. dans [0, 1] (pas seulement binaires).
|
| 88 |
+
"""
|
| 89 |
+
n = len(losses)
|
| 90 |
+
if n == 0:
|
| 91 |
+
return 1.0
|
| 92 |
+
r = float(np.mean(losses))
|
| 93 |
+
p_h = math.exp(-n * _h1(min(r, alpha), alpha))
|
| 94 |
+
p_b = math.e * stats.binom.cdf(math.ceil(n * r), n, alpha)
|
| 95 |
+
return float(min(p_h, p_b, 1.0))
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def pval_hoeffding(losses: np.ndarray, alpha: float) -> float:
|
| 99 |
+
n = len(losses)
|
| 100 |
+
if n == 0:
|
| 101 |
+
return 1.0
|
| 102 |
+
gap = max(alpha - float(np.mean(losses)), 0.0)
|
| 103 |
+
return math.exp(-2 * n * gap**2)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def pval_betting(y: np.ndarray, m: float, *, delta: float = 0.05, c: float = 0.95,
|
| 107 |
+
strategy: str = "prpl", seed: int = 0) -> float:
|
| 108 |
+
"""p-value par mise (Waudby-Smith & Ramdas 2024) pour H0 : E[y] <= m, y dans [0, 1].
|
| 109 |
+
|
| 110 |
+
Richesse K_t = prod_{i<=t} (1 + lam_i (y_i - m)) avec lam_i prévisible dans
|
| 111 |
+
[0, c/m] ; sous H0 c'est une surmartingale positive de valeur initiale 1, donc
|
| 112 |
+
par l'inégalité de Ville p = min(1, 1 / max_t K_t) est une p-value valide.
|
| 113 |
+
L'ordre des observations est une permutation aléatoire indépendante des données.
|
| 114 |
+
"""
|
| 115 |
+
y = np.asarray(y, float)
|
| 116 |
+
n = len(y)
|
| 117 |
+
if n == 0:
|
| 118 |
+
return 1.0
|
| 119 |
+
y = y[np.random.default_rng(seed).permutation(n)]
|
| 120 |
+
t = np.arange(1, n + 1)
|
| 121 |
+
# moyennes/variances régularisées utilisant les observations < t (prévisibles)
|
| 122 |
+
mu = (m + np.concatenate([[0.0], np.cumsum(y)[:-1]])) / t
|
| 123 |
+
mu_incl = (m + np.cumsum(y)) / (t + 1)
|
| 124 |
+
sq = np.concatenate([[0.0], np.cumsum((y - mu_incl) ** 2)[:-1]])
|
| 125 |
+
var = (0.25 + sq) / t
|
| 126 |
+
if strategy == "agrapa":
|
| 127 |
+
gap = mu - m
|
| 128 |
+
lam = gap / (var + gap**2)
|
| 129 |
+
elif strategy == "prpl": # plug-in prévisible réglé pour n fixé
|
| 130 |
+
lam = np.sqrt(2 * math.log(1 / delta) / (var * n))
|
| 131 |
+
else:
|
| 132 |
+
raise ValueError(strategy)
|
| 133 |
+
lam = np.clip(lam, 0.0, c / m)
|
| 134 |
+
logK = np.cumsum(np.log1p(lam * (y - m)))
|
| 135 |
+
return float(min(1.0, math.exp(-logK.max()))) if logK.max() > 0 else 1.0
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
# ---------------------------------------------------------------------------
|
| 139 |
+
# Chemin de configurations (construit sur D1) et Fixed Sequence Testing
|
| 140 |
+
# ---------------------------------------------------------------------------
|
| 141 |
+
|
| 142 |
+
def _empirical_threshold(h, a, level):
|
| 143 |
+
"""Plus grand seuil tel que la moyenne de a parmi {h <= seuil} soit >= level."""
|
| 144 |
+
if len(h) == 0:
|
| 145 |
+
return -np.inf
|
| 146 |
+
o = np.argsort(h, kind="stable")
|
| 147 |
+
cm = np.cumsum(a[o]) / np.arange(1, len(h) + 1)
|
| 148 |
+
ok = np.nonzero(cm >= level)[0]
|
| 149 |
+
return float(h[o][ok[-1]]) if len(ok) else -np.inf
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def naive_thresholds(H, A, mask, level, cascade=True):
|
| 153 |
+
"""Seuils empiriques « plug-in » au niveau `level` (sans correction).
|
| 154 |
+
|
| 155 |
+
cascade=False reproduit la procédure actuelle du papier (chaque tête sur tous
|
| 156 |
+
les tokens) ; cascade=True calibre la tête k sur les tokens rejetés par les
|
| 157 |
+
têtes < k, comme à l'inférence.
|
| 158 |
+
"""
|
| 159 |
+
K = H.shape[-1]
|
| 160 |
+
remaining = mask.copy()
|
| 161 |
+
tau = np.full(K, -np.inf)
|
| 162 |
+
for k in range(K):
|
| 163 |
+
h, a = H[..., k][remaining], A[..., k][remaining]
|
| 164 |
+
tau[k] = _empirical_threshold(h, a, level)
|
| 165 |
+
if cascade:
|
| 166 |
+
remaining &= ~(H[..., k] <= tau[k])
|
| 167 |
+
return tau
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def build_path(H, A, mask, levels):
|
| 171 |
+
"""Chemin ordonné (du plus strict au plus lâche) de configurations distinctes."""
|
| 172 |
+
path = []
|
| 173 |
+
for lv in sorted(levels, reverse=True):
|
| 174 |
+
tau = naive_thresholds(H, A, mask, lv, cascade=True)
|
| 175 |
+
if not path or not np.array_equal(tau, path[-1]):
|
| 176 |
+
path.append(tau)
|
| 177 |
+
return path
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def fixed_sequence_test(pval, n_configs, delta, start=0):
|
| 181 |
+
"""Teste start, start+1, ... ; s'arrête au premier non-rejet.
|
| 182 |
+
|
| 183 |
+
Renvoie la liste des indices rejetés (= configurations certifiées). Contrôle
|
| 184 |
+
du FWER à delta pour tout ordre fixé indépendamment des données de test.
|
| 185 |
+
"""
|
| 186 |
+
rejected = []
|
| 187 |
+
for j in range(start, n_configs):
|
| 188 |
+
if pval(j) <= delta:
|
| 189 |
+
rejected.append(j)
|
| 190 |
+
else:
|
| 191 |
+
break
|
| 192 |
+
return rejected
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
# ---------------------------------------------------------------------------
|
| 196 |
+
# Procédure complète
|
| 197 |
+
# ---------------------------------------------------------------------------
|
| 198 |
+
|
| 199 |
+
@dataclass
|
| 200 |
+
class CalibrationResult:
|
| 201 |
+
tau: np.ndarray # seuils certifiés (-inf partout si rien n'est certifié)
|
| 202 |
+
certified: bool
|
| 203 |
+
path: list = field(repr=False)
|
| 204 |
+
index: int | None # indice dans le chemin
|
| 205 |
+
start: int
|
| 206 |
+
n_tested: int
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def config_pvalue(H, A, mask, tau, eps, delta, method="seq", per_head=False, seed=0,
|
| 210 |
+
c=0.95, strategy="prpl"):
|
| 211 |
+
"""p-value de H0(tau) : taux d'acceptation des tokens rédigés < eps.
|
| 212 |
+
|
| 213 |
+
method="seq" : unités = séquences, test par mise sur G (valide sans hypothèse
|
| 214 |
+
sur la dépendance intra-séquence) — méthode recommandée.
|
| 215 |
+
method="token" : unités = tokens rédigés supposés i.i.d. (binomial exact si A
|
| 216 |
+
binaire, Hoeffding–Bentkus sinon) — hypothèse forte.
|
| 217 |
+
per_head=True : garantie pour chaque tête parmi les tokens qu'elle fait sortir
|
| 218 |
+
dans la cascade (p = max_k p_k, test intersection-union).
|
| 219 |
+
"""
|
| 220 |
+
T = mask.shape[1]
|
| 221 |
+
heads = [k for k in range(len(tau)) if np.isfinite(tau[k])] if per_head else [None]
|
| 222 |
+
if not heads:
|
| 223 |
+
return 1.0
|
| 224 |
+
if method == "seq":
|
| 225 |
+
N, S, Nk, Sk = seq_stats(H, A, mask, tau)
|
| 226 |
+
elif method == "token":
|
| 227 |
+
ex = np.where(mask, cascade_exit(H, tau), -1)
|
| 228 |
+
a_ex = np.take_along_axis(A, np.maximum(ex, 0)[..., None], -1)[..., 0]
|
| 229 |
+
else:
|
| 230 |
+
raise ValueError(method)
|
| 231 |
+
ps = []
|
| 232 |
+
for k in heads:
|
| 233 |
+
if method == "seq":
|
| 234 |
+
Ng, Sg = (N, S) if k is None else (Nk[:, k], Sk[:, k])
|
| 235 |
+
ps.append(pval_betting(g_stat(Ng, Sg, eps, T), eps, delta=delta, c=c,
|
| 236 |
+
strategy=strategy, seed=seed))
|
| 237 |
+
else:
|
| 238 |
+
a = a_ex[ex >= 0] if k is None else a_ex[ex == k]
|
| 239 |
+
if np.all((a == 0) | (a == 1)):
|
| 240 |
+
ps.append(pval_binom(len(a), int(len(a) - a.sum()), eps))
|
| 241 |
+
else:
|
| 242 |
+
ps.append(pval_hb(1.0 - a, 1.0 - eps))
|
| 243 |
+
return max(ps)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def calibrate(H, A, mask, eps, delta, *, method="seq", per_head=False,
|
| 247 |
+
split=0.25, levels=None, seed=0, c=0.95, strategy="prpl") -> CalibrationResult:
|
| 248 |
+
"""Learn-then-Test : chemin sur D1, Fixed Sequence Testing sur D2.
|
| 249 |
+
|
| 250 |
+
Garantie (Théorème 1 de 03-calibration.md) : avec probabilité >= 1 - delta sur
|
| 251 |
+
le tirage de D2 (conditionnellement à D1), le taux d'acceptation alpha(tau) des
|
| 252 |
+
tokens rédigés par la cascade déployée est >= eps.
|
| 253 |
+
"""
|
| 254 |
+
n = H.shape[0]
|
| 255 |
+
rng = np.random.default_rng(seed)
|
| 256 |
+
perm = rng.permutation(n)
|
| 257 |
+
n1 = max(1, int(round(split * n)))
|
| 258 |
+
i1, i2 = perm[:n1], perm[n1:]
|
| 259 |
+
if levels is None:
|
| 260 |
+
levels = np.linspace(0.999, max(0.5 * eps, eps - 0.3), 150)
|
| 261 |
+
path = build_path(H[i1], A[i1], mask[i1], levels)
|
| 262 |
+
|
| 263 |
+
def pv(data_idx, j):
|
| 264 |
+
return config_pvalue(H[data_idx], A[data_idx], mask[data_idx], path[j], eps,
|
| 265 |
+
delta, method=method, per_head=per_head, seed=seed + j,
|
| 266 |
+
c=c, strategy=strategy)
|
| 267 |
+
|
| 268 |
+
# point de départ choisi sur D1 (indépendant de D2) : la configuration la plus
|
| 269 |
+
# « certifiable » ; on parcourt ensuite le chemin vers les seuils plus lâches.
|
| 270 |
+
start = int(np.argmin([pv(i1, j) for j in range(len(path))]))
|
| 271 |
+
rej = fixed_sequence_test(lambda j: pv(i2, j), len(path), delta, start=start)
|
| 272 |
+
if not rej:
|
| 273 |
+
return CalibrationResult(np.full(H.shape[-1], -np.inf), False, path, None,
|
| 274 |
+
start, 1)
|
| 275 |
+
j = rej[-1]
|
| 276 |
+
return CalibrationResult(path[j], True, path, j, start, len(rej) + 1)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def acceptance_rate(H, A, mask, tau):
|
| 280 |
+
"""alpha(tau) empirique groupé et proportion de positions rédigées."""
|
| 281 |
+
N, S, _, _ = seq_stats(H, A, mask, tau)
|
| 282 |
+
tot = N.sum()
|
| 283 |
+
return (S.sum() / tot if tot > 0 else 1.0), tot / mask.sum()
|
dssd/calibration/online.py
ADDED
|
@@ -0,0 +1,791 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Copie du code de référence de chantiers/03b-calibration-en-ligne/online.py (16/10/2026),
|
| 3 |
+
seul l'import de `calib` (chantier 3, copié à côté) est adapté au paquet.
|
| 4 |
+
Calibration EN LIGNE des seuils DSSD (chantier 3b).
|
| 5 |
+
|
| 6 |
+
Voir ../03b-calibration-en-ligne.md pour le formalisme et les preuves.
|
| 7 |
+
|
| 8 |
+
Contenu :
|
| 9 |
+
* AcceptancePredictor : â_k(s), probabilité d'acceptation prédite par tête,
|
| 10 |
+
isotone dans le score, ajustée hors ligne (D1). Elle définit la famille à un
|
| 11 |
+
paramètre « sortie à la première tête k telle que â_k >= lambda ».
|
| 12 |
+
* calibrate_lambda : Learn-then-Test (chantier 3) le long de cette famille.
|
| 13 |
+
* replay : rejoue le flot de contrôle de DSSD cycle par cycle (brouillon puis
|
| 14 |
+
vérification) sur des tableaux (H, A) de flux concaténés (régime A), avec un
|
| 15 |
+
contrôleur appelé après chaque vérification.
|
| 16 |
+
* Contrôleurs : Fixed, ACI (scalaire, avec ré-entrée par sondes contrefactuelles),
|
| 17 |
+
ACIPerHead, Deadband (hybride LTT + garde), DraftVerify (heuristique de Zhang et
|
| 18 |
+
al. 2024, éq. (2)–(4)), NoSlowdown (contrôle du coût par token, sur gamma).
|
| 19 |
+
|
| 20 |
+
Conventions : H (n, T, K) score, plus petit = plus confiant ; A (n, T, K) variable
|
| 21 |
+
d'acceptation dans [0, 1] (indicatrice d'accord en greedy, 1 - TV en échantillonnage) ;
|
| 22 |
+
mask (n, T). lambda > 1 <=> aucun brouillon (état AR, toujours sûr).
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import math
|
| 28 |
+
import pathlib
|
| 29 |
+
import sys
|
| 30 |
+
from dataclasses import dataclass, field
|
| 31 |
+
|
| 32 |
+
import numpy as np
|
| 33 |
+
from scipy.optimize import isotonic_regression
|
| 34 |
+
|
| 35 |
+
from . import calib # chantier 3 : p-values par mise, FST
|
| 36 |
+
|
| 37 |
+
AR_LEVEL = 1.0 # lambda > AR_LEVEL : aucune tête ne sort (â <= 1)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# ---------------------------------------------------------------------------
|
| 41 |
+
# Prédicteur d'acceptation et famille à un paramètre
|
| 42 |
+
# ---------------------------------------------------------------------------
|
| 43 |
+
|
| 44 |
+
class AcceptancePredictor:
|
| 45 |
+
"""â_k(s) : régression isotone (décroissante en s) de A_k sur H_k, par tête.
|
| 46 |
+
|
| 47 |
+
Ajustée sur tous les tokens valides de D1 (calibration marginale, pas « en
|
| 48 |
+
cascade ») : le biais de cascade est corrigé ensuite par LTT (hors ligne) ou
|
| 49 |
+
par le contrôleur (en ligne). Les deux n'exigent aucune qualité de â.
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
def __init__(self, H, A, mask, n_bins: int = 256):
|
| 53 |
+
K = H.shape[-1]
|
| 54 |
+
self.edges, self.values = [], []
|
| 55 |
+
for k in range(K):
|
| 56 |
+
h, a = H[..., k][mask], A[..., k][mask]
|
| 57 |
+
qs = np.unique(np.quantile(h, np.linspace(0, 1, n_bins + 1)[1:-1]))
|
| 58 |
+
b = np.searchsorted(qs, h, side="left")
|
| 59 |
+
cnt = np.bincount(b, minlength=len(qs) + 1).astype(float)
|
| 60 |
+
sm = np.bincount(b, weights=a, minlength=len(qs) + 1)
|
| 61 |
+
keep = cnt > 0
|
| 62 |
+
y = np.zeros_like(cnt)
|
| 63 |
+
y[keep] = isotonic_regression(sm[keep] / cnt[keep], weights=cnt[keep],
|
| 64 |
+
increasing=False).x
|
| 65 |
+
# bacs vides : valeur du bac non vide précédent (reste monotone)
|
| 66 |
+
last = y[np.argmax(keep)]
|
| 67 |
+
for i in range(len(y)):
|
| 68 |
+
if keep[i]:
|
| 69 |
+
last = y[i]
|
| 70 |
+
else:
|
| 71 |
+
y[i] = last
|
| 72 |
+
self.edges.append(qs)
|
| 73 |
+
self.values.append(np.clip(y, 0.0, 1.0))
|
| 74 |
+
|
| 75 |
+
def predict(self, H):
|
| 76 |
+
out = np.empty(H.shape, float)
|
| 77 |
+
for k, (e, v) in enumerate(zip(self.edges, self.values)):
|
| 78 |
+
out[..., k] = v[np.searchsorted(e, H[..., k], side="left")]
|
| 79 |
+
return out
|
| 80 |
+
|
| 81 |
+
def thresholds(self, lam):
|
| 82 |
+
"""tau_k(lambda) : {â_k(s) >= lambda} = {s <= tau_k} (-inf : tête jamais confiante).
|
| 83 |
+
|
| 84 |
+
Permet au moteur de garder ses comparaisons de seuils sur le GPU : le CPU
|
| 85 |
+
recalcule les K seuils après chaque mise à jour de lambda.
|
| 86 |
+
"""
|
| 87 |
+
lam = np.broadcast_to(np.asarray(lam, float), (len(self.values),))
|
| 88 |
+
tau = np.full(len(self.values), -np.inf)
|
| 89 |
+
for k, (e, v) in enumerate(zip(self.edges, self.values)):
|
| 90 |
+
ok = np.nonzero(v >= lam[k])[0]
|
| 91 |
+
if len(ok):
|
| 92 |
+
b = ok[-1]
|
| 93 |
+
tau[k] = np.inf if b == len(e) else e[b]
|
| 94 |
+
return tau
|
| 95 |
+
|
| 96 |
+
def to_json(self):
|
| 97 |
+
return {"edges": [e.tolist() for e in self.edges], "values": [v.tolist() for v in self.values]}
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def lambda_stats(Ahat, A, mask, lam):
|
| 101 |
+
"""(N, S) par séquence pour la règle « première tête avec â_k >= lam_k »."""
|
| 102 |
+
lam = np.broadcast_to(np.asarray(lam, float), (Ahat.shape[-1],))
|
| 103 |
+
N, S, _, _ = calib.seq_stats(-Ahat, A, mask, -lam)
|
| 104 |
+
return N, S
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def calibrate_lambda(H, A, mask, eps, delta, *, split=0.25, grid=None, seed=0):
|
| 108 |
+
"""LTT + FST (chantier 3, Th. 1) le long de la famille lambda.
|
| 109 |
+
|
| 110 |
+
D1 : ajuste â et choisit le point de départ ; D2 : p-values par mise (séquences).
|
| 111 |
+
Renvoie (lambda_hat, predictor) ; lambda_hat = inf si rien n'est certifié.
|
| 112 |
+
"""
|
| 113 |
+
n = H.shape[0]
|
| 114 |
+
rng = np.random.default_rng(seed)
|
| 115 |
+
perm = rng.permutation(n)
|
| 116 |
+
n1 = max(1, int(round(split * n)))
|
| 117 |
+
i1, i2 = perm[:n1], perm[n1:]
|
| 118 |
+
pred = AcceptancePredictor(H[i1], A[i1], mask[i1])
|
| 119 |
+
if grid is None:
|
| 120 |
+
grid = np.round(np.arange(0.999, 0.3, -0.005), 4) # du strict au lâche
|
| 121 |
+
T = mask.shape[1]
|
| 122 |
+
Ah1, Ah2 = pred.predict(H[i1]), pred.predict(H[i2])
|
| 123 |
+
|
| 124 |
+
def pv(Ah, idx, lam, s):
|
| 125 |
+
N, S = lambda_stats(Ah, A[idx], mask[idx], lam)
|
| 126 |
+
return calib.pval_betting(calib.g_stat(N, S, eps, T), eps, delta=delta, seed=s)
|
| 127 |
+
|
| 128 |
+
start = int(np.argmin([pv(Ah1, i1, l, seed + j) for j, l in enumerate(grid)]))
|
| 129 |
+
rej = calib.fixed_sequence_test(lambda j: pv(Ah2, i2, grid[j], seed + j), len(grid),
|
| 130 |
+
delta, start=start)
|
| 131 |
+
lam_hat = float(grid[rej[-1]]) if rej else math.inf
|
| 132 |
+
return lam_hat, pred
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def calibrate_gate(P, HIT, mask, eps, delta, *, split=0.25, grid=None, seed=0):
|
| 136 |
+
"""LTT + FST pour la PORTE d'un brouillon en arbre (greedy, régime A).
|
| 137 |
+
|
| 138 |
+
P[i, t] : valeur de la porte p(u) au nœud du VRAI chemin qui précède la position t,
|
| 139 |
+
calculée sur tous les candidats enfants issus du vrai préfixe (copie, têtes,
|
| 140 |
+
étoile), avant élagage ;
|
| 141 |
+
HIT[i, t] : 1 si l'un de ces candidats égale le token cible ;
|
| 142 |
+
mask[i, t] : positions valides.
|
| 143 |
+
En régime A, chaque position de sortie est tentée une fois sur son vrai préfixe, et la
|
| 144 |
+
profondeur correspondante est rédigée ssi p >= lambda : (N, S) par séquence se lisent
|
| 145 |
+
directement sur (P, HIT), comme pour une chaîne à une tête virtuelle (Prop. 1 du
|
| 146 |
+
chantier 3 ; le plafond de profondeur est ignoré). Les cartes q qui définissent p doivent
|
| 147 |
+
avoir été ajustées sur d'AUTRES données que (P, HIT).
|
| 148 |
+
Renvoie lambda_hat (inf si rien n'est certifié).
|
| 149 |
+
"""
|
| 150 |
+
n = P.shape[0]
|
| 151 |
+
perm = np.random.default_rng(seed).permutation(n)
|
| 152 |
+
n1 = max(1, int(round(split * n)))
|
| 153 |
+
i1, i2 = perm[:n1], perm[n1:]
|
| 154 |
+
if grid is None:
|
| 155 |
+
grid = np.round(np.arange(0.999, 0.3, -0.005), 4)
|
| 156 |
+
T = mask.shape[1]
|
| 157 |
+
Ah, Aa = P[..., None].astype(float), HIT[..., None].astype(float)
|
| 158 |
+
|
| 159 |
+
def pv(idx, lam, s):
|
| 160 |
+
N, S = lambda_stats(Ah[idx], Aa[idx], mask[idx], lam)
|
| 161 |
+
return calib.pval_betting(calib.g_stat(N, S, eps, T), eps, delta=delta, seed=s)
|
| 162 |
+
|
| 163 |
+
start = int(np.argmin([pv(i1, l, seed + j) for j, l in enumerate(grid)]))
|
| 164 |
+
rej = calib.fixed_sequence_test(lambda j: pv(i2, grid[j], seed + j), len(grid), delta, start=start)
|
| 165 |
+
return float(grid[rej[-1]]) if rej else math.inf
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
# ---------------------------------------------------------------------------
|
| 169 |
+
# Retour d'un cycle brouillon / vérification
|
| 170 |
+
# ---------------------------------------------------------------------------
|
| 171 |
+
|
| 172 |
+
@dataclass
|
| 173 |
+
class Cycle:
|
| 174 |
+
heads: list # tête de sortie de chaque brouillon du préfixe de sortie
|
| 175 |
+
labels: list # étiquette d'acceptation (a, ou beta en mode RB) de ces brouillons
|
| 176 |
+
accepted: int # brouillons acceptés (réalisés)
|
| 177 |
+
drafted: int # brouillons émis, y compris ceux gaspillés après un rejet
|
| 178 |
+
emitted: int # tokens émis par le cycle
|
| 179 |
+
cost: float # coût modélisé du cycle (1 = une passe du modèle complet)
|
| 180 |
+
stop_ahat: np.ndarray | None = None # â de TOUTES les têtes en position d'arrêt
|
| 181 |
+
stop_a: np.ndarray | None = None # étiquettes de TOUTES les têtes en position d'arrêt
|
| 182 |
+
ar: bool = False # cycle en mode AR (aucun brouillon tenté)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def cycle_from_hook(per_token, emitted, cost=1.0, betas=None, stop=None):
|
| 186 |
+
"""Adaptateur pour le crochet du moteur ``policy.update`` (chantier 1).
|
| 187 |
+
|
| 188 |
+
per_token : liste ORDONNÉE (k, accepté) pour chaque token rédigé du cycle, gaspillés
|
| 189 |
+
compris. Le préfixe vérifié = tokens jusqu'au premier refus inclus ; les suivants sont
|
| 190 |
+
gaspillés (préfixe faux) et ne reçoivent pas d'étiquette.
|
| 191 |
+
betas : beta_j = 1 - TV exact par token rédigé (T > 0), sinon indicatrices.
|
| 192 |
+
stop : (ahat (K,), a (K,)) à la position d'arrêt si le bloc est entièrement accepté.
|
| 193 |
+
"""
|
| 194 |
+
heads, labels, n_ok = [], [], 0
|
| 195 |
+
for j, (k, acc) in enumerate(per_token):
|
| 196 |
+
heads.append(int(k))
|
| 197 |
+
labels.append(float(betas[j]) if betas is not None else float(bool(acc)))
|
| 198 |
+
if not acc:
|
| 199 |
+
break
|
| 200 |
+
n_ok += 1
|
| 201 |
+
c = Cycle(heads, labels, n_ok, len(per_token), emitted, cost)
|
| 202 |
+
if stop is not None and n_ok == len(per_token):
|
| 203 |
+
c.stop_ahat, c.stop_a = np.asarray(stop[0], float), np.asarray(stop[1], float)
|
| 204 |
+
return c
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def cycle_from_tree(path_sources, failed, emitted, cost=1.0, stop=None, n_nodes=None):
|
| 208 |
+
"""Adaptateur pour un cycle de brouillon EN ARBRE (copie + têtes + étoile), en greedy.
|
| 209 |
+
|
| 210 |
+
Étiquettes = une par PROFONDEUR du chemin vérifié (généralisation de alpha_pf) :
|
| 211 |
+
1 pour chacune des m profondeurs acceptées, puis 0 pour la profondeur m + 1 si le nœud
|
| 212 |
+
accepté de profondeur m avait au moins un enfant rédigé (aucun n'a été accepté).
|
| 213 |
+
Au plus un 0 par cycle, donc g_min = eps et le Théorème 1 s'applique tel quel.
|
| 214 |
+
|
| 215 |
+
path_sources : origine des m nœuds acceptés, dans l'ordre (entier ou chaîne, pour stats) ;
|
| 216 |
+
failed : origine du meilleur enfant refusé à la profondeur m + 1, ou None si le
|
| 217 |
+
nœud accepté n'avait pas d'enfant (bloc arrêté par la politique) ;
|
| 218 |
+
stop : (p, a) à la profondeur d'arrêt si failed is None : p = acceptation prédite
|
| 219 |
+
de la profondeur suivante (non rédigée), a = 1 si un candidat calculé y
|
| 220 |
+
coïncide avec le token cible (information contrefactuelle du ré-ancrage) ;
|
| 221 |
+
n_nodes : nombre total de nœuds rédigés (gaspillés compris), pour les statistiques.
|
| 222 |
+
"""
|
| 223 |
+
heads = [0] * len(path_sources)
|
| 224 |
+
labels = [1.0] * len(path_sources)
|
| 225 |
+
if failed is not None:
|
| 226 |
+
heads.append(0)
|
| 227 |
+
labels.append(0.0)
|
| 228 |
+
c = Cycle(heads, labels, len(path_sources),
|
| 229 |
+
n_nodes if n_nodes is not None else len(labels), emitted, cost)
|
| 230 |
+
if failed is None and stop is not None:
|
| 231 |
+
c.stop_ahat = np.atleast_1d(np.asarray(stop[0], float))
|
| 232 |
+
c.stop_a = np.atleast_1d(np.asarray(stop[1], float))
|
| 233 |
+
return c
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
@dataclass
|
| 237 |
+
class CostModel:
|
| 238 |
+
"""Coût modélisé, en unités « une passe du modèle complet » (batch 1)."""
|
| 239 |
+
layer_frac: tuple = (0.15, 0.3, 0.45, 0.6) # couches jusqu'à la tête k / L (illustratif)
|
| 240 |
+
head: float = 0.005 # une tête élaguée
|
| 241 |
+
verify: float = 1.0 # passe de vérification (<= 16 lignes)
|
| 242 |
+
|
| 243 |
+
def draft(self, k):
|
| 244 |
+
return self.layer_frac[k] + self.head * (k + 1)
|
| 245 |
+
|
| 246 |
+
def heads(self, k): # cascade évaluée jusqu'à la tête k sans brouillon
|
| 247 |
+
return self.head * (k + 1)
|
| 248 |
+
|
| 249 |
+
def stop(self, K): # aucune tête confiante : toutes évaluées
|
| 250 |
+
return self.head * K
|
| 251 |
+
|
| 252 |
+
def probe(self, K, allowed=None): # sonde en mode AR
|
| 253 |
+
return self.head * (K if allowed is None else int(np.sum(allowed)))
|
| 254 |
+
|
| 255 |
+
def verify_cost(self, n, kmin):
|
| 256 |
+
"""kmin : tête la moins profonde atteinte par une ligne du bloc (None : couche 0)."""
|
| 257 |
+
return self.verify
|
| 258 |
+
|
| 259 |
+
def position(self, deepest, evaluated, drafted):
|
| 260 |
+
"""Coût d'une position de brouillon : têtes `evaluated` évaluées (la plus profonde :
|
| 261 |
+
`deepest`) ; les couches ne sont comptées que si l'on rédige (modèle illustratif)."""
|
| 262 |
+
return (self.layer_frac[deepest] if drafted else 0.0) + self.head * len(evaluated)
|
| 263 |
+
|
| 264 |
+
def verify_rows(self, depths, allowed=None):
|
| 265 |
+
"""depths : tête atteinte par chaque ligne du bloc (None : aucune couche)."""
|
| 266 |
+
ks = [k for k in depths if k is not None]
|
| 267 |
+
kmin = None if (None in depths or not ks) else min(ks)
|
| 268 |
+
return self.verify_cost(len(depths), kmin)
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
class MeasuredCost(CostModel):
|
| 272 |
+
"""Coût mesuré (chantier 1, ``costs.json`` : secondes sur MI210), en unités d'un pas AR.
|
| 273 |
+
|
| 274 |
+
Brouillon à la tête k : seg0 + l_k couches (1 ligne) + têtes 0..k. Vérification et
|
| 275 |
+
rattrapage paresseux (régime A) : passe groupée des n = brouillons + 1 lignes depuis la
|
| 276 |
+
profondeur MINIMALE atteinte par une ligne du bloc (lignes « verify » de costs.json,
|
| 277 |
+
interpolées en n ; depuis la couche 0 si le bloc a atteint L_max, la dernière ligne
|
| 278 |
+
n'ayant traversé aucune couche). C'est une borne supérieure du coût réel du
|
| 279 |
+
rattrapage + vérification [c_max, L).
|
| 280 |
+
"""
|
| 281 |
+
|
| 282 |
+
def __init__(self, path, layers, lazy=True, c_max=None):
|
| 283 |
+
import json
|
| 284 |
+
d = json.load(open(path))
|
| 285 |
+
f = d["fit"]
|
| 286 |
+
self.lazy = lazy
|
| 287 |
+
self.ar, self.layers = f["ar_measured"], list(layers)
|
| 288 |
+
self.seg0, self.layer, L = f["seg0"], f["layer"], f["L"]
|
| 289 |
+
hk = {r["k"]: r["t"] for r in d["rows"] if r["kind"] == "head"}
|
| 290 |
+
self.head_t = np.array([hk.get(k, f["head"]) for k in range(len(layers))])
|
| 291 |
+
self.ver = {}
|
| 292 |
+
for r in d["rows"]:
|
| 293 |
+
if r["kind"] == "verify":
|
| 294 |
+
self.ver.setdefault(r["from"], []).append((r["n"], r["t"]))
|
| 295 |
+
elif r["kind"] == "seg" and r["layers"] == L: # pile entière : ligne partie de 0
|
| 296 |
+
self.ver.setdefault(0, []).append((r["m"], r["t"] + f["lm"]))
|
| 297 |
+
for k in self.ver:
|
| 298 |
+
self.ver[k].sort()
|
| 299 |
+
self.layer_frac = tuple(np.array(layers) / L)
|
| 300 |
+
self.head = float(self.head_t.mean() / self.ar)
|
| 301 |
+
self.L, self.lm = L, f["lm"]
|
| 302 |
+
self.c_max = self.layers[-1] if c_max is None else c_max
|
| 303 |
+
full = sorted((r["m"], r["t"]) for r in d["rows"] if r["kind"] == "seg" and r["layers"] == L)
|
| 304 |
+
self._m_scale = (np.array([m for m, _ in full]), np.array([t for _, t in full]) / full[0][1])
|
| 305 |
+
|
| 306 |
+
def seg(self, n_layers, m):
|
| 307 |
+
"""Segment de n_layers couches sur m lignes (ajustement seg0 + couches, facteur en m)."""
|
| 308 |
+
if n_layers <= 0 or m <= 0:
|
| 309 |
+
return 0.0
|
| 310 |
+
return (self.seg0 + n_layers * self.layer) * float(np.interp(m, *self._m_scale))
|
| 311 |
+
|
| 312 |
+
def verify_rows(self, depths, allowed=None):
|
| 313 |
+
"""Régime A du moteur (chantier 1, 14/10) : rattrapage paresseux groupé des lignes en
|
| 314 |
+
retard, segment par segment entre frontières de têtes autorisées, jusqu'à c_max (tête
|
| 315 |
+
autorisée la plus profonde) ; puis vérification [c_max, L) sur toutes les lignes."""
|
| 316 |
+
if not self.lazy:
|
| 317 |
+
return super().verify_rows(depths)
|
| 318 |
+
lay = self.layers if allowed is None else [l for l, a in zip(self.layers, allowed) if a]
|
| 319 |
+
c_max = lay[-1] if allowed is not None else self.c_max
|
| 320 |
+
bounds = [0] + [l for l in lay if l <= c_max]
|
| 321 |
+
dep = [0 if k is None else self.layers[k] for k in depths]
|
| 322 |
+
t = 0.0
|
| 323 |
+
for a, b in zip(bounds[:-1], bounds[1:]):
|
| 324 |
+
m = sum(1 for x in dep if x <= a)
|
| 325 |
+
t += self.seg(b - a, m)
|
| 326 |
+
ns, ts = zip(*self.ver[c_max])
|
| 327 |
+
t += float(np.interp(len(depths), ns, ts))
|
| 328 |
+
return t / self.ar
|
| 329 |
+
|
| 330 |
+
def draft(self, k):
|
| 331 |
+
return (self.seg0 + self.layers[k] * self.layer + self.head_t[:k + 1].sum()) / self.ar
|
| 332 |
+
|
| 333 |
+
def heads(self, k):
|
| 334 |
+
return self.draft(k)
|
| 335 |
+
|
| 336 |
+
def stop(self, K):
|
| 337 |
+
return self.draft(K - 1)
|
| 338 |
+
|
| 339 |
+
def probe(self, K, allowed=None):
|
| 340 |
+
idx = range(K) if allowed is None else np.nonzero(allowed)[0]
|
| 341 |
+
return float(sum(self.head_t[k] for k in idx)) / self.ar
|
| 342 |
+
|
| 343 |
+
def position(self, deepest, evaluated, drafted):
|
| 344 |
+
return (self.seg0 + self.layers[deepest] * self.layer
|
| 345 |
+
+ float(sum(self.head_t[k] for k in evaluated))) / self.ar
|
| 346 |
+
|
| 347 |
+
def verify_cost(self, n, kmin):
|
| 348 |
+
tab = self.ver[0 if kmin is None else self.layers[kmin]]
|
| 349 |
+
ns, ts = zip(*tab)
|
| 350 |
+
return float(np.interp(n, ns, ts)) / self.ar
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
# ---------------------------------------------------------------------------
|
| 354 |
+
# Contrôleurs
|
| 355 |
+
# ---------------------------------------------------------------------------
|
| 356 |
+
|
| 357 |
+
class Controller:
|
| 358 |
+
"""Interface : policy() -> (lam (K,), gamma, ar) ; update(cycle) après vérification."""
|
| 359 |
+
K = 4
|
| 360 |
+
probe = False # True : évaluer la cascade en mode AR (étiquettes contrefactuelles)
|
| 361 |
+
|
| 362 |
+
def policy(self):
|
| 363 |
+
raise NotImplementedError
|
| 364 |
+
|
| 365 |
+
def update(self, c: Cycle):
|
| 366 |
+
pass
|
| 367 |
+
|
| 368 |
+
def begin_sequence(self, i: int):
|
| 369 |
+
"""Appelé au début de chaque requête (i = indice dans le flux)."""
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
class PerSegment(Controller):
|
| 373 |
+
"""Seuil fixe par segment du flux (oracle « calibrer par domaine »)."""
|
| 374 |
+
|
| 375 |
+
def __init__(self, lams, bounds):
|
| 376 |
+
self.lams, self.bounds, self.lam = lams, list(bounds.values()), lams[0]
|
| 377 |
+
|
| 378 |
+
def begin_sequence(self, i):
|
| 379 |
+
for lam, (lo, hi) in zip(self.lams, self.bounds):
|
| 380 |
+
if lo <= i < hi:
|
| 381 |
+
self.lam = lam
|
| 382 |
+
|
| 383 |
+
def policy(self):
|
| 384 |
+
return np.full(self.K, self.lam), 0.0, self.lam > AR_LEVEL
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
class Static(Controller):
|
| 388 |
+
"""Seuils par tête fixes (inf = tête exclue), gamma et L_max fixes : une politique (h)."""
|
| 389 |
+
|
| 390 |
+
def __init__(self, taus, gamma=0.0, lmax=None):
|
| 391 |
+
self.taus, self.gamma, self.lmax = np.asarray(taus, float), gamma, lmax
|
| 392 |
+
self.lam = float(np.min(self.taus))
|
| 393 |
+
|
| 394 |
+
def policy(self):
|
| 395 |
+
return self.taus.copy(), self.gamma, False
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
class MapPredictor:
|
| 399 |
+
"""â_k(conf) du chantier 1 (``policy_h.json``) : interpolation linéaire entre nœuds,
|
| 400 |
+
constante au-delà ; conf = -H. Tête sans carte : â = 0."""
|
| 401 |
+
|
| 402 |
+
def __init__(self, acc_maps, K):
|
| 403 |
+
self.maps = {int(k): (np.asarray(v["knots"]), np.asarray(v["values"])) for k, v in acc_maps.items()}
|
| 404 |
+
self.K = K
|
| 405 |
+
|
| 406 |
+
def predict(self, H):
|
| 407 |
+
out = np.zeros(H.shape, float)
|
| 408 |
+
for k, (x, y) in self.maps.items():
|
| 409 |
+
out[..., k] = np.interp(-H[..., k], x, y)
|
| 410 |
+
return out
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
class Fixed(Controller):
|
| 414 |
+
def __init__(self, lam, gamma=0.0):
|
| 415 |
+
self.lam, self.gamma = lam, gamma
|
| 416 |
+
|
| 417 |
+
def policy(self):
|
| 418 |
+
return np.full(self.K, self.lam), self.gamma, self.lam > AR_LEVEL
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
class ACI(Controller):
|
| 422 |
+
"""Intégrateur sur lambda (Gibbs & Candès 2021, adapté au risque sélectif), avec registre.
|
| 423 |
+
|
| 424 |
+
Mise à jour réalisée : lambda <- lambda - eta * sum_j (a_j - target), somme sur les
|
| 425 |
+
brouillons du préfixe vérifié du cycle (target = eps par défaut). lambda > 1 : mode AR.
|
| 426 |
+
|
| 427 |
+
Registre (Théorème 1) : tout déplacement de lambda qui ne vient pas de cette mise à
|
| 428 |
+
jour est « externe ». `ext` = déplacement externe net vers l'agressif (D - U). Les
|
| 429 |
+
déplacements vers le prudent (plafond lam_min, réinitialisations prudentes) sont
|
| 430 |
+
gratuits et créditent `ext` ; un déplacement vers l'agressif n'est autorisé que dans
|
| 431 |
+
la limite du budget eta * (J0 + rho * N), N = brouillons du préfixe déjà vérifiés.
|
| 432 |
+
Garantie : W_C >= -(g_min + (1 - lambda_1)/eta + J0 + rho N_C).
|
| 433 |
+
|
| 434 |
+
Ré-ancrage contrefactuel (reanchor=dict(...)) : aux positions d'arrêt (aucune tête
|
| 435 |
+
confiante, toutes évaluées) et, en mode AR, à chaque position sondée, on connaît
|
| 436 |
+
l'acceptation de TOUTES les têtes : on garde ces positions récentes, et toutes les
|
| 437 |
+
`every` positions on calcule le lambda le plus agressif dont l'acceptation
|
| 438 |
+
contrefactuelle y est >= target + margin (pour lui et tous les niveaux plus prudents) ;
|
| 439 |
+
s'il est plus bas que lambda de plus de min_jump, on y saute (dans le budget).
|
| 440 |
+
"""
|
| 441 |
+
|
| 442 |
+
def __init__(self, eps, eta, lam0, lam_min=0.0, gamma=0.0, target=None, reanchor=None,
|
| 443 |
+
offsets=None):
|
| 444 |
+
"""offsets : décalages d_k >= 0 par tête dans l'espace de â (inf = tête exclue) ;
|
| 445 |
+
la tête k sort si â_k >= lambda + d_k. Pour des seuils de base tau_k^0 (chantier 1) :
|
| 446 |
+
lam0 = min_k tau_k^0 et d_k = tau_k^0 - lam0. lambda > 1 <=> aucune tête ne sort."""
|
| 447 |
+
self.eps, self.eta, self.lam, self.lam_min, self.gamma = eps, eta, float(lam0), lam_min, gamma
|
| 448 |
+
self.offsets = None if offsets is None else np.asarray(offsets, float)
|
| 449 |
+
assert self.offsets is None or np.nanmin(self.offsets) >= 0
|
| 450 |
+
self.lam1 = float(lam0)
|
| 451 |
+
self.target = eps if target is None else target
|
| 452 |
+
self.ra = None if reanchor is None else dict(
|
| 453 |
+
dict(horizon=3000, every=512, n_min=300, margin=0.0, rho=0.005, J0=20.0,
|
| 454 |
+
min_jump=0.03, grid=np.round(np.arange(1.0, 0.295, -0.01), 3)), **reanchor)
|
| 455 |
+
self.probe = reanchor is not None
|
| 456 |
+
self.ext, self.n_draft, self.n_jumps = 0.0, 0, 0
|
| 457 |
+
self.t, self._since, self._buf = 0, 0, []
|
| 458 |
+
|
| 459 |
+
# -- registre -----------------------------------------------------------
|
| 460 |
+
def budget(self):
|
| 461 |
+
if self.ra is None:
|
| 462 |
+
return 0.0
|
| 463 |
+
return self.eta * (self.ra["J0"] + self.ra["rho"] * self.n_draft)
|
| 464 |
+
|
| 465 |
+
def move(self, new_lam):
|
| 466 |
+
"""Déplacement externe (reset, saut...) : vers le prudent toujours accepté,
|
| 467 |
+
vers l'agressif tronqué au budget. Renvoie le lambda effectif."""
|
| 468 |
+
delta = self.lam - new_lam
|
| 469 |
+
if delta > 0:
|
| 470 |
+
delta = min(delta, max(0.0, self.budget() - self.ext))
|
| 471 |
+
self.lam -= delta
|
| 472 |
+
self.ext += delta
|
| 473 |
+
return self.lam
|
| 474 |
+
|
| 475 |
+
def bound(self, g_min=None):
|
| 476 |
+
"""Borne déterministe courante sur le déficit -W (en tokens)."""
|
| 477 |
+
g_min = self.target if g_min is None else g_min
|
| 478 |
+
J = 0.0 if self.ra is None else self.ra["J0"] + self.ra["rho"] * self.n_draft
|
| 479 |
+
return g_min + max(0.0, AR_LEVEL - self.lam1) / self.eta + J
|
| 480 |
+
|
| 481 |
+
# -- contrôleur ---------------------------------------------------------
|
| 482 |
+
def _taus(self, lam):
|
| 483 |
+
return np.full(self.K, lam) if self.offsets is None else lam + self.offsets
|
| 484 |
+
|
| 485 |
+
def policy(self):
|
| 486 |
+
return self._taus(self.lam), self.gamma, self.lam > AR_LEVEL
|
| 487 |
+
|
| 488 |
+
def update(self, c: Cycle):
|
| 489 |
+
self.t += c.emitted
|
| 490 |
+
if self.lam <= AR_LEVEL and c.labels:
|
| 491 |
+
g = sum(c.labels) - self.target * len(c.labels)
|
| 492 |
+
self.n_draft += len(c.labels)
|
| 493 |
+
new = self.lam - self.eta * g
|
| 494 |
+
if new < self.lam_min: # plafond côté agressif : crédit (U)
|
| 495 |
+
self.ext -= self.lam_min - new
|
| 496 |
+
new = self.lam_min
|
| 497 |
+
self.lam = new
|
| 498 |
+
if self.ra is not None:
|
| 499 |
+
if c.stop_ahat is not None:
|
| 500 |
+
self._buf.append((self.t, c.stop_ahat, c.stop_a))
|
| 501 |
+
self._since += c.emitted
|
| 502 |
+
if self._since >= self.ra["every"]:
|
| 503 |
+
self._since = 0
|
| 504 |
+
self._reanchor()
|
| 505 |
+
|
| 506 |
+
def _reanchor(self):
|
| 507 |
+
ra = self._buf = [b for b in self._buf if b[0] > self.t - self.ra["horizon"]]
|
| 508 |
+
if len(ra) < self.ra["n_min"]:
|
| 509 |
+
return
|
| 510 |
+
Ah = np.array([b[1] for b in ra])
|
| 511 |
+
Aa = np.array([b[2] for b in ra])
|
| 512 |
+
best = None
|
| 513 |
+
for lam in self.ra["grid"]: # du prudent vers l'agressif
|
| 514 |
+
ok = Ah >= self._taus(lam)[None, :]
|
| 515 |
+
ex = np.where(ok.any(1), ok.argmax(1), -1)
|
| 516 |
+
sel = ex >= 0
|
| 517 |
+
n = int(sel.sum())
|
| 518 |
+
if n < self.ra["n_min"]:
|
| 519 |
+
continue
|
| 520 |
+
if Aa[sel, ex[sel]].mean() < self.target + self.ra["margin"]:
|
| 521 |
+
break
|
| 522 |
+
best = lam
|
| 523 |
+
if best is None:
|
| 524 |
+
return
|
| 525 |
+
if best < min(self.lam, AR_LEVEL) - self.ra["min_jump"] or (self.lam > AR_LEVEL >= best):
|
| 526 |
+
before = self.lam
|
| 527 |
+
self.move(best)
|
| 528 |
+
if self.lam < before:
|
| 529 |
+
self.n_jumps += 1
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
class ACIPerHead(Controller):
|
| 533 |
+
"""Un intégrateur par tête : lambda_k <- lambda_k - eta * sum_{j sortis à k}(a_j - eps).
|
| 534 |
+
|
| 535 |
+
Garantie par tête (déficit de la tête k <= g_min + (1 - lambda_k,1)/eta), donc aussi au
|
| 536 |
+
niveau système (somme). Pas de ré-ancrage dans cette variante.
|
| 537 |
+
"""
|
| 538 |
+
|
| 539 |
+
def __init__(self, eps, eta, lam0, lam_min=0.0, K=4):
|
| 540 |
+
self.eps, self.eta, self.lam_min = eps, eta, lam_min
|
| 541 |
+
self.lam = np.full(K, float(lam0))
|
| 542 |
+
|
| 543 |
+
def policy(self):
|
| 544 |
+
return self.lam.copy(), 0.0, bool(np.all(self.lam > AR_LEVEL))
|
| 545 |
+
|
| 546 |
+
def update(self, c: Cycle):
|
| 547 |
+
frozen = self.lam > AR_LEVEL
|
| 548 |
+
g = np.zeros(len(self.lam))
|
| 549 |
+
for k, a in zip(c.heads, c.labels):
|
| 550 |
+
g[k] += a - self.eps
|
| 551 |
+
self.lam = np.where(frozen, self.lam, np.maximum(self.lam_min, self.lam - self.eta * g))
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
class Deadband(Controller):
|
| 555 |
+
"""Hybride LTT + garde : lambda = lam_ltt + eta * (-B - W)^+, W = marge cumulée.
|
| 556 |
+
|
| 557 |
+
Tant que la marge réalisée W = sum (a - eps) reste >= -B, on garde exactement la
|
| 558 |
+
configuration certifiée hors ligne ; au-delà, on devient plus prudent, jusqu'au
|
| 559 |
+
mode AR. Plafond anti-saturation : W <= w_cap.
|
| 560 |
+
"""
|
| 561 |
+
|
| 562 |
+
def __init__(self, eps, eta, lam_ltt, B, w_cap=None, gamma=0.0):
|
| 563 |
+
self.eps, self.eta, self.lam_ltt, self.B, self.gamma = eps, eta, lam_ltt, B, gamma
|
| 564 |
+
self.w_cap = B if w_cap is None else w_cap
|
| 565 |
+
self.W = 0.0
|
| 566 |
+
|
| 567 |
+
@property
|
| 568 |
+
def lam(self):
|
| 569 |
+
return self.lam_ltt + self.eta * max(0.0, -self.B - self.W)
|
| 570 |
+
|
| 571 |
+
def policy(self):
|
| 572 |
+
lam = self.lam
|
| 573 |
+
return np.full(self.K, lam), self.gamma, lam > AR_LEVEL
|
| 574 |
+
|
| 575 |
+
def update(self, c: Cycle):
|
| 576 |
+
self.W = min(self.w_cap, self.W + sum(c.labels) - self.eps * len(c.labels))
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
class DraftVerify(Controller):
|
| 580 |
+
"""Heuristique de Draft & Verify (Zhang et al. 2024, §3.4, éq. (2)–(4)), sur lambda.
|
| 581 |
+
|
| 582 |
+
AR <- b1 AR + (1 - b1) AR_e (AR_e = acceptés / rédigés du cycle, gaspillés compris) ;
|
| 583 |
+
lam~ = lam + s si AR <= alpha, lam - s sinon ; lam <- b2 lam + (1 - b2) lam~.
|
| 584 |
+
Défauts de l'article : s = 0,01, b1 = 0,5, b2 = 0,9. Aucune garantie.
|
| 585 |
+
"""
|
| 586 |
+
|
| 587 |
+
def __init__(self, alpha, lam0, s=0.01, b1=0.5, b2=0.9, prefix=False):
|
| 588 |
+
self.alpha, self.lam, self.s, self.b1, self.b2 = alpha, lam0, s, b1, b2
|
| 589 |
+
self.prefix = prefix # True : AR_e calculé sur le préfixe vérifié (variante)
|
| 590 |
+
self.ar = None
|
| 591 |
+
|
| 592 |
+
def policy(self):
|
| 593 |
+
return np.full(self.K, self.lam), 0.0, False # jamais en mode AR
|
| 594 |
+
|
| 595 |
+
def update(self, c: Cycle):
|
| 596 |
+
if self.prefix:
|
| 597 |
+
if not c.labels:
|
| 598 |
+
return
|
| 599 |
+
ar_e = sum(c.labels) / len(c.labels)
|
| 600 |
+
else:
|
| 601 |
+
if c.drafted == 0:
|
| 602 |
+
return
|
| 603 |
+
ar_e = c.accepted / c.drafted
|
| 604 |
+
self.ar = ar_e if self.ar is None else self.b1 * self.ar + (1 - self.b1) * ar_e
|
| 605 |
+
tilde = self.lam + self.s if self.ar <= self.alpha else self.lam - self.s
|
| 606 |
+
self.lam = min(1.0, self.b2 * self.lam + (1 - self.b2) * tilde)
|
| 607 |
+
|
| 608 |
+
|
| 609 |
+
class NoSlowdown(Controller):
|
| 610 |
+
"""Contrôle du coût par token émis sur gamma (politique guidée par le coût).
|
| 611 |
+
|
| 612 |
+
g_c = (1 + xi) * emitted - cost = xi > 0 en mode AR (coût 1 par token, sans sonde) :
|
| 613 |
+
l'état AR n'est pas absorbant. gamma <- max(gamma_min, gamma - eta_g * g_c) ;
|
| 614 |
+
gamma > 1 : AR. Garantie (Prop. 4) : coût total <= (1 + xi) * tokens + borne.
|
| 615 |
+
Se compose avec un ACI sur lambda (sécurité sur l'acceptation).
|
| 616 |
+
"""
|
| 617 |
+
|
| 618 |
+
def __init__(self, xi, eta_g, gamma0, aci: Controller, gamma_min=0.0):
|
| 619 |
+
self.xi, self.eta_g, self.gamma, self.gamma_min, self.aci = xi, eta_g, gamma0, gamma_min, aci
|
| 620 |
+
|
| 621 |
+
@property
|
| 622 |
+
def probe(self):
|
| 623 |
+
# en mode AR dû à gamma, aucune sonde : un pas AR doit coûter exactement 1
|
| 624 |
+
return self.aci.probe and self.gamma <= 1.0
|
| 625 |
+
|
| 626 |
+
def policy(self):
|
| 627 |
+
self.aci.K = self.K
|
| 628 |
+
lam, _, ar = self.aci.policy()
|
| 629 |
+
return lam, self.gamma, ar or self.gamma > 1.0
|
| 630 |
+
|
| 631 |
+
def update(self, c: Cycle):
|
| 632 |
+
if not (c.ar and self.aci.lam <= AR_LEVEL): # cycle AR dû à gamma : lambda inchangé
|
| 633 |
+
self.aci.update(c)
|
| 634 |
+
else:
|
| 635 |
+
self.aci.t = getattr(self.aci, "t", 0) + c.emitted
|
| 636 |
+
g = (1 + self.xi) * c.emitted - c.cost
|
| 637 |
+
self.gamma = max(self.gamma_min, self.gamma - self.eta_g * g)
|
| 638 |
+
|
| 639 |
+
|
| 640 |
+
# ---------------------------------------------------------------------------
|
| 641 |
+
# Rejeu du flot de contrôle de DSSD (régime A)
|
| 642 |
+
# ---------------------------------------------------------------------------
|
| 643 |
+
|
| 644 |
+
@dataclass
|
| 645 |
+
class Trace:
|
| 646 |
+
"""Statistiques par cycle d'un rejeu."""
|
| 647 |
+
seq: list = field(default_factory=list) # indice de flux de la séquence
|
| 648 |
+
n_pf: list = field(default_factory=list) # brouillons sur le préfixe de sortie
|
| 649 |
+
s_pf: list = field(default_factory=list) # somme des étiquettes (a ou beta)
|
| 650 |
+
s_real: list = field(default_factory=list) # acceptations réalisées sur le préfixe
|
| 651 |
+
drafted: list = field(default_factory=list)
|
| 652 |
+
emitted: list = field(default_factory=list)
|
| 653 |
+
cost: list = field(default_factory=list)
|
| 654 |
+
lam: list = field(default_factory=list)
|
| 655 |
+
|
| 656 |
+
def arrays(self):
|
| 657 |
+
return {k: np.asarray(v, float) for k, v in self.__dict__.items()}
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def replay(Ahat, A, mask, ctrl: Controller, *, kind="binary", label="rb", Lmax=16,
|
| 661 |
+
cost=CostModel(), rng=None, rule="first") -> Trace:
|
| 662 |
+
"""Rejoue DSSD sur un flux de séquences (lignes de A, dans l'ordre).
|
| 663 |
+
|
| 664 |
+
En régime A, chaque position de sortie est évaluée une fois sur son vrai préfixe
|
| 665 |
+
(Prop. 1 du chantier 3) : on lit donc â et A aux positions de sortie. Les
|
| 666 |
+
brouillons émis après un rejet (sur un préfixe faux) ne sont que comptés
|
| 667 |
+
(approximation : on prolonge avec les scores des positions suivantes).
|
| 668 |
+
kind="binary" : A est l'indicatrice d'accord (greedy) ;
|
| 669 |
+
kind="tv" : A = beta = 1 - TV, acceptation réalisée ~ Bernoulli(beta) ;
|
| 670 |
+
label="rb" donne beta au contrôleur, label="real" l'indicatrice.
|
| 671 |
+
Têtes autorisées : seuil fini (tau_k = +inf exclut la tête : jamais évaluée).
|
| 672 |
+
rule="first" : sortie à la première tête avec â_k >= tau_k ; si la survie
|
| 673 |
+
cum * â_k < gamma, le bloc s'arrête.
|
| 674 |
+
rule="h" : politique (h) du chantier 1 : sortie à la première tête avec
|
| 675 |
+
â_k >= tau_k ET cum * â_k >= gamma ; sinon on évalue la tête autorisée
|
| 676 |
+
suivante ; arrêt du bloc si aucune ne sort.
|
| 677 |
+
L_max : attribut `lmax` du contrôleur s'il existe, sinon l'argument Lmax.
|
| 678 |
+
"""
|
| 679 |
+
rng = np.random.default_rng(rng)
|
| 680 |
+
tr = Trace()
|
| 681 |
+
K = A.shape[-1]
|
| 682 |
+
ctrl.K = K
|
| 683 |
+
lengths = mask.sum(1)
|
| 684 |
+
for i in range(A.shape[0]):
|
| 685 |
+
ah_i, a_i, Ti = Ahat[i], A[i], int(lengths[i])
|
| 686 |
+
ctrl.begin_sequence(i)
|
| 687 |
+
t = 0
|
| 688 |
+
while t < Ti:
|
| 689 |
+
lam, gamma, ar = ctrl.policy()
|
| 690 |
+
allowed = np.isfinite(lam)
|
| 691 |
+
idx = np.nonzero(allowed)[0]
|
| 692 |
+
lmax = getattr(ctrl, "lmax", None) or Lmax
|
| 693 |
+
tr.lam.append(float(np.min(lam)))
|
| 694 |
+
if ar or len(idx) == 0:
|
| 695 |
+
c = Cycle([], [], 0, 0, 1, 1.0, ar=True)
|
| 696 |
+
if ctrl.probe:
|
| 697 |
+
c.stop_ahat, c.stop_a = ah_i[t], a_i[t]
|
| 698 |
+
c.cost += cost.probe(K, allowed)
|
| 699 |
+
_record(tr, i, c, 0.0)
|
| 700 |
+
ctrl.update(c)
|
| 701 |
+
t += 1
|
| 702 |
+
continue
|
| 703 |
+
j, surv, drafted, stop_full, ccost = t, 1.0, [], False, 0.0
|
| 704 |
+
k_last = None # tête la plus profonde évaluée sur la dernière ligne (None : aucune)
|
| 705 |
+
while j < Ti and len(drafted) < lmax:
|
| 706 |
+
k_exit, evaluated, full = -1, [], True
|
| 707 |
+
for k in idx:
|
| 708 |
+
evaluated.append(k)
|
| 709 |
+
if ah_i[j, k] >= lam[k]:
|
| 710 |
+
if surv * ah_i[j, k] >= gamma:
|
| 711 |
+
k_exit = k
|
| 712 |
+
break
|
| 713 |
+
if rule == "first": # arrêt du bloc par la survie
|
| 714 |
+
full = False
|
| 715 |
+
break
|
| 716 |
+
deepest = evaluated[-1]
|
| 717 |
+
ccost += cost.position(deepest, evaluated, k_exit >= 0)
|
| 718 |
+
if k_exit < 0:
|
| 719 |
+
stop_full = full
|
| 720 |
+
k_last = deepest
|
| 721 |
+
break
|
| 722 |
+
drafted.append((j, k_exit))
|
| 723 |
+
surv *= ah_i[j, k_exit]
|
| 724 |
+
j += 1
|
| 725 |
+
rows = [k for _, k in drafted]
|
| 726 |
+
if k_last is not None:
|
| 727 |
+
rows.append(k_last)
|
| 728 |
+
elif j < Ti: # L_max atteint : dernière ligne sans aucune couche
|
| 729 |
+
rows.append(None)
|
| 730 |
+
ccost += cost.verify_rows(rows, allowed)
|
| 731 |
+
heads, labels, s_real, m = [], [], 0, None
|
| 732 |
+
for jdx, (jj, k) in enumerate(drafted):
|
| 733 |
+
a = float(a_i[jj, k])
|
| 734 |
+
acc = a >= 1.0 if kind == "binary" else (rng.random() < a)
|
| 735 |
+
heads.append(k)
|
| 736 |
+
labels.append(a if (kind == "binary" or label == "rb") else float(acc))
|
| 737 |
+
s_real += int(acc)
|
| 738 |
+
if not acc:
|
| 739 |
+
m = jdx
|
| 740 |
+
break
|
| 741 |
+
n_ok = len(drafted) if m is None else m
|
| 742 |
+
nxt = min(Ti, t + n_ok + 1)
|
| 743 |
+
c = Cycle(heads, labels, n_ok, len(drafted), nxt - t, ccost)
|
| 744 |
+
if m is None and stop_full and j < Ti:
|
| 745 |
+
c.stop_ahat, c.stop_a = ah_i[j], a_i[j]
|
| 746 |
+
_record(tr, i, c, s_real)
|
| 747 |
+
ctrl.update(c)
|
| 748 |
+
t = nxt
|
| 749 |
+
return tr
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def _record(tr, i, c, s_real):
|
| 753 |
+
tr.seq.append(i)
|
| 754 |
+
tr.n_pf.append(len(c.labels))
|
| 755 |
+
tr.s_pf.append(sum(c.labels))
|
| 756 |
+
tr.s_real.append(s_real)
|
| 757 |
+
tr.drafted.append(c.drafted)
|
| 758 |
+
tr.emitted.append(c.emitted)
|
| 759 |
+
tr.cost.append(c.cost)
|
| 760 |
+
|
| 761 |
+
|
| 762 |
+
def aci_bound(eps, eta, lam0, g_min, J=0.0):
|
| 763 |
+
"""Borne déterministe du Théorème 1 : W_C >= -aci_bound(...).
|
| 764 |
+
|
| 765 |
+
g_min : perte maximale d'un cycle (eps en greedy ou avec étiquettes réalisées,
|
| 766 |
+
eps * Lmax avec beta) ; J : budget de déplacements externes consommable (tokens),
|
| 767 |
+
J0 + rho * N_C pour le ré-ancrage.
|
| 768 |
+
"""
|
| 769 |
+
return max(0.0, AR_LEVEL - lam0) / eta + g_min + J
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
def summarize(tr: Trace, eps, segments=None):
|
| 773 |
+
"""alpha_pf, alpha réalisé, alpha tous brouillons, couverture, speedup modélisé,
|
| 774 |
+
déficit maximal min_c W_c (W = sum (s - eps n)), globalement et par segment."""
|
| 775 |
+
d = tr.arrays()
|
| 776 |
+
|
| 777 |
+
def block(sel):
|
| 778 |
+
n, s, sr = d["n_pf"][sel].sum(), d["s_pf"][sel].sum(), d["s_real"][sel].sum()
|
| 779 |
+
em, co, dr = d["emitted"][sel].sum(), d["cost"][sel].sum(), d["drafted"][sel].sum()
|
| 780 |
+
return dict(alpha_pf=s / n if n else float("nan"), alpha_real=sr / n if n else float("nan"),
|
| 781 |
+
alpha_all=sr / dr if dr else float("nan"), coverage=n / em if em else 0.0,
|
| 782 |
+
speedup=em / co if co else float("nan"), n_draft=int(n), tokens=int(em))
|
| 783 |
+
|
| 784 |
+
out = {"all": block(slice(None))}
|
| 785 |
+
W = np.cumsum(d["s_pf"] - eps * d["n_pf"])
|
| 786 |
+
out["all"]["min_W"] = float(min(0.0, W.min())) if len(W) else 0.0
|
| 787 |
+
if segments:
|
| 788 |
+
seq = d["seq"]
|
| 789 |
+
for name, (lo, hi) in segments.items():
|
| 790 |
+
out[name] = block((seq >= lo) & (seq < hi))
|
| 791 |
+
return out
|
dssd/decode.py
ADDED
|
@@ -0,0 +1,495 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Autoregressive and self-speculative decoding on a LayeredDecoder.
|
| 2 |
+
|
| 3 |
+
One cycle of self-speculative decoding:
|
| 4 |
+
|
| 5 |
+
1. Drafting. The last committed token (not yet cached) and then each draft
|
| 6 |
+
token climb the layers; at every head allowed by the policy we ask the
|
| 7 |
+
policy whether to exit. An exit yields a draft token from that head; the
|
| 8 |
+
K/V of the deeper drafting layers are filled from the exit hidden state
|
| 9 |
+
(CALM-style copy) so later draft tokens can attend to it. A token with no
|
| 10 |
+
exit, the draft budget, or a drafted EOS ends the draft.
|
| 11 |
+
2. Verification. Every row of the cycle has passed through layer c_min (the
|
| 12 |
+
shallowest exit the policy can use) with exact inputs, so its hidden state
|
| 13 |
+
there is exact. One pass runs layers [c_min, L) over all rows, overwriting
|
| 14 |
+
the approximate K/V, and yields the full-model distribution at each row.
|
| 15 |
+
3. Acceptance. Standard speculative sampling (Leviathan et al. 2023): accept
|
| 16 |
+
draft x with prob min(1, p(x)/q(x)), else resample from norm(max(p-q, 0));
|
| 17 |
+
if all drafts are accepted, sample a bonus token from the last row. At T=0
|
| 18 |
+
this is exact argmax matching. The output law (greedy: the output) equals
|
| 19 |
+
the full model's.
|
| 20 |
+
|
| 21 |
+
Cache invariant between cycles: positions [0, len(seq)-1) are exact in every
|
| 22 |
+
layer; the last committed token is processed at the next cycle.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import bisect
|
| 28 |
+
import time
|
| 29 |
+
from dataclasses import dataclass, field
|
| 30 |
+
|
| 31 |
+
import torch
|
| 32 |
+
import torch.nn.functional as F
|
| 33 |
+
|
| 34 |
+
from .heads import ExitHead, entropy, head_draft_distribution
|
| 35 |
+
from .layered import LayeredDecoder
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# ---------------------------------------------------------------------- policies
|
| 39 |
+
class ExitPolicy:
|
| 40 |
+
"""Which heads may be tried, in which order, and when to exit."""
|
| 41 |
+
|
| 42 |
+
head_order: list[int] # indices into `heads`, shallow to deep
|
| 43 |
+
max_draft: int
|
| 44 |
+
|
| 45 |
+
def exit(self, k: int, logits: torch.Tensor, n_drafted: int) -> bool:
|
| 46 |
+
raise NotImplementedError
|
| 47 |
+
|
| 48 |
+
def decide(self, k: int, conf: float, n_drafted: int) -> bool:
|
| 49 |
+
"""Exit decision from the head confidence (graph engine)."""
|
| 50 |
+
raise NotImplementedError
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class StaticPolicy(ExitPolicy):
|
| 54 |
+
"""Always exit at head k; draft exactly `length` tokens (LayerSkip-like)."""
|
| 55 |
+
|
| 56 |
+
def __init__(self, k: int, length: int):
|
| 57 |
+
self.head_order = [k]
|
| 58 |
+
self.max_draft = length
|
| 59 |
+
|
| 60 |
+
def exit(self, k, logits, n_drafted):
|
| 61 |
+
return True
|
| 62 |
+
|
| 63 |
+
def decide(self, k, conf, n_drafted):
|
| 64 |
+
return True
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class ThresholdPolicy(ExitPolicy):
|
| 68 |
+
"""Exit at the first head (in `head_order`) whose score passes its threshold.
|
| 69 |
+
|
| 70 |
+
score: "maxprob" (max over V_k ∪ ⊥, 0 if ⊥ wins), "neg_entropy", or
|
| 71 |
+
"one_minus_reject". With a single head this is the Kangaroo rule; with all
|
| 72 |
+
heads it is DSSD.
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
def __init__(self, heads: list[ExitHead], thresholds: dict[int, float],
|
| 76 |
+
max_draft: int = 32, score: str = "maxprob", cum_gamma: float = 0.0,
|
| 77 |
+
acc_maps: dict | None = None):
|
| 78 |
+
self.heads = heads
|
| 79 |
+
# Optional per-head calibrated acceptance a_hat_k(conf) (isotonic, piecewise linear
|
| 80 |
+
# knots/values): thresholds and the cumulative product then apply to a_hat.
|
| 81 |
+
self.acc_maps = {int(k): (list(v[0]), list(v[1])) for k, v in (acc_maps or {}).items()}
|
| 82 |
+
self.thresholds = thresholds
|
| 83 |
+
self.head_order = sorted(thresholds)
|
| 84 |
+
self.max_draft = max_draft
|
| 85 |
+
self.score = score
|
| 86 |
+
# Cumulative stop (g): a draft token is only emitted while the product of the
|
| 87 |
+
# exit confidences of the block stays >= cum_gamma (proxy for the probability
|
| 88 |
+
# that the whole block survives verification, cf. SpecDec++).
|
| 89 |
+
self.cum_gamma = cum_gamma
|
| 90 |
+
self._cum = 1.0
|
| 91 |
+
|
| 92 |
+
def start_block(self) -> None:
|
| 93 |
+
self._cum = 1.0
|
| 94 |
+
|
| 95 |
+
def _cum_ok(self, conf: float, k: int | None = None) -> bool:
|
| 96 |
+
# cum_gamma: a scalar, or a per-head dict {k: gamma_k} (F7 option B: the exit test
|
| 97 |
+
# at head k is cum * a_hat_k >= gamma_k)
|
| 98 |
+
g = self.cum_gamma.get(k, 0.0) if isinstance(self.cum_gamma, dict) else self.cum_gamma
|
| 99 |
+
if g <= 0.0:
|
| 100 |
+
self._cum *= conf
|
| 101 |
+
return True
|
| 102 |
+
if self._cum * conf < g:
|
| 103 |
+
return False
|
| 104 |
+
self._cum *= conf
|
| 105 |
+
return True
|
| 106 |
+
|
| 107 |
+
def exit(self, k, logits, n_drafted):
|
| 108 |
+
head = self.heads[k]
|
| 109 |
+
if self.score == "maxprob":
|
| 110 |
+
_, _, s = head_draft_distribution(logits, head.reject, 1.0)
|
| 111 |
+
elif self.score == "neg_entropy":
|
| 112 |
+
s = -entropy(logits)
|
| 113 |
+
if head.reject and int(logits.argmax(-1)) == logits.shape[-1] - 1:
|
| 114 |
+
return False
|
| 115 |
+
elif self.score == "one_minus_reject":
|
| 116 |
+
_, p_rej, s = head_draft_distribution(logits, head.reject, 1.0)
|
| 117 |
+
s = 1.0 - p_rej
|
| 118 |
+
else:
|
| 119 |
+
raise ValueError(self.score)
|
| 120 |
+
return float(s) >= self.thresholds[k] and self._cum_ok(float(s), k)
|
| 121 |
+
|
| 122 |
+
def _ahat(self, k, conf):
|
| 123 |
+
if k not in self.acc_maps:
|
| 124 |
+
return conf
|
| 125 |
+
xs, ys = self.acc_maps[k]
|
| 126 |
+
if conf <= xs[0]:
|
| 127 |
+
return ys[0]
|
| 128 |
+
if conf >= xs[-1]:
|
| 129 |
+
return ys[-1]
|
| 130 |
+
j = bisect.bisect_right(xs, conf)
|
| 131 |
+
x0, x1, y0, y1 = xs[j - 1], xs[j], ys[j - 1], ys[j]
|
| 132 |
+
return y0 + (y1 - y0) * (conf - x0) / (x1 - x0 if x1 > x0 else 1.0)
|
| 133 |
+
|
| 134 |
+
def decide(self, k, conf, n_drafted):
|
| 135 |
+
if conf <= 0.0 and self.heads[k].reject: # ⊥ won: never draft (T4: gamma_k > a_hat(⊥))
|
| 136 |
+
return False
|
| 137 |
+
a = self._ahat(k, conf)
|
| 138 |
+
return a >= self.thresholds[k] and self._cum_ok(a, k)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
class ScriptedPolicy(ExitPolicy):
|
| 142 |
+
"""Testing: draft token i exits at head schedule[i % len] (None = no exit)."""
|
| 143 |
+
|
| 144 |
+
def __init__(self, n_heads: int, schedule: list, max_draft: int = 15):
|
| 145 |
+
self.head_order = list(range(n_heads))
|
| 146 |
+
self.schedule = schedule
|
| 147 |
+
self.max_draft = max_draft
|
| 148 |
+
|
| 149 |
+
def _want(self, n_drafted):
|
| 150 |
+
return self.schedule[n_drafted % len(self.schedule)]
|
| 151 |
+
|
| 152 |
+
def exit(self, k, logits, n_drafted):
|
| 153 |
+
return self._want(n_drafted) == k
|
| 154 |
+
|
| 155 |
+
def decide(self, k, conf, n_drafted):
|
| 156 |
+
return self._want(n_drafted) == k
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
class AdaptiveThresholdPolicy(ThresholdPolicy):
|
| 160 |
+
"""Single head, threshold adapted online toward a target acceptance rate
|
| 161 |
+
(Draft & Verify / AdaEDL style): after each cycle, raise the threshold if the
|
| 162 |
+
cycle's acceptance fell below `target`, lower it otherwise."""
|
| 163 |
+
|
| 164 |
+
def __init__(self, heads, k: int, gamma0: float, target: float, step: float = 0.01,
|
| 165 |
+
max_draft: int = 15):
|
| 166 |
+
super().__init__(heads, {k: gamma0}, max_draft=max_draft, score="maxprob")
|
| 167 |
+
self.k, self.target, self.step = k, target, step
|
| 168 |
+
|
| 169 |
+
def update(self, n_drafted: int, n_accepted: int) -> None:
|
| 170 |
+
if n_drafted == 0:
|
| 171 |
+
return
|
| 172 |
+
g = self.thresholds[self.k] + (self.step if n_accepted / n_drafted < self.target else -self.step)
|
| 173 |
+
self.thresholds[self.k] = min(0.999, max(0.0, g))
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
# ---------------------------------------------------------------------- stats
|
| 177 |
+
@dataclass
|
| 178 |
+
class GenStats:
|
| 179 |
+
new_tokens: int = 0
|
| 180 |
+
cycles: int = 0 # full-model passes after prefill
|
| 181 |
+
drafted: int = 0
|
| 182 |
+
accepted: int = 0
|
| 183 |
+
empty_drafts: int = 0 # cycles whose draft had length 0 (fallback)
|
| 184 |
+
rejected_cycles: int = 0 # cycles ended by a rejected draft
|
| 185 |
+
ar_steps: int = 0 # online control: plain AR steps (lambda > 1, no probe)
|
| 186 |
+
lam_start: float = float("nan") # online control: lambda at the start / end of the request
|
| 187 |
+
lam_end: float = float("nan")
|
| 188 |
+
gamma_start: float = float("nan") # cost safety (NoSlowdown): gamma at the start / end
|
| 189 |
+
gamma_end: float = float("nan")
|
| 190 |
+
ar_step_median: float = float("nan") # AR runs: median per-token step time
|
| 191 |
+
lab_n: int = 0 # online control: labels sent to the controller (and their sum)
|
| 192 |
+
lab_s: float = 0.0
|
| 193 |
+
n_jumps: int = -1 # online control: re-anchoring jumps of the ACI (cumulative)
|
| 194 |
+
pld_drafted: int = 0 # prompt lookup: copied drafts / accepted copied drafts
|
| 195 |
+
pld_accepted: int = 0
|
| 196 |
+
cost_units: float = 0.0 # online control: sum of cycle costs (1 = one AR step)
|
| 197 |
+
h_max: float = float("-inf") # max over drafting cycles of cost - (1 + xi) * emitted
|
| 198 |
+
excess_end: float = 0.0 # sum over the request of cost - (1 + xi) * emitted
|
| 199 |
+
excess_max: float = float("-inf") # running maximum of that sum within the request
|
| 200 |
+
exits: dict = field(default_factory=dict) # head index -> count
|
| 201 |
+
sources: list = field(default_factory=list) # per output token: copy/head/star/draft/model
|
| 202 |
+
head_evals: int = 0
|
| 203 |
+
layer_rows: int = 0
|
| 204 |
+
kv_fill_rows: int = 0
|
| 205 |
+
vocab_proj_rows: int = 0
|
| 206 |
+
seconds: float = 0.0
|
| 207 |
+
prefill_seconds: float = 0.0
|
| 208 |
+
|
| 209 |
+
@property
|
| 210 |
+
def tau(self) -> float:
|
| 211 |
+
"""Tokens committed per full-model pass (incl. correction/bonus)."""
|
| 212 |
+
return self.new_tokens / max(self.cycles, 1)
|
| 213 |
+
|
| 214 |
+
def as_dict(self) -> dict:
|
| 215 |
+
d = dict(self.__dict__)
|
| 216 |
+
d.pop("sources") # per-token list: kept out of the bench jsonl
|
| 217 |
+
d["tau"] = self.tau
|
| 218 |
+
d["acceptance"] = self.accepted / max(self.drafted, 1)
|
| 219 |
+
# drafts emitted on the output prefix = accepted + one rejected per rejecting cycle
|
| 220 |
+
d["acceptance_prefix"] = self.accepted / max(self.accepted + self.rejected_cycles, 1)
|
| 221 |
+
d["tokens_per_s"] = self.new_tokens / max(self.seconds, 1e-9)
|
| 222 |
+
return d
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def _sync(device) -> None:
|
| 226 |
+
if device.type == "cuda":
|
| 227 |
+
torch.cuda.synchronize(device)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _sample(probs: torch.Tensor, gen: torch.Generator | None) -> int:
|
| 231 |
+
return int(torch.multinomial(probs, 1, generator=gen))
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _target_token(logits: torch.Tensor, temperature: float, gen) -> int:
|
| 235 |
+
if temperature == 0.0:
|
| 236 |
+
return int(logits.argmax())
|
| 237 |
+
return _sample(F.softmax(logits / temperature, dim=-1), gen)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
# ---------------------------------------------------------------------- decoders
|
| 241 |
+
@torch.no_grad()
|
| 242 |
+
def generate_ar(dec: LayeredDecoder, prompt: list[int], max_new: int,
|
| 243 |
+
temperature: float = 0.0, eos_ids: set[int] | None = None,
|
| 244 |
+
gen: torch.Generator | None = None) -> tuple[list[int], GenStats]:
|
| 245 |
+
eos_ids = eos_ids or set()
|
| 246 |
+
st = GenStats()
|
| 247 |
+
dec.reset_counters()
|
| 248 |
+
ids = torch.tensor(prompt, device=dec.device)
|
| 249 |
+
_sync(dec.device)
|
| 250 |
+
t0 = time.perf_counter()
|
| 251 |
+
logits = dec.prefill(ids)
|
| 252 |
+
_sync(dec.device)
|
| 253 |
+
st.prefill_seconds = time.perf_counter() - t0
|
| 254 |
+
out = [_target_token(logits, temperature, gen)]
|
| 255 |
+
t1 = time.perf_counter()
|
| 256 |
+
while len(out) < max_new and out[-1] not in eos_ids:
|
| 257 |
+
pos = len(prompt) + len(out) - 1
|
| 258 |
+
logits = dec.full_step(torch.tensor([out[-1]], device=dec.device), pos)[0]
|
| 259 |
+
out.append(_target_token(logits, temperature, gen))
|
| 260 |
+
st.cycles += 1
|
| 261 |
+
_sync(dec.device)
|
| 262 |
+
st.seconds = time.perf_counter() - t1 + st.prefill_seconds
|
| 263 |
+
st.new_tokens = len(out)
|
| 264 |
+
st.layer_rows = dec.counters["layer_rows"]
|
| 265 |
+
st.vocab_proj_rows = dec.counters["vocab_proj_rows"]
|
| 266 |
+
return out, st
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
@torch.no_grad()
|
| 270 |
+
def generate_spec(dec: LayeredDecoder, heads: list[ExitHead], head_layers: list[int],
|
| 271 |
+
policy: ExitPolicy, prompt: list[int], max_new: int,
|
| 272 |
+
temperature: float = 0.0, eos_ids: set[int] | None = None,
|
| 273 |
+
gen: torch.Generator | None = None,
|
| 274 |
+
kv_fill: str = "lazy", trace: list | None = None) -> tuple[list[int], GenStats]:
|
| 275 |
+
"""Self-speculative decoding; head k reads the hidden state after head_layers[k] layers.
|
| 276 |
+
|
| 277 |
+
kv_fill:
|
| 278 |
+
"lazy" regime A (reference): before a draft row climbs past the depth of
|
| 279 |
+
earlier rows of the block, those rows are caught up exactly (batched),
|
| 280 |
+
so every head score depends only on the exact prefix. Verification
|
| 281 |
+
then runs only layers [c_max, L).
|
| 282 |
+
"copy" regime B: CALM state copy of the K/V of skipped drafting layers;
|
| 283 |
+
verification runs [c_min, L).
|
| 284 |
+
"recompute" regime B variant: run the skipped drafting layers for exited tokens.
|
| 285 |
+
trace: if a list, receives (position, head, logits) for every head evaluation.
|
| 286 |
+
"""
|
| 287 |
+
eos_ids = eos_ids or set()
|
| 288 |
+
st = GenStats()
|
| 289 |
+
dec.reset_counters()
|
| 290 |
+
L = dec.spec.num_layers
|
| 291 |
+
order = policy.head_order
|
| 292 |
+
c_min = head_layers[order[0]]
|
| 293 |
+
c_max = head_layers[order[-1]]
|
| 294 |
+
ids = torch.tensor(prompt, device=dec.device)
|
| 295 |
+
|
| 296 |
+
_sync(dec.device)
|
| 297 |
+
t0 = time.perf_counter()
|
| 298 |
+
logits = dec.prefill(ids)
|
| 299 |
+
_sync(dec.device)
|
| 300 |
+
st.prefill_seconds = time.perf_counter() - t0
|
| 301 |
+
out = [_target_token(logits, temperature, gen)]
|
| 302 |
+
|
| 303 |
+
t1 = time.perf_counter()
|
| 304 |
+
while len(out) < max_new and out[-1] not in eos_ids:
|
| 305 |
+
pos0 = len(prompt) + len(out) - 1
|
| 306 |
+
inputs = [out[-1]]
|
| 307 |
+
h_min: list[torch.Tensor] = [] # hidden after c_min layers, per row
|
| 308 |
+
drafts: list[int] = []
|
| 309 |
+
q_rows: list[torch.Tensor | None] = []
|
| 310 |
+
budget = min(policy.max_draft, max_new - len(out))
|
| 311 |
+
if hasattr(policy, "start_block"):
|
| 312 |
+
policy.start_block()
|
| 313 |
+
|
| 314 |
+
if kv_fill == "lazy":
|
| 315 |
+
drafts, q_rows, H, verify_from = _draft_lazy(
|
| 316 |
+
dec, heads, head_layers, policy, out[-1], pos0, budget, temperature, gen,
|
| 317 |
+
eos_ids, st, trace, context=(prompt + out[:-1]) if trace is not None else None)
|
| 318 |
+
inputs = [out[-1]] + drafts
|
| 319 |
+
H = dec.run_layers(H, pos0, verify_from, L)
|
| 320 |
+
target = dec.final_logits(H)
|
| 321 |
+
st.cycles += 1
|
| 322 |
+
st.drafted += len(drafts)
|
| 323 |
+
if not drafts:
|
| 324 |
+
st.empty_drafts += 1
|
| 325 |
+
n_acc, committed = _accept(drafts, q_rows, target, temperature, gen)
|
| 326 |
+
st.accepted += n_acc
|
| 327 |
+
for t in committed:
|
| 328 |
+
out.append(t)
|
| 329 |
+
if t in eos_ids or len(out) >= max_new:
|
| 330 |
+
break
|
| 331 |
+
continue
|
| 332 |
+
|
| 333 |
+
# ---------------- drafting (regime B)
|
| 334 |
+
while True:
|
| 335 |
+
j = len(inputs) - 1
|
| 336 |
+
if len(drafts) >= budget or (drafts and drafts[-1] in eos_ids):
|
| 337 |
+
break # last input is processed by the verification pass
|
| 338 |
+
pos = pos0 + j
|
| 339 |
+
h = dec.run_layers(dec.embed(torch.tensor([inputs[j]], device=dec.device)),
|
| 340 |
+
pos, 0, c_min)
|
| 341 |
+
h_min.append(h)
|
| 342 |
+
depth = c_min
|
| 343 |
+
exited = None
|
| 344 |
+
for k in order:
|
| 345 |
+
if head_layers[k] > depth:
|
| 346 |
+
h = dec.run_layers(h, pos, depth, head_layers[k])
|
| 347 |
+
depth = head_layers[k]
|
| 348 |
+
logits_k = heads[k](h)[0]
|
| 349 |
+
st.head_evals += 1
|
| 350 |
+
if trace is not None:
|
| 351 |
+
trace.append((pos, k, logits_k.detach().clone()))
|
| 352 |
+
if policy.exit(k, logits_k, len(drafts)):
|
| 353 |
+
exited = k
|
| 354 |
+
break
|
| 355 |
+
if exited is None:
|
| 356 |
+
break
|
| 357 |
+
head = heads[exited]
|
| 358 |
+
q_k, _, _ = head_draft_distribution(logits_k, head.reject, temperature)
|
| 359 |
+
if temperature == 0.0:
|
| 360 |
+
tok_local = int(q_k.argmax())
|
| 361 |
+
q_full = None
|
| 362 |
+
else:
|
| 363 |
+
tok_local = _sample(q_k, gen)
|
| 364 |
+
q_full = head.scatter_full(q_k)
|
| 365 |
+
tok = int(head.to_token_ids(torch.tensor(tok_local, device=dec.device)))
|
| 366 |
+
if depth < c_max:
|
| 367 |
+
if kv_fill == "copy":
|
| 368 |
+
dec.fill_kv_from_hidden(h, pos, depth, c_max)
|
| 369 |
+
elif kv_fill == "recompute":
|
| 370 |
+
dec.run_layers(h, pos, depth, c_max)
|
| 371 |
+
else:
|
| 372 |
+
raise ValueError(kv_fill)
|
| 373 |
+
st.exits[exited] = st.exits.get(exited, 0) + 1
|
| 374 |
+
drafts.append(tok)
|
| 375 |
+
q_rows.append(q_full)
|
| 376 |
+
inputs.append(tok)
|
| 377 |
+
|
| 378 |
+
# ---------------- verification (one full-model pass)
|
| 379 |
+
if len(h_min) < len(inputs): # last draft token never entered the stack
|
| 380 |
+
j = len(inputs) - 1
|
| 381 |
+
h_min.append(dec.run_layers(
|
| 382 |
+
dec.embed(torch.tensor([inputs[j]], device=dec.device)), pos0 + j, 0, c_min))
|
| 383 |
+
H = torch.cat(h_min, dim=0)
|
| 384 |
+
H = dec.run_layers(H, pos0, c_min, L)
|
| 385 |
+
target = dec.final_logits(H) # (len(inputs), V)
|
| 386 |
+
st.cycles += 1
|
| 387 |
+
st.drafted += len(drafts)
|
| 388 |
+
if not drafts:
|
| 389 |
+
st.empty_drafts += 1
|
| 390 |
+
|
| 391 |
+
# ---------------- acceptance
|
| 392 |
+
n_acc, committed = _accept(drafts, q_rows, target, temperature, gen)
|
| 393 |
+
st.accepted += n_acc
|
| 394 |
+
|
| 395 |
+
for t in committed:
|
| 396 |
+
out.append(t)
|
| 397 |
+
if t in eos_ids or len(out) >= max_new:
|
| 398 |
+
break
|
| 399 |
+
|
| 400 |
+
_sync(dec.device)
|
| 401 |
+
st.seconds = time.perf_counter() - t1 + st.prefill_seconds
|
| 402 |
+
st.new_tokens = len(out)
|
| 403 |
+
st.layer_rows = dec.counters["layer_rows"]
|
| 404 |
+
st.kv_fill_rows = dec.counters["kv_fill_rows"]
|
| 405 |
+
st.vocab_proj_rows = dec.counters["vocab_proj_rows"]
|
| 406 |
+
return out, st
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _accept(drafts, q_rows, target, temperature, gen):
|
| 410 |
+
"""Speculative acceptance (Leviathan et al.): returns (#accepted, committed tokens)."""
|
| 411 |
+
committed: list[int] = []
|
| 412 |
+
for i, d in enumerate(drafts):
|
| 413 |
+
if temperature == 0.0:
|
| 414 |
+
if int(target[i].argmax()) != d:
|
| 415 |
+
committed.append(int(target[i].argmax()))
|
| 416 |
+
return len(committed) - 1, committed
|
| 417 |
+
else:
|
| 418 |
+
p = F.softmax(target[i] / temperature, dim=-1)
|
| 419 |
+
q = q_rows[i]
|
| 420 |
+
ratio = (p[d] / q[d].clamp_min(1e-20)).clamp(max=1.0)
|
| 421 |
+
u = torch.rand((), device=p.device, generator=gen)
|
| 422 |
+
if not bool(u < ratio):
|
| 423 |
+
resid = (p - q).clamp_min(0.0)
|
| 424 |
+
z = resid.sum()
|
| 425 |
+
committed.append(_sample(resid / z if z > 0 else p, gen))
|
| 426 |
+
return len(committed) - 1, committed
|
| 427 |
+
committed.append(d)
|
| 428 |
+
n_acc = len(committed)
|
| 429 |
+
committed.append(_target_token(target[len(drafts)], temperature, gen))
|
| 430 |
+
return n_acc, committed
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def _draft_lazy(dec, heads, head_layers, policy, first_token, pos0, budget, temperature, gen,
|
| 434 |
+
eos_ids, st, trace, context=None):
|
| 435 |
+
"""Regime-A drafting. Returns (drafts, q_rows, hidden rows at c_max, c_max).
|
| 436 |
+
|
| 437 |
+
With `trace`, records (position, head, logits, exact input context) per head evaluation.
|
| 438 |
+
"""
|
| 439 |
+
order = policy.head_order
|
| 440 |
+
c_min, c_max = head_layers[order[0]], head_layers[order[-1]]
|
| 441 |
+
rows: list[dict] = [] # {"pos", "h", "depth"} for every input row of the block
|
| 442 |
+
|
| 443 |
+
def advance(target: int) -> None:
|
| 444 |
+
while True:
|
| 445 |
+
lag = [r for r in rows if r["depth"] < target]
|
| 446 |
+
if not lag:
|
| 447 |
+
return
|
| 448 |
+
u = min(r["depth"] for r in lag)
|
| 449 |
+
v = min([r["depth"] for r in lag if r["depth"] > u] + [target])
|
| 450 |
+
batch = [r for r in lag if r["depth"] == u]
|
| 451 |
+
out = dec.run_layers(torch.cat([r["h"] for r in batch]), 0, u, v,
|
| 452 |
+
positions=[r["pos"] for r in batch])
|
| 453 |
+
for i, r in enumerate(batch):
|
| 454 |
+
r["h"], r["depth"] = out[i:i + 1], v
|
| 455 |
+
|
| 456 |
+
def new_row(tok: int, pos: int) -> dict:
|
| 457 |
+
r = {"pos": pos, "h": dec.embed(torch.tensor([tok], device=dec.device)), "depth": 0}
|
| 458 |
+
rows.append(r)
|
| 459 |
+
return r
|
| 460 |
+
|
| 461 |
+
drafts: list[int] = []
|
| 462 |
+
q_rows: list = []
|
| 463 |
+
tok = first_token
|
| 464 |
+
while True:
|
| 465 |
+
row = new_row(tok, pos0 + len(drafts))
|
| 466 |
+
if len(drafts) >= budget or (drafts and drafts[-1] in eos_ids):
|
| 467 |
+
break # trailing row: only verified
|
| 468 |
+
exited = None
|
| 469 |
+
for k in order:
|
| 470 |
+
advance(head_layers[k])
|
| 471 |
+
logits_k = heads[k](row["h"])[0]
|
| 472 |
+
st.head_evals += 1
|
| 473 |
+
if trace is not None:
|
| 474 |
+
ctx = context + [first_token] + drafts
|
| 475 |
+
trace.append((row["pos"], k, logits_k.detach().clone(), ctx))
|
| 476 |
+
if policy.exit(k, logits_k, len(drafts)):
|
| 477 |
+
exited = k
|
| 478 |
+
break
|
| 479 |
+
if exited is None:
|
| 480 |
+
break
|
| 481 |
+
head = heads[exited]
|
| 482 |
+
q_k, _, _ = head_draft_distribution(logits_k, head.reject, temperature)
|
| 483 |
+
if temperature == 0.0:
|
| 484 |
+
tok_local, q_full = int(q_k.argmax()), None
|
| 485 |
+
else:
|
| 486 |
+
tok_local = _sample(q_k, gen)
|
| 487 |
+
q_full = head.scatter_full(q_k)
|
| 488 |
+
tok = int(head.to_token_ids(torch.tensor(tok_local, device=dec.device)))
|
| 489 |
+
st.exits[exited] = st.exits.get(exited, 0) + 1
|
| 490 |
+
drafts.append(tok)
|
| 491 |
+
q_rows.append(q_full)
|
| 492 |
+
if hasattr(policy, "update_draft"):
|
| 493 |
+
policy.update_draft()
|
| 494 |
+
advance(c_max)
|
| 495 |
+
return drafts, q_rows, torch.cat([r["h"] for r in rows]), c_max
|
dssd/fast.py
ADDED
|
@@ -0,0 +1,1327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Graph-captured greedy decoding: AR baseline and self-speculative decoding.
|
| 2 |
+
|
| 3 |
+
All compute runs in captured CUDA/HIP graphs with static buffers:
|
| 4 |
+
ar token -> all layers -> LM head argmax (1 row)
|
| 5 |
+
seg[a, b, m] m hidden rows (buffer) -> layers [a, b) at positions pos[:m] (m rows)
|
| 6 |
+
head[k] one hidden row -> head k -> (confidence, token id)
|
| 7 |
+
fill[a, b] CALM K/V copy for layers [a, b) from one hidden row (regime B)
|
| 8 |
+
verify[c, n] n hidden rows at layer c -> layers [c, L) -> argmax (n rows)
|
| 9 |
+
Positions are read from a device buffer, so graphs replay anywhere. Attention
|
| 10 |
+
uses the row-invariant Triton kernel and GEMMs use rocBLAS, so AR and every
|
| 11 |
+
batched path compute each row bitwise identically (greedy parity).
|
| 12 |
+
|
| 13 |
+
Regime A (default, kv_fill="lazy"): before a draft row climbs to a deeper head,
|
| 14 |
+
the rows of the block that are shallower are caught up exactly in one batched
|
| 15 |
+
segment pass; head scores depend only on the exact prefix and verification runs
|
| 16 |
+
only [c_max, L). Regime B (kv_fill="copy"): K/V of skipped layers are copied from
|
| 17 |
+
the exit hidden state; verification runs [c_min, L).
|
| 18 |
+
Prefill (shared by all methods) runs eagerly.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import math
|
| 24 |
+
import time
|
| 25 |
+
|
| 26 |
+
import numpy as np
|
| 27 |
+
|
| 28 |
+
import torch
|
| 29 |
+
import torch.nn.functional as F
|
| 30 |
+
|
| 31 |
+
from .decode import GenStats, StaticPolicy
|
| 32 |
+
from .heads import ExitHead
|
| 33 |
+
try:
|
| 34 |
+
from .kernels import row_attention
|
| 35 |
+
except ImportError: # no Triton (CPU tests)
|
| 36 |
+
row_attention = None
|
| 37 |
+
from .layered import LayeredDecoder, _rotate_half
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _lam(controller) -> float:
|
| 41 |
+
"""Scalar lambda of an online controller (NoSlowdown wraps it in .aci)."""
|
| 42 |
+
c = getattr(controller, "aci", controller)
|
| 43 |
+
return float(np.min(getattr(c, "lam", float("nan"))))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class DraftModel:
|
| 47 |
+
"""Classical speculative decoding: a small model of the same tokenizer drafts for the target
|
| 48 |
+
(P1 baseline). Own FastDecoder (own KV, AR graph). Per cycle: catch up the committed tokens it
|
| 49 |
+
has not processed, then draft `gamma` tokens greedily, or stop early when its max probability
|
| 50 |
+
falls below `conf` (adaptive). The drafts are verified exactly by the target like copies."""
|
| 51 |
+
|
| 52 |
+
def __init__(self, fast: "FastDecoder", gamma: int = 4, conf: float | None = None):
|
| 53 |
+
self.fast, self.gamma, self.conf = fast, gamma, conf
|
| 54 |
+
|
| 55 |
+
def start(self, prompt: list[int]) -> None:
|
| 56 |
+
self.fast._check_capacity(prompt, 0)
|
| 57 |
+
self.fast._prefill(prompt)
|
| 58 |
+
self.d_len = len(prompt) # positions whose draft KV is valid
|
| 59 |
+
|
| 60 |
+
def _step(self, tok: int, pos: int):
|
| 61 |
+
f = self.fast
|
| 62 |
+
f._set_pos([pos])
|
| 63 |
+
f.ids.fill_(tok)
|
| 64 |
+
a, c = f._g_ar_conf().tolist()
|
| 65 |
+
return int(a), float(c)
|
| 66 |
+
|
| 67 |
+
def _catchup(self, seq: list[int], start: int, end: int) -> None:
|
| 68 |
+
"""Committed tokens [start, end) the draft has not processed (e.g. after prompt-lookup
|
| 69 |
+
cycles): processed in blocks of max_rows through the draft's layer-0 verify graphs (KV
|
| 70 |
+
only; a draft-side error could only lower acceptance, never exactness)."""
|
| 71 |
+
f = self.fast
|
| 72 |
+
W = f.max_rows
|
| 73 |
+
ids = torch.tensor(seq[start:end], device=f.dec.device)
|
| 74 |
+
for s0 in range(0, end - start, W):
|
| 75 |
+
m = min(W, end - start - s0)
|
| 76 |
+
f._set_pos(list(range(start + s0, start + s0 + m)))
|
| 77 |
+
f.buf[:m].copy_(f.dec.embed(ids[s0:s0 + m]))
|
| 78 |
+
f._g_verify(0, m)
|
| 79 |
+
|
| 80 |
+
def propose(self, seq: list[int], budget: int) -> list[int]:
|
| 81 |
+
pos0 = len(seq) - 1
|
| 82 |
+
if pos0 - self.d_len > 1:
|
| 83 |
+
self._catchup(seq, self.d_len, pos0)
|
| 84 |
+
else:
|
| 85 |
+
for p in range(self.d_len, pos0): # committed tokens the draft has not processed yet
|
| 86 |
+
self._step(seq[p], p)
|
| 87 |
+
x, drafts, self.n_inputs = seq[pos0], [], 0
|
| 88 |
+
for j in range(min(self.gamma, budget)):
|
| 89 |
+
t, c = self._step(x, pos0 + j)
|
| 90 |
+
self.n_inputs += 1
|
| 91 |
+
if self.conf is not None and j > 0 and c < self.conf:
|
| 92 |
+
break
|
| 93 |
+
drafts.append(t)
|
| 94 |
+
x = t
|
| 95 |
+
self.pos0 = pos0
|
| 96 |
+
return drafts
|
| 97 |
+
|
| 98 |
+
def commit(self, accepted: int) -> None:
|
| 99 |
+
# inputs at pos0 + i (i >= 1) were d_i: valid iff i <= accepted
|
| 100 |
+
self.d_len = self.pos0 + 1 + min(accepted, self.n_inputs - 1)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class FastDecoder:
|
| 104 |
+
def __init__(self, dec: LayeredDecoder, heads: list[ExitHead], head_layers: list[int],
|
| 105 |
+
max_rows: int = 16, use_graphs: bool | None = None, shared_pool: bool = False,
|
| 106 |
+
prefill: str = "engine", prefill_chunk: int = 2048):
|
| 107 |
+
assert prefill in ("engine", "full"), prefill
|
| 108 |
+
self.prefill_mode, self.prefill_chunk = prefill, prefill_chunk
|
| 109 |
+
self.dec = dec
|
| 110 |
+
self.heads = heads
|
| 111 |
+
self.head_layers = head_layers
|
| 112 |
+
self.max_rows = max_rows
|
| 113 |
+
dev, d = dec.device, dec.spec.hidden_size
|
| 114 |
+
# Row metadata in ONE device buffer [pos | slot | cl | anc], filled by one copy from a
|
| 115 |
+
# host staging buffer (pinned on GPU): audit T5, piste 1 (3-4 pageable copies before).
|
| 116 |
+
W = max_rows
|
| 117 |
+
self._meta_d = torch.zeros(3 * W + W * W, dtype=torch.int64, device=dev)
|
| 118 |
+
self._meta_h = torch.zeros(3 * W + W * W, dtype=torch.int64)
|
| 119 |
+
if dev.type == "cuda":
|
| 120 |
+
self._meta_h = self._meta_h.pin_memory()
|
| 121 |
+
self.pos = self._meta_d[0:W]
|
| 122 |
+
# Tree drafts: slot = KV write index of each row; keys t < cl are read at index t,
|
| 123 |
+
# keys t >= cl at anc[row, t - cl] (the row's path). Chains: slot = pos, cl = pos + 1.
|
| 124 |
+
self.slot = self._meta_d[W:2 * W]
|
| 125 |
+
self.cl = self._meta_d[2 * W:3 * W]
|
| 126 |
+
self.anc = self._meta_d[3 * W:].view(W, W)
|
| 127 |
+
self.cl.fill_(1)
|
| 128 |
+
self.idx = torch.zeros(max_rows, dtype=torch.int64, device=dev)
|
| 129 |
+
self.ids = torch.zeros(1, dtype=torch.long, device=dev)
|
| 130 |
+
self.buf = torch.zeros(max_rows, d, device=dev, dtype=dec.dtype) # segment rows
|
| 131 |
+
self.hrows = torch.zeros(max_rows, d, device=dev, dtype=dec.dtype) # block rows
|
| 132 |
+
self.hb = torch.zeros(1, d, device=dev, dtype=dec.dtype) # head input
|
| 133 |
+
# sampling (T > 0): Gumbel noise filled eagerly before each head replay (no RNG inside
|
| 134 |
+
# graphs), per-head renormalised V_k distribution of the last sample, saved per draft row
|
| 135 |
+
vmax = max([h.n_vocab for h in heads] + [1])
|
| 136 |
+
self.gnoise = torch.zeros(vmax, device=dev, dtype=torch.float32)
|
| 137 |
+
self.qout = torch.zeros(vmax, device=dev, dtype=torch.float32)
|
| 138 |
+
self.qsave = torch.zeros(max_rows, vmax, device=dev, dtype=torch.float32)
|
| 139 |
+
self.temperature = 1.0
|
| 140 |
+
self._sampling, self._gen, self._rng = False, None, None
|
| 141 |
+
self.graphs: dict = {}
|
| 142 |
+
self._trace = None # debug: per-layer tensors (eager mode only)
|
| 143 |
+
# ROCm 7.0 (CLR graph "packet capture"): a graph replay is not reliably ordered
|
| 144 |
+
# with later work on the same stream; this faulted on MI210 (jobs 893662-893901).
|
| 145 |
+
# Synchronising after every replay removes it (job 893903) and is the fastest
|
| 146 |
+
# fix measured (96 tok/s AR vs 72 with DEBUG_CLR_GRAPH_PACKET_CAPTURE=0).
|
| 147 |
+
self.sync_each = bool(torch.version.hip)
|
| 148 |
+
self.use_graphs = dev.type == "cuda" if use_graphs is None else use_graphs
|
| 149 |
+
# One memory pool for all graphs: safe because graphs never run concurrently and
|
| 150 |
+
# every graph output is consumed (or copied into a persistent buffer) right away.
|
| 151 |
+
self.pool = torch.cuda.graph_pool_handle() if (shared_pool and self.use_graphs) else None
|
| 152 |
+
self.capture_seconds = 0.0
|
| 153 |
+
self.stats = {"seg_replays": 0, "seg_rows": 0, "noncontig": 0}
|
| 154 |
+
|
| 155 |
+
def _sync(self) -> None:
|
| 156 |
+
if self.dec.device.type == "cuda":
|
| 157 |
+
torch.cuda.synchronize()
|
| 158 |
+
|
| 159 |
+
# ------------------------------------------------------------------ compute
|
| 160 |
+
def _rope_tables(self, n):
|
| 161 |
+
pos = self.pos[:n]
|
| 162 |
+
return pos, self.dec.cos.index_select(0, pos)[:, None], self.dec.sin.index_select(0, pos)[:, None]
|
| 163 |
+
|
| 164 |
+
def _pad(self, x: torch.Tensor) -> torch.Tensor:
|
| 165 |
+
n = x.shape[0]
|
| 166 |
+
return x if n == self.max_rows else F.pad(x, (0, 0) * (x.dim() - 1) + (0, self.max_rows - n))
|
| 167 |
+
|
| 168 |
+
def _lin(self, mod, x: torch.Tensor) -> torch.Tensor:
|
| 169 |
+
"""GEMM on a fixed-width (max_rows) input; returns all max_rows rows."""
|
| 170 |
+
return F.linear(self._pad(x), mod.weight, mod.bias)
|
| 171 |
+
|
| 172 |
+
def _mlp(self, mlp, x: torch.Tensor) -> torch.Tensor:
|
| 173 |
+
return self._lin(mlp.down_proj, F.silu(self._lin(mlp.gate_proj, x)) * self._lin(mlp.up_proj, x))
|
| 174 |
+
|
| 175 |
+
def _final_logits(self, h: torch.Tensor) -> torch.Tensor:
|
| 176 |
+
n = h.shape[0]
|
| 177 |
+
return self._lin(self.dec.lm_head, self.dec.norm(self._pad(h))).float()[:n]
|
| 178 |
+
|
| 179 |
+
def _layers(self, h: torch.Tensor, first: int, last: int) -> torch.Tensor:
|
| 180 |
+
"""Layers [first, last) on n real rows, at FIXED WIDTH.
|
| 181 |
+
|
| 182 |
+
Every elementwise op, reduction (RMSNorm, q/k norm) and GEMM runs on exactly
|
| 183 |
+
`max_rows` rows (real rows first, zero padding after), so each kernel sees the
|
| 184 |
+
same shape whatever n is and a row is computed bitwise identically in AR (n=1),
|
| 185 |
+
catch-up (m rows) and verification (n rows). Library kernels pick different
|
| 186 |
+
code paths by shape: hipBLASLt/rocBLAS GEMMs (jobs 893616, 893848) and the
|
| 187 |
+
MLP-block elementwise/reduction kernels (job 893872) were not row-invariant.
|
| 188 |
+
Only the K/V cache writes (exact copies) and attention (row-invariant Triton
|
| 189 |
+
kernel) touch the n real rows alone. Pad rows use position pos[n:] (kept valid).
|
| 190 |
+
"""
|
| 191 |
+
dec = self.dec
|
| 192 |
+
n = h.shape[0]
|
| 193 |
+
W = self.max_rows
|
| 194 |
+
hd = dec.spec.head_dim
|
| 195 |
+
pos, cos, sin = self._rope_tables(W)
|
| 196 |
+
h = self._pad(h)
|
| 197 |
+
for li in range(first, last):
|
| 198 |
+
layer = dec.layers[li]
|
| 199 |
+
attn = layer.self_attn
|
| 200 |
+
x = layer.input_layernorm(h)
|
| 201 |
+
q = self._lin(attn.q_proj, x).view(W, -1, hd)
|
| 202 |
+
k = self._lin(attn.k_proj, x).view(W, -1, hd)
|
| 203 |
+
v = self._lin(attn.v_proj, x).view(W, -1, hd)
|
| 204 |
+
if dec.spec.has_qk_norm:
|
| 205 |
+
q, k = attn.q_norm(q), attn.k_norm(k)
|
| 206 |
+
q = q * cos + _rotate_half(q) * sin
|
| 207 |
+
k = k * cos + _rotate_half(k) * sin
|
| 208 |
+
dec.k_cache[li].index_copy_(1, self.slot[:n], k[:n].transpose(0, 1))
|
| 209 |
+
dec.v_cache[li].index_copy_(1, self.slot[:n], v[:n].transpose(0, 1))
|
| 210 |
+
o = self._attention(q[:n].contiguous(), li, n)
|
| 211 |
+
h = h + self._lin(attn.o_proj, o.reshape(n, -1))
|
| 212 |
+
if self._trace is not None:
|
| 213 |
+
self._trace.append(("attn_out", li, o.clone(), h[:n].clone()))
|
| 214 |
+
h = h + self._mlp(layer.mlp, layer.post_attention_layernorm(h))
|
| 215 |
+
if self._trace is not None:
|
| 216 |
+
self._trace.append(("layer_out", li, h[:n].clone()))
|
| 217 |
+
return h[:n]
|
| 218 |
+
|
| 219 |
+
def _attention(self, q: torch.Tensor, li: int, n: int) -> torch.Tensor:
|
| 220 |
+
dec = self.dec
|
| 221 |
+
if row_attention is not None and q.is_cuda:
|
| 222 |
+
return row_attention(q, dec.k_cache[li], dec.v_cache[li], self.pos, dec.scaling,
|
| 223 |
+
cl=self.cl, anc=self.anc)
|
| 224 |
+
outs = [] # reference path (CPU tests): one query per row over its exact path
|
| 225 |
+
cls, ancs = self.cl[:n].tolist(), self.anc[:n].tolist()
|
| 226 |
+
for i, p in enumerate(self.pos[:n].tolist()):
|
| 227 |
+
idx = torch.tensor(list(range(min(cls[i], p + 1))) + ancs[i][: max(p + 1 - cls[i], 0)])
|
| 228 |
+
outs.append(F.scaled_dot_product_attention(
|
| 229 |
+
q[i:i + 1].transpose(0, 1)[None], dec.k_cache[li, :, idx][None],
|
| 230 |
+
dec.v_cache[li, :, idx][None], scale=dec.scaling, enable_gqa=True)[0].transpose(0, 1))
|
| 231 |
+
return torch.cat(outs)
|
| 232 |
+
|
| 233 |
+
def _fill(self, h: torch.Tensor, first: int, last: int) -> None:
|
| 234 |
+
dec = self.dec
|
| 235 |
+
hd = dec.spec.head_dim
|
| 236 |
+
W = self.max_rows
|
| 237 |
+
pos, cos, sin = self._rope_tables(W)
|
| 238 |
+
x0 = self._pad(h)
|
| 239 |
+
for li in range(first, last):
|
| 240 |
+
layer = dec.layers[li]
|
| 241 |
+
attn = layer.self_attn
|
| 242 |
+
x = layer.input_layernorm(x0)
|
| 243 |
+
k = self._lin(attn.k_proj, x).view(W, -1, hd)
|
| 244 |
+
if dec.spec.has_qk_norm:
|
| 245 |
+
k = attn.k_norm(k)
|
| 246 |
+
k = k * cos + _rotate_half(k) * sin
|
| 247 |
+
v = self._lin(attn.v_proj, x).view(W, -1, hd)
|
| 248 |
+
dec.k_cache[li].index_copy_(1, pos[:1], k[:1].transpose(0, 1))
|
| 249 |
+
dec.v_cache[li].index_copy_(1, pos[:1], v[:1].transpose(0, 1))
|
| 250 |
+
|
| 251 |
+
def _head_out(self, k: int) -> torch.Tensor:
|
| 252 |
+
head = self.heads[k]
|
| 253 |
+
logits = head(self.hb)[0]
|
| 254 |
+
probs = F.softmax(logits, -1)
|
| 255 |
+
conf, arg = probs.max(-1)
|
| 256 |
+
if head.reject:
|
| 257 |
+
body = logits[:-1].argmax(-1)
|
| 258 |
+
conf = torch.where(arg == logits.shape[-1] - 1, torch.zeros_like(conf), conf)
|
| 259 |
+
else:
|
| 260 |
+
body = arg
|
| 261 |
+
return torch.stack([conf.float(), head.to_token_ids(body).float()])
|
| 262 |
+
|
| 263 |
+
def _head_out4(self, k: int) -> torch.Tensor:
|
| 264 |
+
"""Head k on hb: [conf, id_1..id_4, p_1..p_4] over V_k ∪ ⊥ (⊥ reported as id -1).
|
| 265 |
+
Iterated argmax instead of torch.topk (ROCm 7.0 graph bug, see _g_verify_topk)."""
|
| 266 |
+
head = self.heads[k]
|
| 267 |
+
logits = head(self.hb)[0].float()
|
| 268 |
+
probs = F.softmax(logits, -1)
|
| 269 |
+
conf, arg = probs.max(-1)
|
| 270 |
+
rej = logits.shape[-1] - 1
|
| 271 |
+
if head.reject:
|
| 272 |
+
conf = torch.where(arg == rej, torch.zeros_like(conf), conf)
|
| 273 |
+
ids, ps, pr = [], [], probs
|
| 274 |
+
for _ in range(4):
|
| 275 |
+
i = pr.argmax(-1)
|
| 276 |
+
ps.append(pr.gather(0, i[None])[0])
|
| 277 |
+
tok = head.to_token_ids(i.clamp_max(head.n_vocab - 1)[None])[0].float()
|
| 278 |
+
ids.append(torch.where(i == rej, torch.full_like(tok, -1.0), tok) if head.reject else tok)
|
| 279 |
+
pr = pr.scatter(0, i[None], float("-inf"))
|
| 280 |
+
return torch.stack([conf] + ids + ps)
|
| 281 |
+
|
| 282 |
+
def _g_head4(self, k):
|
| 283 |
+
return self._run(("head4", k), lambda: self._head_out4(k))
|
| 284 |
+
|
| 285 |
+
# ------------------------------------------------------------------ graphs
|
| 286 |
+
def _run(self, key, fn):
|
| 287 |
+
if not self.use_graphs:
|
| 288 |
+
return fn()
|
| 289 |
+
if key not in self.graphs:
|
| 290 |
+
t0 = time.perf_counter()
|
| 291 |
+
s = torch.cuda.Stream()
|
| 292 |
+
s.wait_stream(torch.cuda.current_stream())
|
| 293 |
+
with torch.cuda.stream(s):
|
| 294 |
+
for _ in range(2):
|
| 295 |
+
fn()
|
| 296 |
+
torch.cuda.current_stream().wait_stream(s)
|
| 297 |
+
g = torch.cuda.CUDAGraph()
|
| 298 |
+
with torch.cuda.graph(g, pool=self.pool):
|
| 299 |
+
out = fn()
|
| 300 |
+
self.graphs[key] = (g, out)
|
| 301 |
+
self._sync()
|
| 302 |
+
self.capture_seconds += time.perf_counter() - t0
|
| 303 |
+
g, out = self.graphs[key]
|
| 304 |
+
if self.sync_each:
|
| 305 |
+
# ROCm 7.0: a replay is not reliably ordered with the eager work queued just
|
| 306 |
+
# before it either (e.g. the buf <- hrows copy): prompt-lookup cycles verified
|
| 307 |
+
# from layer 0 read a stale buf (job 899287, parity 0.67; exact without graphs or
|
| 308 |
+
# with packet capture off, 899290-899291). A bare sync costs ~4 us (job 899254).
|
| 309 |
+
torch.cuda.synchronize()
|
| 310 |
+
g.replay()
|
| 311 |
+
if self.sync_each:
|
| 312 |
+
torch.cuda.synchronize()
|
| 313 |
+
return out
|
| 314 |
+
|
| 315 |
+
def _g_ar(self):
|
| 316 |
+
L = self.dec.spec.num_layers
|
| 317 |
+
return self._run(("ar",), lambda: self._final_logits(
|
| 318 |
+
self._layers(self.dec.embed(self.ids), 0, L)).argmax(-1))
|
| 319 |
+
|
| 320 |
+
def _head_sample_out(self, k: int) -> torch.Tensor:
|
| 321 |
+
"""Head k on hb at temperature self.temperature: [conf (exit score, 0 if ⊥ wins),
|
| 322 |
+
sampled token id from q' = softmax(V_k logits / T) via Gumbel-max on gnoise, q'(sample)];
|
| 323 |
+
q' is left in qout[:n_vocab]."""
|
| 324 |
+
head = self.heads[k]
|
| 325 |
+
lg = head(self.hb)[0].float()
|
| 326 |
+
probs = F.softmax(lg, -1)
|
| 327 |
+
conf, arg = probs.max(-1)
|
| 328 |
+
if head.reject:
|
| 329 |
+
conf = torch.where(arg == lg.shape[-1] - 1, torch.zeros_like(conf), conf)
|
| 330 |
+
nv = head.n_vocab
|
| 331 |
+
qb = F.softmax(lg[:nv] / self.temperature, -1)
|
| 332 |
+
x = (qb.clamp_min(1e-30).log() + self.gnoise[:nv]).argmax(-1)
|
| 333 |
+
self.qout[:nv].copy_(qb)
|
| 334 |
+
tok = head.to_token_ids(x[None])[0].float()
|
| 335 |
+
return torch.stack([conf, tok, qb.gather(0, x[None])[0]])
|
| 336 |
+
|
| 337 |
+
def _g_head_sample(self, k):
|
| 338 |
+
return self._run(("head_sample", k), lambda: self._head_sample_out(k))
|
| 339 |
+
|
| 340 |
+
def _g_ar_logits(self):
|
| 341 |
+
L = self.dec.spec.num_layers
|
| 342 |
+
return self._run(("ar_logits",), lambda: self._final_logits(
|
| 343 |
+
self._layers(self.dec.embed(self.ids), 0, L)).float()[0])
|
| 344 |
+
|
| 345 |
+
def _g_verify_logits(self, c, n):
|
| 346 |
+
L = self.dec.spec.num_layers
|
| 347 |
+
return self._run(("verify_logits", c, n), lambda: self._final_logits(
|
| 348 |
+
self._layers(self.buf[:n], c, L)).float())
|
| 349 |
+
|
| 350 |
+
def _g_ar_conf(self):
|
| 351 |
+
"""AR step returning [argmax, max prob] (draft model with adaptive stop)."""
|
| 352 |
+
L = self.dec.spec.num_layers
|
| 353 |
+
|
| 354 |
+
def fn():
|
| 355 |
+
pr = F.softmax(self._final_logits(self._layers(self.dec.embed(self.ids), 0, L)).float(), -1)[0]
|
| 356 |
+
conf, arg = pr.max(-1)
|
| 357 |
+
return torch.stack([arg.float(), conf])
|
| 358 |
+
return self._run(("ar_conf",), fn)
|
| 359 |
+
|
| 360 |
+
def _g_seg(self, a, b, m):
|
| 361 |
+
def fn():
|
| 362 |
+
self.buf[:m].copy_(self._layers(self.buf[:m], a, b))
|
| 363 |
+
return self.buf
|
| 364 |
+
self.stats["seg_replays"] += 1
|
| 365 |
+
self.stats["seg_rows"] += m
|
| 366 |
+
return self._run(("seg", a, b, m), fn)
|
| 367 |
+
|
| 368 |
+
def _g_head(self, k):
|
| 369 |
+
return self._run(("head", k), lambda: self._head_out(k))
|
| 370 |
+
|
| 371 |
+
def _g_fill(self, a, b):
|
| 372 |
+
def fn():
|
| 373 |
+
self._fill(self.hb, a, b)
|
| 374 |
+
return self.hb
|
| 375 |
+
return self._run(("fill", a, b), fn)
|
| 376 |
+
|
| 377 |
+
TOPK = 8
|
| 378 |
+
|
| 379 |
+
def _g_verify_topk(self, c, n):
|
| 380 |
+
"""Verification returning, per row, the argmax then the top-TOPK token ids (lossy rules)."""
|
| 381 |
+
L = self.dec.spec.num_layers
|
| 382 |
+
|
| 383 |
+
def fn():
|
| 384 |
+
# Iterated argmax instead of torch.topk: topk on a 151k vocabulary crashed inside a
|
| 385 |
+
# HIP graph (illegal address, ROCm 7.0); argmax is the kernel the lossless path uses.
|
| 386 |
+
lg = self._final_logits(self._layers(self.buf[:n], c, L)).float()
|
| 387 |
+
cols = []
|
| 388 |
+
for _ in range(self.TOPK):
|
| 389 |
+
i = lg.argmax(-1, keepdim=True)
|
| 390 |
+
cols.append(i)
|
| 391 |
+
lg = lg.scatter(-1, i, float("-inf"))
|
| 392 |
+
return torch.cat([cols[0]] + cols, -1)
|
| 393 |
+
return self._run(("verify_topk", c, n), fn)
|
| 394 |
+
|
| 395 |
+
def _g_verify(self, c, n):
|
| 396 |
+
L = self.dec.spec.num_layers
|
| 397 |
+
return self._run(("verify", c, n), lambda: self._final_logits(
|
| 398 |
+
self._layers(self.buf[:n], c, L)).argmax(-1))
|
| 399 |
+
|
| 400 |
+
# ------------------------------------------------------------------ helpers
|
| 401 |
+
def _set_pos(self, plist: list[int], slots: list[int] | None = None) -> None:
|
| 402 |
+
# Pad rows (fixed width) reuse the last real position: always a valid index.
|
| 403 |
+
# slots: KV indices of the rows (default = positions; differ only for star siblings).
|
| 404 |
+
W = self.max_rows
|
| 405 |
+
full = list(plist) + [plist[-1]] * (W - len(plist))
|
| 406 |
+
sl = full if slots is None else list(slots) + [slots[-1]] * (W - len(slots))
|
| 407 |
+
h = self._meta_h
|
| 408 |
+
h[0:W] = torch.tensor(full)
|
| 409 |
+
h[W:2 * W] = torch.tensor(sl)
|
| 410 |
+
# a row whose slot differs from its position reads its own key from the slot
|
| 411 |
+
h[2 * W:3 * W] = torch.tensor([p + 1 if s_ == p else p for p, s_ in zip(full, sl)])
|
| 412 |
+
if slots is None: # chains never read anc (cl = pos + 1): skip its transfer
|
| 413 |
+
self._meta_d[:3 * W].copy_(h[:3 * W])
|
| 414 |
+
else:
|
| 415 |
+
a = h[3 * W:].view(W, W)
|
| 416 |
+
a.zero_()
|
| 417 |
+
a[:, 0] = h[W:2 * W]
|
| 418 |
+
self._meta_d.copy_(h)
|
| 419 |
+
|
| 420 |
+
def _set_rows(self, pos: list[int], slots: list[int], cls: list[int], ancs: list[list[int]]) -> None:
|
| 421 |
+
"""Tree rows: positions, KV slots, contiguous prefix lengths and path slots."""
|
| 422 |
+
W = self.max_rows
|
| 423 |
+
assert len(set(slots)) == len(slots), "two rows would write the same KV slot"
|
| 424 |
+
pad = W - len(pos)
|
| 425 |
+
h = self._meta_h
|
| 426 |
+
h[0:W] = torch.tensor(pos + [pos[-1]] * pad)
|
| 427 |
+
h[W:2 * W] = torch.tensor(slots + [slots[-1]] * pad)
|
| 428 |
+
h[2 * W:3 * W] = torch.tensor(cls + [cls[-1]] * pad)
|
| 429 |
+
a = h[3 * W:].view(W, W)
|
| 430 |
+
a.zero_()
|
| 431 |
+
for i, row in enumerate(ancs + [ancs[-1]] * pad):
|
| 432 |
+
if row:
|
| 433 |
+
a[i, :len(row)] = torch.tensor(row)
|
| 434 |
+
self._meta_d.copy_(h)
|
| 435 |
+
|
| 436 |
+
def _gather(self, rows: list[int]) -> None:
|
| 437 |
+
m = len(rows)
|
| 438 |
+
if rows == list(range(rows[0], rows[0] + m)):
|
| 439 |
+
self.buf[:m].copy_(self.hrows[rows[0]:rows[0] + m])
|
| 440 |
+
else:
|
| 441 |
+
self.stats["noncontig"] = self.stats.get("noncontig", 0) + 1
|
| 442 |
+
self.idx[:m].copy_(torch.tensor(rows, dtype=torch.int64))
|
| 443 |
+
torch.index_select(self.hrows, 0, self.idx[:m], out=self.buf[:m])
|
| 444 |
+
|
| 445 |
+
def _scatter(self, rows: list[int]) -> None:
|
| 446 |
+
m = len(rows)
|
| 447 |
+
if rows == list(range(rows[0], rows[0] + m)):
|
| 448 |
+
self.hrows[rows[0]:rows[0] + m].copy_(self.buf[:m])
|
| 449 |
+
else:
|
| 450 |
+
self.hrows.index_copy_(0, self.idx[:m], self.buf[:m])
|
| 451 |
+
|
| 452 |
+
def _embed_row(self, j: int, tok: int) -> None:
|
| 453 |
+
self.ids.fill_(tok)
|
| 454 |
+
self.hrows[j:j + 1].copy_(self.dec.embed(self.ids))
|
| 455 |
+
|
| 456 |
+
def _prefill(self, prompt: list[int]) -> int:
|
| 457 |
+
"""Prompt processed by the engine itself, in causal chunks of max_rows rows
|
| 458 |
+
through the fixed-width graphs (verify graphs from layer 0). No third-party
|
| 459 |
+
attention kernel runs between graph replays, and the prompt KV is computed
|
| 460 |
+
with exactly the same operators as generated tokens. prefill="full": wide GEMMs
|
| 461 |
+
and standard causal attention instead (_prefill_full)."""
|
| 462 |
+
self._prefix = None # a greedy prefill overwrites the cached sampling prefix
|
| 463 |
+
if self.prefill_mode == "full":
|
| 464 |
+
return self._prefill_full(prompt)
|
| 465 |
+
W = self.max_rows
|
| 466 |
+
ids = torch.tensor(prompt, device=self.dec.device)
|
| 467 |
+
last = None
|
| 468 |
+
for s0 in range(0, len(prompt), W):
|
| 469 |
+
m = min(W, len(prompt) - s0)
|
| 470 |
+
self._set_pos(list(range(s0, s0 + m)))
|
| 471 |
+
self.buf[:m].copy_(self.dec.embed(ids[s0:s0 + m]))
|
| 472 |
+
last = self._g_verify(0, m)
|
| 473 |
+
return int(last[m - 1])
|
| 474 |
+
|
| 475 |
+
@torch.no_grad()
|
| 476 |
+
def _prefill_full(self, prompt: list[int]) -> int:
|
| 477 |
+
"""Standard prefill: eager wide GEMMs over the whole prompt (chunks of prefill_chunk
|
| 478 |
+
rows) and causal scaled-dot-product attention, writing the prompt KV in place.
|
| 479 |
+
|
| 480 |
+
Decoding exactness is unchanged: AR and speculative decoding both start from this
|
| 481 |
+
same KV and first token, and every decode-time row still goes through the
|
| 482 |
+
fixed-width graphs. Only the prompt KV differs (bf16 rounding of other kernels)
|
| 483 |
+
from the in-engine prefill, so outputs may differ from an engine-prefill run."""
|
| 484 |
+
dec = self.dec
|
| 485 |
+
hd = dec.spec.head_dim
|
| 486 |
+
ids = torch.tensor(prompt, device=dec.device)
|
| 487 |
+
n = len(prompt)
|
| 488 |
+
h = None
|
| 489 |
+
for s0 in range(0, n, self.prefill_chunk):
|
| 490 |
+
m = min(self.prefill_chunk, n - s0)
|
| 491 |
+
pos = torch.arange(s0, s0 + m, device=dec.device)
|
| 492 |
+
cos, sin = dec.cos.index_select(0, pos)[:, None], dec.sin.index_select(0, pos)[:, None]
|
| 493 |
+
h = dec.embed(ids[s0:s0 + m])
|
| 494 |
+
if s0 == 0:
|
| 495 |
+
mask, causal = None, True
|
| 496 |
+
else: # query i (position s0 + i) sees keys t <= s0 + i
|
| 497 |
+
mask = (torch.arange(s0 + m, device=dec.device)[None] <= pos[:, None])[None, None]
|
| 498 |
+
causal = False
|
| 499 |
+
for li in range(dec.spec.num_layers):
|
| 500 |
+
layer = dec.layers[li]
|
| 501 |
+
attn = layer.self_attn
|
| 502 |
+
x = layer.input_layernorm(h)
|
| 503 |
+
q = attn.q_proj(x).view(m, -1, hd)
|
| 504 |
+
k = attn.k_proj(x).view(m, -1, hd)
|
| 505 |
+
v = attn.v_proj(x).view(m, -1, hd)
|
| 506 |
+
if dec.spec.has_qk_norm:
|
| 507 |
+
q, k = attn.q_norm(q), attn.k_norm(k)
|
| 508 |
+
q = q * cos + _rotate_half(q) * sin
|
| 509 |
+
k = k * cos + _rotate_half(k) * sin
|
| 510 |
+
dec.k_cache[li, :, s0:s0 + m].copy_(k.transpose(0, 1))
|
| 511 |
+
dec.v_cache[li, :, s0:s0 + m].copy_(v.transpose(0, 1))
|
| 512 |
+
o = F.scaled_dot_product_attention(
|
| 513 |
+
q.transpose(0, 1)[None], dec.k_cache[li, :, :s0 + m][None], dec.v_cache[li, :, :s0 + m][None],
|
| 514 |
+
attn_mask=mask, is_causal=causal, scale=dec.scaling, enable_gqa=True)[0]
|
| 515 |
+
h = h + attn.o_proj(o.transpose(0, 1).reshape(m, -1))
|
| 516 |
+
h = h + layer.mlp(layer.post_attention_layernorm(h))
|
| 517 |
+
tok = int(dec.lm_head(dec.norm(h[-1:])).float().argmax(-1)[0])
|
| 518 |
+
self._sync() # eager kernels must finish before the next graph replay (ROCm ordering)
|
| 519 |
+
return tok
|
| 520 |
+
|
| 521 |
+
@staticmethod
|
| 522 |
+
def _boundaries(policy, head_layers):
|
| 523 |
+
# extra_bounds: segment boundaries without a head. Prompt lookup alone uses the h0
|
| 524 |
+
# layer: copies go through seg(0, L_h0) + verify(L_h0) like every other policy.
|
| 525 |
+
# Decode-time verification straight from layer 0 gave a wrong first replay after
|
| 526 |
+
# the prefill on MI210 (ROCm 7.0, jobs 899287/899321: graph token 0, eager correct).
|
| 527 |
+
return sorted({0} | {head_layers[k] for k in policy.head_order}
|
| 528 |
+
| set(getattr(policy, "extra_bounds", ())))
|
| 529 |
+
|
| 530 |
+
def warm_blas(self, max_prefill: int = 0) -> None:
|
| 531 |
+
"""Kept for compatibility: with the in-engine prefill, every GEMM has the fixed
|
| 532 |
+
width max_rows, so no larger BLAS call happens after capture."""
|
| 533 |
+
self._warm = self.dec.max_len
|
| 534 |
+
|
| 535 |
+
def precapture(self, policies: list, kv_fill: str = "lazy") -> None:
|
| 536 |
+
"""Capture every graph the given policies can use, at scratch positions."""
|
| 537 |
+
self._precapture_body(policies, kv_fill)
|
| 538 |
+
|
| 539 |
+
def _precapture_body(self, policies: list, kv_fill: str = "lazy") -> None:
|
| 540 |
+
if not getattr(self, "_warm", 0):
|
| 541 |
+
self.warm_blas()
|
| 542 |
+
scratch = self.dec.max_len - self.max_rows - 2
|
| 543 |
+
self._set_pos(list(range(scratch, scratch + self.max_rows)))
|
| 544 |
+
self.ids.fill_(0)
|
| 545 |
+
self._g_ar()
|
| 546 |
+
for m in range(1, self.max_rows + 1): # in-engine prefill chunks
|
| 547 |
+
self._g_verify(0, m)
|
| 548 |
+
for pol in policies:
|
| 549 |
+
b = self._boundaries(pol, self.head_layers)
|
| 550 |
+
c_min, c_max = (b[1] if len(b) > 1 else 0), b[-1] # head-less (prompt lookup): [0]
|
| 551 |
+
n_rows = self.max_rows if getattr(pol, "tree", None) else min(pol.max_draft + 1, self.max_rows)
|
| 552 |
+
for k in pol.head_order:
|
| 553 |
+
self._g_head(k)
|
| 554 |
+
for u, v in zip(b, b[1:]):
|
| 555 |
+
for m in range(1, (n_rows if kv_fill == "lazy" else 1) + 1):
|
| 556 |
+
self._g_seg(u, v, m)
|
| 557 |
+
if kv_fill == "copy":
|
| 558 |
+
for k in pol.head_order:
|
| 559 |
+
if self.head_layers[k] < c_max:
|
| 560 |
+
self._g_fill(self.head_layers[k], c_max)
|
| 561 |
+
for n in range(1, n_rows + 1):
|
| 562 |
+
self._g_verify(c_max if kv_fill == "lazy" else c_min, n)
|
| 563 |
+
if getattr(pol, "lossy", False):
|
| 564 |
+
self._g_verify_topk(c_max if kv_fill == "lazy" else c_min, n)
|
| 565 |
+
for pol in policies: # tree drafts (star, or trie with star siblings): the top-4 head graph
|
| 566 |
+
st_cfg = getattr(pol, "star", None) or (getattr(pol, "tree", None) or {}).get("star")
|
| 567 |
+
if st_cfg is not None:
|
| 568 |
+
self.hb.zero_()
|
| 569 |
+
self._g_head4(st_cfg.get("head", 0))
|
| 570 |
+
self.stats = {"seg_replays": 0, "seg_rows": 0, "noncontig": 0}
|
| 571 |
+
self._sync()
|
| 572 |
+
|
| 573 |
+
# ------------------------------------------------------------------ decoding
|
| 574 |
+
@torch.no_grad()
|
| 575 |
+
def _check_capacity(self, prompt, max_new) -> None:
|
| 576 |
+
# drafts write KV up to max_rows slots beyond the last committed position
|
| 577 |
+
if len(prompt) + max_new + self.max_rows > self.dec.max_len:
|
| 578 |
+
raise ValueError(f"prompt {len(prompt)} + max_new {max_new} + {self.max_rows} draft rows "
|
| 579 |
+
f"exceeds the KV capacity {self.dec.max_len}")
|
| 580 |
+
|
| 581 |
+
def _sample(self, probs: torch.Tensor) -> int:
|
| 582 |
+
return int(torch.multinomial(probs, 1, generator=self._gen))
|
| 583 |
+
|
| 584 |
+
def _first_token_sampled(self, prompt: list[int]) -> int:
|
| 585 |
+
"""Prefill prompt[:-1] (KV only), then the last prompt token through the AR graph: the
|
| 586 |
+
first new token is SAMPLED from its logits (same arithmetic for AR and speculation)."""
|
| 587 |
+
# prompt[:-1] KV is never rewritten by generation (writes start at len(prompt) - 1): reuse it
|
| 588 |
+
# across samples of the same prompt (distribution tests draw thousands)
|
| 589 |
+
if len(prompt) > 1 and not (getattr(self, "reuse_prefix", False)
|
| 590 |
+
and getattr(self, "_prefix", None) == tuple(prompt[:-1])):
|
| 591 |
+
self._prefill(prompt[:-1])
|
| 592 |
+
self._prefix = tuple(prompt[:-1])
|
| 593 |
+
self._set_pos([len(prompt) - 1])
|
| 594 |
+
self.ids.fill_(prompt[-1])
|
| 595 |
+
return self._sample(F.softmax(self._g_ar_logits() / self.temperature, -1))
|
| 596 |
+
|
| 597 |
+
def _sample_verify(self, c_verify: int, drafts: list[int], st) -> list[int]:
|
| 598 |
+
"""Exact speculative sampling on a chain (Leviathan et al.): draft i is accepted with
|
| 599 |
+
probability min(1, p_i(x_i) / q_i(x_i)); at the first rejection the token is resampled
|
| 600 |
+
from norm(max(p_i - q_i, 0)); if all are accepted the bonus is sampled from p_n. q_i is
|
| 601 |
+
the head's renormalised V_k distribution at T (saved at drafting) or a point mass (copy)."""
|
| 602 |
+
n = len(drafts) + 1
|
| 603 |
+
P = F.softmax(self._g_verify_logits(c_verify, n)[:n] / self.temperature, -1)
|
| 604 |
+
st.cycles += 1
|
| 605 |
+
st.drafted += len(drafts)
|
| 606 |
+
st.empty_drafts += int(not drafts)
|
| 607 |
+
committed = []
|
| 608 |
+
if drafts:
|
| 609 |
+
pv = P[torch.arange(len(drafts), device=P.device), torch.tensor(drafts, device=P.device)].tolist()
|
| 610 |
+
for i, x in enumerate(drafts):
|
| 611 |
+
k, qx = self._draft_q[i]
|
| 612 |
+
if self._rng.random() < min(1.0, pv[i] / max(qx, 1e-30)):
|
| 613 |
+
committed.append(x)
|
| 614 |
+
continue
|
| 615 |
+
res = P[i].clone()
|
| 616 |
+
if k < 0:
|
| 617 |
+
res[x] = 0.0
|
| 618 |
+
else:
|
| 619 |
+
h = self.heads[k]
|
| 620 |
+
q = torch.zeros_like(res)
|
| 621 |
+
q[h.vocab_ids] = self.qsave[i, :h.n_vocab]
|
| 622 |
+
res = (res - q).clamp_min(0.0)
|
| 623 |
+
tot = float(res.sum())
|
| 624 |
+
committed.append(self._sample(res / tot if tot > 0 else P[i]))
|
| 625 |
+
st.rejected_cycles += 1
|
| 626 |
+
break
|
| 627 |
+
else:
|
| 628 |
+
committed.append(self._sample(P[len(drafts)]))
|
| 629 |
+
st.accepted += min(len(committed) - 1, len(drafts))
|
| 630 |
+
return committed
|
| 631 |
+
|
| 632 |
+
@staticmethod
|
| 633 |
+
def _emit(st: GenStats, on_commit, toks: list[int], srcs: list[str]) -> None:
|
| 634 |
+
st.sources.extend(srcs)
|
| 635 |
+
if on_commit is not None:
|
| 636 |
+
on_commit(list(toks), list(srcs))
|
| 637 |
+
|
| 638 |
+
def _commit(self, out, committed, srcs, index, stop, eos_ids, max_new, st, on_commit) -> bool:
|
| 639 |
+
"""Append a cycle's committed tokens to out (truncated at stop / EOS / max_new), feed the
|
| 640 |
+
prompt-lookup index, record their sources. Returns True if the stop rule fired."""
|
| 641 |
+
n0, stopped = len(out), False
|
| 642 |
+
for t in committed:
|
| 643 |
+
out.append(t)
|
| 644 |
+
if index is not None:
|
| 645 |
+
index.extend([t])
|
| 646 |
+
if stop is not None and stop(t):
|
| 647 |
+
stopped = True
|
| 648 |
+
break
|
| 649 |
+
if t in eos_ids or len(out) >= max_new:
|
| 650 |
+
break
|
| 651 |
+
self._emit(st, on_commit, out[n0:], srcs[:len(out) - n0])
|
| 652 |
+
return stopped
|
| 653 |
+
|
| 654 |
+
def generate_ar(self, prompt: list[int], max_new: int, eos_ids=frozenset(), stop=None,
|
| 655 |
+
temperature: float = 0.0, gen: torch.Generator | None = None,
|
| 656 |
+
on_commit=None) -> tuple[list[int], GenStats]:
|
| 657 |
+
"""stop: optional callable fed every new token, True = stop after it (same rule as
|
| 658 |
+
generate_spec, so both stop at the same token). on_commit(tokens, sources): optional
|
| 659 |
+
callable fed the tokens appended to the output (and their sources) after the prefill
|
| 660 |
+
and after every step."""
|
| 661 |
+
self._check_capacity(prompt, max_new)
|
| 662 |
+
st = GenStats()
|
| 663 |
+
self._sync()
|
| 664 |
+
t0 = time.perf_counter()
|
| 665 |
+
if temperature > 0:
|
| 666 |
+
self.temperature, self._gen = temperature, gen
|
| 667 |
+
out = [self._first_token_sampled(prompt)]
|
| 668 |
+
self._sync()
|
| 669 |
+
st.prefill_seconds = time.perf_counter() - t0
|
| 670 |
+
self._emit(st, on_commit, out[:1], ["model"])
|
| 671 |
+
while len(out) < max_new and out[-1] not in eos_ids and not (stop is not None and stop(out[-1])):
|
| 672 |
+
self._set_pos([len(prompt) + len(out) - 1])
|
| 673 |
+
self.ids.fill_(out[-1])
|
| 674 |
+
out.append(self._sample(F.softmax(self._g_ar_logits() / temperature, -1)))
|
| 675 |
+
self._emit(st, on_commit, out[-1:], ["model"])
|
| 676 |
+
st.cycles += 1
|
| 677 |
+
self._sync()
|
| 678 |
+
st.seconds = time.perf_counter() - t0
|
| 679 |
+
st.new_tokens = len(out)
|
| 680 |
+
return out, st
|
| 681 |
+
out = [self._prefill(prompt)]
|
| 682 |
+
self._sync()
|
| 683 |
+
st.prefill_seconds = time.perf_counter() - t0
|
| 684 |
+
self._emit(st, on_commit, out[:1], ["model"])
|
| 685 |
+
stopped = stop is not None and stop(out[0])
|
| 686 |
+
steps, t_prev = [], time.perf_counter()
|
| 687 |
+
while not stopped and len(out) < max_new and out[-1] not in eos_ids:
|
| 688 |
+
self._set_pos([len(prompt) + len(out) - 1])
|
| 689 |
+
self.ids.fill_(out[-1])
|
| 690 |
+
out.append(int(self._g_ar())) # .item() synchronises: per-step wall time
|
| 691 |
+
self._emit(st, on_commit, out[-1:], ["model"])
|
| 692 |
+
if stop is not None and stop(out[-1]):
|
| 693 |
+
break
|
| 694 |
+
t_now = time.perf_counter()
|
| 695 |
+
steps.append(t_now - t_prev)
|
| 696 |
+
t_prev = t_now
|
| 697 |
+
st.cycles += 1
|
| 698 |
+
if steps: # robust unit for cost controllers (outliers: syncs, GC)
|
| 699 |
+
st.ar_step_median = float(np.median(steps))
|
| 700 |
+
self._sync()
|
| 701 |
+
st.seconds = time.perf_counter() - t0
|
| 702 |
+
st.new_tokens = len(out)
|
| 703 |
+
return out, st
|
| 704 |
+
|
| 705 |
+
def _draft_lazy(self, policy, first_tok, pos0, budget, eos_ids, st, proposal=None, pld_mode=None):
|
| 706 |
+
"""Regime A. Returns drafts; on exit all block rows sit in hrows at c_max.
|
| 707 |
+
|
| 708 |
+
proposal / pld_mode (prompt lookup): "first" drafts the copied tokens without
|
| 709 |
+
evaluating any head; "fallback" uses them only if no head exits at the first
|
| 710 |
+
position (an empty cycle). Copied drafts enter at depth 0 and are caught up
|
| 711 |
+
lazily with the other rows (head index -1)."""
|
| 712 |
+
hl = self.head_layers
|
| 713 |
+
b = self._boundaries(policy, hl)
|
| 714 |
+
c_max = b[-1]
|
| 715 |
+
nxt = {u: v for u, v in zip(b, b[1:])}
|
| 716 |
+
depth: list[int] = []
|
| 717 |
+
|
| 718 |
+
def advance(target: int) -> None:
|
| 719 |
+
while True:
|
| 720 |
+
lag = [j for j, d in enumerate(depth) if d < target]
|
| 721 |
+
if not lag:
|
| 722 |
+
return
|
| 723 |
+
u = min(depth[j] for j in lag)
|
| 724 |
+
batch = [j for j in lag if depth[j] == u]
|
| 725 |
+
self._set_pos([pos0 + j for j in batch])
|
| 726 |
+
self._gather(batch)
|
| 727 |
+
self._g_seg(u, nxt[u], len(batch))
|
| 728 |
+
self._scatter(batch)
|
| 729 |
+
for j in batch:
|
| 730 |
+
depth[j] = nxt[u]
|
| 731 |
+
|
| 732 |
+
drafts: list[int] = []
|
| 733 |
+
self._draft_heads, self._stop_info, self._draft_q = [], None, []
|
| 734 |
+
tok = first_tok
|
| 735 |
+
|
| 736 |
+
def copy_in() -> None:
|
| 737 |
+
for t in proposal[: budget - len(drafts)]:
|
| 738 |
+
drafts.append(t)
|
| 739 |
+
self._draft_heads.append(-1)
|
| 740 |
+
if self._sampling:
|
| 741 |
+
self._draft_q.append((-1, 1.0)) # copy: deterministic draft, q = delta
|
| 742 |
+
st.exits["pld"] = st.exits.get("pld", 0) + 1
|
| 743 |
+
self._embed_row(len(depth), t)
|
| 744 |
+
depth.append(0)
|
| 745 |
+
if t in eos_ids:
|
| 746 |
+
break
|
| 747 |
+
|
| 748 |
+
while True:
|
| 749 |
+
j = len(depth)
|
| 750 |
+
self._embed_row(j, tok)
|
| 751 |
+
depth.append(0)
|
| 752 |
+
if len(drafts) >= budget or (drafts and drafts[-1] in eos_ids):
|
| 753 |
+
break
|
| 754 |
+
if j == 0 and proposal and pld_mode == "first":
|
| 755 |
+
copy_in()
|
| 756 |
+
break
|
| 757 |
+
exited, row = None, {}
|
| 758 |
+
for k in policy.head_order:
|
| 759 |
+
advance(hl[k])
|
| 760 |
+
self.hb.copy_(self.hrows[j:j + 1])
|
| 761 |
+
if self._sampling:
|
| 762 |
+
self.gnoise.uniform_(1e-10, 1.0, generator=self._gen).log_().neg_().log_().neg_()
|
| 763 |
+
conf, tok_f, qx = self._g_head_sample(k).tolist()
|
| 764 |
+
else:
|
| 765 |
+
conf, tok_f = self._g_head(k).tolist()
|
| 766 |
+
row[k] = (conf, int(tok_f))
|
| 767 |
+
st.head_evals += 1
|
| 768 |
+
if policy.decide(k, conf, len(drafts)):
|
| 769 |
+
exited = k
|
| 770 |
+
if self._sampling: # keep q'_k of this draft row for the residual
|
| 771 |
+
self.qsave[len(drafts), :self.heads[k].n_vocab].copy_(self.qout[:self.heads[k].n_vocab])
|
| 772 |
+
self._draft_q.append((k, qx))
|
| 773 |
+
break
|
| 774 |
+
if exited is None:
|
| 775 |
+
self._stop_info = row # every head was evaluated at the stop position
|
| 776 |
+
if j == 0 and proposal and pld_mode == "fallback":
|
| 777 |
+
self._stop_info = None
|
| 778 |
+
copy_in()
|
| 779 |
+
break
|
| 780 |
+
tok = int(tok_f)
|
| 781 |
+
st.exits[exited] = st.exits.get(exited, 0) + 1
|
| 782 |
+
drafts.append(tok)
|
| 783 |
+
self._draft_heads.append(exited)
|
| 784 |
+
advance(c_max)
|
| 785 |
+
return drafts, c_max
|
| 786 |
+
|
| 787 |
+
def _draft_copy(self, policy, first_tok, pos0, budget, eos_ids, st):
|
| 788 |
+
"""Regime B (CALM copy). Returns drafts; rows at c_min are left in hrows."""
|
| 789 |
+
hl = self.head_layers
|
| 790 |
+
b = self._boundaries(policy, hl)
|
| 791 |
+
c_min, c_max = b[1], b[-1]
|
| 792 |
+
nxt = {u: v for u, v in zip(b, b[1:])}
|
| 793 |
+
drafts: list[int] = []
|
| 794 |
+
self._draft_heads = [] # head of each draft (provenance)
|
| 795 |
+
tok = first_tok
|
| 796 |
+
while True:
|
| 797 |
+
j = len(drafts)
|
| 798 |
+
self._set_pos([pos0 + j])
|
| 799 |
+
self._embed_row(j, tok)
|
| 800 |
+
self.buf[:1].copy_(self.hrows[j:j + 1])
|
| 801 |
+
self._g_seg(0, c_min, 1)
|
| 802 |
+
self.hrows[j:j + 1].copy_(self.buf[:1]) # hidden at c_min (verified later)
|
| 803 |
+
if len(drafts) >= budget or (drafts and drafts[-1] in eos_ids):
|
| 804 |
+
break
|
| 805 |
+
depth, exited = c_min, None
|
| 806 |
+
for k in policy.head_order:
|
| 807 |
+
while depth < hl[k]:
|
| 808 |
+
self._g_seg(depth, nxt[depth], 1)
|
| 809 |
+
depth = nxt[depth]
|
| 810 |
+
self.hb.copy_(self.buf[:1])
|
| 811 |
+
conf, tok_f = self._g_head(k).tolist()
|
| 812 |
+
st.head_evals += 1
|
| 813 |
+
if policy.decide(k, conf, len(drafts)):
|
| 814 |
+
exited = k
|
| 815 |
+
break
|
| 816 |
+
if exited is None:
|
| 817 |
+
break
|
| 818 |
+
if depth < c_max:
|
| 819 |
+
self._g_fill(depth, c_max)
|
| 820 |
+
tok = int(tok_f)
|
| 821 |
+
st.exits[exited] = st.exits.get(exited, 0) + 1
|
| 822 |
+
drafts.append(tok)
|
| 823 |
+
self._draft_heads.append(exited)
|
| 824 |
+
return drafts, c_min
|
| 825 |
+
|
| 826 |
+
def _star_cycle(self, star: dict, first_tok: int, pos0: int, budget: int, st) -> list[int]:
|
| 827 |
+
"""Depth-1 tree draft (star): root = last committed token; the shallow head (index
|
| 828 |
+
star["head"]) proposes up to C children from its top-4 at the root; children are
|
| 829 |
+
computed on their own KV slots pos0+1+i at position pos0+1, verified together with
|
| 830 |
+
the root; the accepted child's KV is compacted to slot pos0+1. Exact (lossless).
|
| 831 |
+
Admission: q_r = p_r (head probability of rank r, ⊥ excluded) >= tau[r], and the
|
| 832 |
+
survival sum of the admitted q_r >= gamma. Returns the committed tokens."""
|
| 833 |
+
k = star.get("head", 0)
|
| 834 |
+
c = self.head_layers[k]
|
| 835 |
+
self._embed_row(0, first_tok)
|
| 836 |
+
self._set_pos([pos0])
|
| 837 |
+
self._gather([0])
|
| 838 |
+
self._g_seg(0, c, 1)
|
| 839 |
+
self._scatter([0])
|
| 840 |
+
self.hb.copy_(self.hrows[0:1])
|
| 841 |
+
r = self._g_head4(k).tolist()
|
| 842 |
+
st.head_evals += 1
|
| 843 |
+
ids, ps = [int(x) for x in r[1:5]], r[5:9]
|
| 844 |
+
tau = star.get("tau", [0.05, 0.02, 0.02, 0.02])
|
| 845 |
+
kids = [t for t, p, th in zip(ids, ps, tau) if t >= 0 and p >= th][: min(star.get("C", 4), budget)]
|
| 846 |
+
qs = [p for t, p, th in zip(ids, ps, tau) if t >= 0 and p >= th][: len(kids)]
|
| 847 |
+
if kids and sum(qs) < star.get("gamma", 0.3):
|
| 848 |
+
kids = []
|
| 849 |
+
m = len(kids)
|
| 850 |
+
if m:
|
| 851 |
+
for i, t in enumerate(kids):
|
| 852 |
+
self._embed_row(1 + i, t)
|
| 853 |
+
self._set_pos([pos0 + 1] * m, [pos0 + 1 + i for i in range(m)])
|
| 854 |
+
self._gather(list(range(1, 1 + m)))
|
| 855 |
+
self._g_seg(0, c, m)
|
| 856 |
+
self._scatter(list(range(1, 1 + m)))
|
| 857 |
+
n = 1 + m
|
| 858 |
+
self._set_pos([pos0] + [pos0 + 1] * m, [pos0] + [pos0 + 1 + i for i in range(m)])
|
| 859 |
+
self.buf[:n].copy_(self.hrows[:n])
|
| 860 |
+
target = self._g_verify(c, n).tolist()
|
| 861 |
+
st.cycles += 1
|
| 862 |
+
st.drafted += m
|
| 863 |
+
st.empty_drafts += int(m == 0)
|
| 864 |
+
committed = [target[0]]
|
| 865 |
+
if m:
|
| 866 |
+
st.exits["star"] = st.exits.get("star", 0) + m
|
| 867 |
+
j = next((i for i, t in enumerate(kids) if t == target[0]), None)
|
| 868 |
+
if j is None:
|
| 869 |
+
st.rejected_cycles += 1
|
| 870 |
+
else:
|
| 871 |
+
st.accepted += 1
|
| 872 |
+
committed.append(target[1 + j])
|
| 873 |
+
if j > 0: # compaction: the accepted child's KV (all layers) to slot pos0+1
|
| 874 |
+
self.stats["star_compactions"] = self.stats.get("star_compactions", 0) + 1
|
| 875 |
+
dk, dv = self.dec.k_cache, self.dec.v_cache
|
| 876 |
+
dk[:, :, pos0 + 1].copy_(dk[:, :, pos0 + 1 + j])
|
| 877 |
+
dv[:, :, pos0 + 1].copy_(dv[:, :, pos0 + 1 + j])
|
| 878 |
+
return committed
|
| 879 |
+
|
| 880 |
+
def _tree_cycle(self, policy, cfg: dict, root_tok: int, pos0: int, eos_ids, st, proposal,
|
| 881 |
+
lam: float = float("-inf"), match_n: int = 0, ext=None):
|
| 882 |
+
"""General tree draft (trie), one verification. Rows = nodes in creation order
|
| 883 |
+
(row 0 = root at pos0); node i sits at position pos0 + depth, writes its KV at slot
|
| 884 |
+
pos0 + i and attends to [0, pos0] plus the slots of its own path.
|
| 885 |
+
1. prompt-lookup copy (proposal, <= cfg["m"]) inserted under the root first, so it
|
| 886 |
+
rides in the root's first segment;
|
| 887 |
+
2. head chain (lazy, policy.decide, <= cfg["head_budget"] drafts) that reuses a
|
| 888 |
+
copy row when its token matches; a head is never evaluated on a row already
|
| 889 |
+
computed beyond its layer;
|
| 890 |
+
3. optional star siblings at the root from the top-4 of cfg["star"]["head"].
|
| 891 |
+
Verification walks the trie from the root; the accepted path's KV is compacted to
|
| 892 |
+
positions pos0+1.. Exact (lossless).
|
| 893 |
+
|
| 894 |
+
Safety gate (chantier 3b §7.6, alpha guarantee on trees): every node v carries a
|
| 895 |
+
predicted acceptance q_v (head: a_hat_k; copy: cfg["gate"]["q_copy"][n][d] for match
|
| 896 |
+
length n and copy depth d; star: isotone map of the rank probability). After the
|
| 897 |
+
rows are computed, the children of u are KEPT (drafted, verified) only if
|
| 898 |
+
p(u) = min(1, sum_v q_v) >= lam; lam > 1 keeps none (pure probe). Returns
|
| 899 |
+
(committed tokens, info) with info = path origins, failed child origin, stop (p, a)
|
| 900 |
+
and the number of drafted nodes, for online.cycle_from_tree."""
|
| 901 |
+
gate = cfg.get("gate") or {}
|
| 902 |
+
qc = gate.get("q_copy")
|
| 903 |
+
qs = gate.get("q_star")
|
| 904 |
+
hl = self.head_layers
|
| 905 |
+
W = self.max_rows
|
| 906 |
+
b = self._boundaries(policy, hl)
|
| 907 |
+
c_max = b[-1]
|
| 908 |
+
nxt = dict(zip(b, b[1:]))
|
| 909 |
+
tok, par, dep, ld, kids, origin, q = [], [], [], [], [], [], []
|
| 910 |
+
|
| 911 |
+
def add(t, parent, how, qv=0.0):
|
| 912 |
+
if len(tok) >= W:
|
| 913 |
+
return None
|
| 914 |
+
i = len(tok)
|
| 915 |
+
tok.append(t); par.append(parent); dep.append(0 if parent is None else dep[parent] + 1)
|
| 916 |
+
ld.append(0); kids.append({}); origin.append(how); q.append(float(qv))
|
| 917 |
+
if parent is not None:
|
| 918 |
+
kids[parent][t] = i
|
| 919 |
+
self._embed_row(i, t)
|
| 920 |
+
return i
|
| 921 |
+
|
| 922 |
+
def path(i):
|
| 923 |
+
out = []
|
| 924 |
+
while par[i] is not None:
|
| 925 |
+
out.append(pos0 + i)
|
| 926 |
+
i = par[i]
|
| 927 |
+
return out[::-1]
|
| 928 |
+
|
| 929 |
+
def set_rows(rows):
|
| 930 |
+
self._set_rows([pos0 + dep[i] for i in rows], [pos0 + i for i in rows],
|
| 931 |
+
[pos0 + 1] * len(rows), [path(i) for i in rows])
|
| 932 |
+
|
| 933 |
+
def advance(target):
|
| 934 |
+
while True:
|
| 935 |
+
lag = [i for i in range(len(tok)) if ld[i] < target]
|
| 936 |
+
if not lag:
|
| 937 |
+
return
|
| 938 |
+
u = min(ld[i] for i in lag)
|
| 939 |
+
batch = [i for i in lag if ld[i] == u]
|
| 940 |
+
set_rows(batch)
|
| 941 |
+
self._gather(batch)
|
| 942 |
+
self._g_seg(u, nxt[u], len(batch))
|
| 943 |
+
self._scatter(batch)
|
| 944 |
+
for i in batch:
|
| 945 |
+
ld[i] = nxt[u]
|
| 946 |
+
|
| 947 |
+
root = add(root_tok, None, "root")
|
| 948 |
+
c = root
|
| 949 |
+
for t in (ext or []): # external draft model chain (P1: SD + PLD in one verification)
|
| 950 |
+
n = kids[c].get(t)
|
| 951 |
+
if n is None:
|
| 952 |
+
n = add(t, c, "draft")
|
| 953 |
+
if n is None:
|
| 954 |
+
break
|
| 955 |
+
st.exits["draft"] = st.exits.get("draft", 0) + 1
|
| 956 |
+
c = n
|
| 957 |
+
if t in eos_ids:
|
| 958 |
+
break
|
| 959 |
+
c = root
|
| 960 |
+
for d, t in enumerate((proposal or [])[: cfg.get("m", 15)], start=1):
|
| 961 |
+
n = kids[c].get(t)
|
| 962 |
+
if n is None:
|
| 963 |
+
qv = 0.0
|
| 964 |
+
if qc is not None:
|
| 965 |
+
row = qc.get(str(match_n)) or qc.get(str(max(int(k) for k in qc)))
|
| 966 |
+
qv = row[min(d, len(row)) - 1]
|
| 967 |
+
n = add(t, c, "copy", qv)
|
| 968 |
+
if n is None:
|
| 969 |
+
break
|
| 970 |
+
st.pld_drafted += 1
|
| 971 |
+
c = n
|
| 972 |
+
if t in eos_ids:
|
| 973 |
+
break
|
| 974 |
+
star = cfg.get("star")
|
| 975 |
+
if hasattr(policy, "start_block"):
|
| 976 |
+
policy.start_block()
|
| 977 |
+
c, drafted = root, 0
|
| 978 |
+
while drafted < cfg.get("head_budget", 8) and len(tok) < W and (c == root or tok[c] not in eos_ids):
|
| 979 |
+
exited = None
|
| 980 |
+
for k in policy.head_order:
|
| 981 |
+
if hl[k] < ld[c]:
|
| 982 |
+
continue # row already beyond this head's layer (shared copy row)
|
| 983 |
+
advance(hl[k])
|
| 984 |
+
self.hb.copy_(self.hrows[c:c + 1])
|
| 985 |
+
if star is not None and c == root and k == star.get("head", 0):
|
| 986 |
+
r = self._g_head4(k).tolist()
|
| 987 |
+
conf, t_k = r[0], int(r[1])
|
| 988 |
+
tau = star.get("tau", [0.05, 0.02, 0.02, 0.02])
|
| 989 |
+
adm = [(int(x), p, rk) for rk, (x, p, th) in enumerate(zip(r[1:5], r[5:9], tau))
|
| 990 |
+
if x >= 0 and p >= th]
|
| 991 |
+
if adm and sum(p for _, p, _ in adm) >= star.get("gamma", 0.3):
|
| 992 |
+
for x, p, rk in adm[: star.get("C", 4)]:
|
| 993 |
+
qv = float(np.interp(p, *qs[rk])) if qs is not None else 0.0
|
| 994 |
+
if x in kids[root]:
|
| 995 |
+
q[kids[root][x]] = max(q[kids[root][x]], qv)
|
| 996 |
+
elif add(x, root, "star", qv) is not None:
|
| 997 |
+
st.exits["star"] = st.exits.get("star", 0) + 1
|
| 998 |
+
else:
|
| 999 |
+
conf, t_k = self._g_head(k).tolist()
|
| 1000 |
+
t_k = int(t_k)
|
| 1001 |
+
st.head_evals += 1
|
| 1002 |
+
if conf > 0 and policy.decide(k, conf, drafted): # conf == 0: ⊥ won
|
| 1003 |
+
exited = k
|
| 1004 |
+
break
|
| 1005 |
+
if exited is None:
|
| 1006 |
+
break
|
| 1007 |
+
if gate.get("q_head") is not None: # head map refitted for the gate (pool A)
|
| 1008 |
+
q_head = float(np.interp(conf, *gate["q_head"]))
|
| 1009 |
+
else:
|
| 1010 |
+
q_head = float(policy._ahat(exited, conf)) if hasattr(policy, "_ahat") else float(conf)
|
| 1011 |
+
n = kids[c].get(t_k)
|
| 1012 |
+
if n is None:
|
| 1013 |
+
n = add(t_k, c, "head", q_head)
|
| 1014 |
+
if n is None:
|
| 1015 |
+
break
|
| 1016 |
+
st.exits[exited] = st.exits.get(exited, 0) + 1
|
| 1017 |
+
else:
|
| 1018 |
+
q[n] = max(q[n], q_head)
|
| 1019 |
+
c = n
|
| 1020 |
+
drafted += 1
|
| 1021 |
+
if t_k in eos_ids:
|
| 1022 |
+
break
|
| 1023 |
+
advance(c_max)
|
| 1024 |
+
# safety gate: keep the children of u only if p(u) = min(1, sum_v q_v) >= lam. Children
|
| 1025 |
+
# are distinct tokens and at most one equals the greedy target: disjoint events, so the
|
| 1026 |
+
# sum is the calibrated probability that one is accepted (chantier 3b §7.6)
|
| 1027 |
+
gate_p = [min(1.0, sum(q[v] for v in kids[u].values())) for u in range(len(tok))]
|
| 1028 |
+
if gate.get("g") is not None: # final isotone recalibration g(P) -> HIT (pool A)
|
| 1029 |
+
gate_p = [float(np.interp(x, *gate["g"])) if kids[u] else 0.0 for u, x in enumerate(gate_p)]
|
| 1030 |
+
keep, frontier = [root], [root]
|
| 1031 |
+
while frontier:
|
| 1032 |
+
u = frontier.pop(0)
|
| 1033 |
+
if kids[u] and gate_p[u] >= lam:
|
| 1034 |
+
ch = sorted(kids[u].values())
|
| 1035 |
+
keep += ch
|
| 1036 |
+
frontier += ch
|
| 1037 |
+
keep.sort()
|
| 1038 |
+
kept = set(keep)
|
| 1039 |
+
n_rows = len(keep)
|
| 1040 |
+
set_rows(keep)
|
| 1041 |
+
if keep == list(range(n_rows)):
|
| 1042 |
+
self.buf[:n_rows].copy_(self.hrows[:n_rows])
|
| 1043 |
+
else:
|
| 1044 |
+
self.buf[:n_rows].copy_(self.hrows.index_select(0, torch.tensor(keep, device=self.hrows.device)))
|
| 1045 |
+
tgt_rows = self._g_verify(c_max, n_rows).tolist()
|
| 1046 |
+
target = {u: tgt_rows[j] for j, u in enumerate(keep)}
|
| 1047 |
+
st.cycles += 1
|
| 1048 |
+
st.drafted += n_rows - 1
|
| 1049 |
+
st.empty_drafts += int(n_rows == 1)
|
| 1050 |
+
acc, node = [], root
|
| 1051 |
+
while True:
|
| 1052 |
+
nxt_node = kids[node].get(target[node])
|
| 1053 |
+
if nxt_node is None or nxt_node not in kept:
|
| 1054 |
+
break
|
| 1055 |
+
acc.append(nxt_node)
|
| 1056 |
+
node = nxt_node
|
| 1057 |
+
st.accepted += len(acc)
|
| 1058 |
+
st.pld_accepted += sum(1 for i in acc if origin[i] == "copy")
|
| 1059 |
+
kept_kids = [v for v in kids[node].values() if v in kept]
|
| 1060 |
+
st.rejected_cycles += int(bool(kept_kids)) # some draft below the last accepted node failed
|
| 1061 |
+
info = {"path": [origin[i] for i in acc], "n_nodes": n_rows - 1,
|
| 1062 |
+
"failed": origin[max(kept_kids, key=lambda v: q[v])] if kept_kids else None,
|
| 1063 |
+
"stop": None if kept_kids else (gate_p[node], float(target[node] in kids[node]))}
|
| 1064 |
+
src = [pos0 + i for i in acc]
|
| 1065 |
+
dst = [pos0 + 1 + d for d in range(len(acc))]
|
| 1066 |
+
if src != dst: # compaction of the accepted path (all layers, K and V)
|
| 1067 |
+
self.stats["tree_compactions"] = self.stats.get("tree_compactions", 0) + 1
|
| 1068 |
+
si = torch.tensor(src, device=self.dec.k_cache.device)
|
| 1069 |
+
di = torch.tensor(dst, device=self.dec.k_cache.device)
|
| 1070 |
+
for cache in (self.dec.k_cache, self.dec.v_cache):
|
| 1071 |
+
cache.index_copy_(2, di, cache.index_select(2, si))
|
| 1072 |
+
return [tok[i] for i in acc] + [target[node]], info
|
| 1073 |
+
|
| 1074 |
+
@torch.no_grad()
|
| 1075 |
+
def generate_spec(self, policy, prompt: list[int], max_new: int, eos_ids=frozenset(),
|
| 1076 |
+
kv_fill: str = "lazy", controller=None, predictor=None,
|
| 1077 |
+
ar_step_seconds: float | None = None, accept_rule=None, pld=None,
|
| 1078 |
+
stop=None, star=None, drafter=None, temperature: float = 0.0,
|
| 1079 |
+
gen: torch.Generator | None = None, on_commit=None) -> tuple[list[int], GenStats]:
|
| 1080 |
+
"""controller/predictor: online calibration (chantier 3b). Before each cycle the
|
| 1081 |
+
controller's lambda is turned into confidence thresholds (exit at the first head
|
| 1082 |
+
with a_hat_k >= lambda); lambda > 1 means a plain AR step (or, if the controller
|
| 1083 |
+
probes, a cycle that evaluates every head without drafting). After verification
|
| 1084 |
+
the cycle feedback (labels on the verified prefix, stop-position head outputs) is
|
| 1085 |
+
sent back. Requires kv_fill="lazy".
|
| 1086 |
+
|
| 1087 |
+
accept_rule: None (exact, lossless) or a LOSSY relaxation of greedy verification:
|
| 1088 |
+
("topk", k) accept a draft that is among the target's top-k tokens (r3)
|
| 1089 |
+
("equiv", cls) accept a draft in the same formatting class as the target
|
| 1090 |
+
argmax, cls = list mapping token id -> class id (r5)
|
| 1091 |
+
An accepted draft is committed as is; rejections and bonus use the target argmax.
|
| 1092 |
+
|
| 1093 |
+
pld: prompt lookup drafts, dict(mode="first"|"fallback", n_max=4, n_min=1, m=15); with
|
| 1094 |
+
a policy without heads, "fallback" is plain PLD. stop, on_commit: as in generate_ar.
|
| 1095 |
+
st.sources[i] is where output token i came from: "copy" (prompt lookup), "head" (head
|
| 1096 |
+
chain), "star" (top-k sibling of h0), "draft" (draft model) or "model" (prefill token,
|
| 1097 |
+
correction or bonus token of each cycle)."""
|
| 1098 |
+
from .calibration import online as onl
|
| 1099 |
+
from .pld import NgramIndex
|
| 1100 |
+
self._check_capacity(prompt, max_new)
|
| 1101 |
+
assert policy.max_draft + 1 <= self.max_rows
|
| 1102 |
+
assert controller is None or kv_fill == "lazy"
|
| 1103 |
+
st = GenStats()
|
| 1104 |
+
K = len(self.heads)
|
| 1105 |
+
gamma_only = bool(getattr(controller, "gamma_only", False))
|
| 1106 |
+
xi = float(getattr(controller, "xi", 0.0))
|
| 1107 |
+
|
| 1108 |
+
def account(cost, emitted, drafting):
|
| 1109 |
+
g = cost - (1 + xi) * emitted
|
| 1110 |
+
st.cost_units += cost
|
| 1111 |
+
st.excess_end += g
|
| 1112 |
+
st.excess_max = max(st.excess_max, st.excess_end)
|
| 1113 |
+
if drafting:
|
| 1114 |
+
st.h_max = max(st.h_max, g)
|
| 1115 |
+
if controller is not None:
|
| 1116 |
+
st.lam_start = _lam(controller)
|
| 1117 |
+
st.gamma_start = float(getattr(controller, "gamma", float("nan")))
|
| 1118 |
+
draft = self._draft_lazy if kv_fill == "lazy" else self._draft_copy
|
| 1119 |
+
seg_rows0 = self.stats["seg_rows"] # engine counters are cumulative: report this request only
|
| 1120 |
+
self._sync()
|
| 1121 |
+
t0 = time.perf_counter()
|
| 1122 |
+
self._sampling = temperature > 0
|
| 1123 |
+
if self._sampling: # exact speculative sampling on chains (heads + copies)
|
| 1124 |
+
assert controller is None and accept_rule is None and star is None and drafter is None
|
| 1125 |
+
assert getattr(policy, "tree", None) is None and kv_fill == "lazy"
|
| 1126 |
+
import random
|
| 1127 |
+
self.temperature, self._gen = temperature, gen
|
| 1128 |
+
self._rng = random.Random(gen.initial_seed() if gen is not None else 0)
|
| 1129 |
+
out = [self._first_token_sampled(prompt)]
|
| 1130 |
+
else:
|
| 1131 |
+
out = [self._prefill(prompt)]
|
| 1132 |
+
self._sync()
|
| 1133 |
+
st.prefill_seconds = time.perf_counter() - t0
|
| 1134 |
+
self._emit(st, on_commit, out[:1], ["model"])
|
| 1135 |
+
assert pld is None or (controller is None and kv_fill == "lazy")
|
| 1136 |
+
index = None
|
| 1137 |
+
if pld is not None:
|
| 1138 |
+
index = NgramIndex(pld.get("n_max", 4), pld.get("n_min", 1))
|
| 1139 |
+
index.extend(prompt)
|
| 1140 |
+
index.extend(out)
|
| 1141 |
+
stopped = stop is not None and stop(out[0])
|
| 1142 |
+
assert star is None or (controller is None and pld is None and accept_rule is None and kv_fill == "lazy")
|
| 1143 |
+
if drafter is not None: # optional prompt lookup FIRST (copy if a match exists, else the draft model)
|
| 1144 |
+
assert controller is None and kv_fill == "lazy" and (pld is None or pld.get("mode") == "first")
|
| 1145 |
+
drafter.start(prompt)
|
| 1146 |
+
tree = getattr(policy, "tree", None)
|
| 1147 |
+
if tree is not None:
|
| 1148 |
+
# controllers on a tree: a gamma-only cost safety (NoSlowdown, Prop. 4), or an ACI
|
| 1149 |
+
# on the safety gate lambda (virtual head, chantier 3b §7.6), possibly wrapped in it
|
| 1150 |
+
assert accept_rule is None and kv_fill == "lazy"
|
| 1151 |
+
if tree.get("copy") and index is None:
|
| 1152 |
+
index = NgramIndex(tree["copy"].get("n_max", 4), tree["copy"].get("n_min", 1))
|
| 1153 |
+
index.extend(prompt)
|
| 1154 |
+
index.extend(out)
|
| 1155 |
+
while tree is not None and not stopped and len(out) < max_new and out[-1] not in eos_ids:
|
| 1156 |
+
pos0 = len(prompt) + len(out) - 1
|
| 1157 |
+
lam, probe = float("-inf"), False
|
| 1158 |
+
if controller is not None: # gamma floor / gate lambda; plain AR step when asked
|
| 1159 |
+
lamv, gamma, ar = controller.policy()
|
| 1160 |
+
if not gamma_only:
|
| 1161 |
+
lam = float(np.atleast_1d(lamv)[0])
|
| 1162 |
+
if ar and controller.probe: # counterfactual probe: compute the tree, draft nothing
|
| 1163 |
+
probe, lam, ar = True, float("inf"), False
|
| 1164 |
+
if ar:
|
| 1165 |
+
self._set_pos([pos0])
|
| 1166 |
+
self.ids.fill_(out[-1])
|
| 1167 |
+
out.append(int(self._g_ar()))
|
| 1168 |
+
self._emit(st, on_commit, out[-1:], ["model"])
|
| 1169 |
+
st.cycles += 1
|
| 1170 |
+
st.ar_steps += 1
|
| 1171 |
+
controller.update(onl.Cycle([], [], 0, 0, 1, 1.0, ar=True))
|
| 1172 |
+
account(1.0, 1, False)
|
| 1173 |
+
if index is not None:
|
| 1174 |
+
index.extend([out[-1]])
|
| 1175 |
+
if stop is not None and stop(out[-1]):
|
| 1176 |
+
stopped = True
|
| 1177 |
+
continue
|
| 1178 |
+
base = getattr(policy, "_gamma_base", None)
|
| 1179 |
+
if base is None:
|
| 1180 |
+
base = policy._gamma_base = policy.cum_gamma
|
| 1181 |
+
policy.cum_gamma = ({k: max(v, float(gamma)) for k, v in base.items()} if isinstance(base, dict)
|
| 1182 |
+
else max(float(base), float(gamma)))
|
| 1183 |
+
t_cycle = time.perf_counter()
|
| 1184 |
+
prop = index.propose(tree["copy"].get("m", 15)) if tree.get("copy") else None
|
| 1185 |
+
ext = drafter.propose(prompt + out, min(max_new - len(out), self.max_rows - 1)) if drafter is not None else None
|
| 1186 |
+
committed, info = self._tree_cycle(policy, tree, out[-1], pos0, eos_ids, st, prop, lam=lam,
|
| 1187 |
+
match_n=getattr(index, "last_n", 0) if index is not None else 0,
|
| 1188 |
+
ext=ext)
|
| 1189 |
+
if drafter is not None: # draft KV valid up to the common prefix with the accepted path
|
| 1190 |
+
k = 0
|
| 1191 |
+
while k < min(len(ext), len(committed) - 1) and ext[k] == committed[k]:
|
| 1192 |
+
k += 1
|
| 1193 |
+
drafter.commit(k)
|
| 1194 |
+
if controller is not None:
|
| 1195 |
+
cost = ((time.perf_counter() - t_cycle) / ar_step_seconds) if ar_step_seconds else 1.0
|
| 1196 |
+
if gamma_only:
|
| 1197 |
+
cyc = onl.Cycle([], [], len(committed) - 1, 0, len(committed), cost)
|
| 1198 |
+
else:
|
| 1199 |
+
cyc = onl.cycle_from_tree(info["path"], info["failed"], emitted=len(committed), cost=cost,
|
| 1200 |
+
stop=info["stop"], n_nodes=info["n_nodes"])
|
| 1201 |
+
cyc.ar = probe
|
| 1202 |
+
controller.update(cyc)
|
| 1203 |
+
st.lab_n += len(cyc.labels)
|
| 1204 |
+
st.lab_s += float(sum(cyc.labels))
|
| 1205 |
+
account(cost, len(committed), True)
|
| 1206 |
+
stopped = self._commit(out, committed, info["path"] + ["model"], index, stop, eos_ids,
|
| 1207 |
+
max_new, st, on_commit) or stopped
|
| 1208 |
+
while star is not None and not stopped and len(out) < max_new and out[-1] not in eos_ids:
|
| 1209 |
+
pos0 = len(prompt) + len(out) - 1
|
| 1210 |
+
committed = self._star_cycle(star, out[-1], pos0, max_new - len(out), st)
|
| 1211 |
+
stopped = self._commit(out, committed, ["star"] * (len(committed) - 1) + ["model"], None, stop,
|
| 1212 |
+
eos_ids, max_new, st, on_commit) or stopped
|
| 1213 |
+
while not stopped and len(out) < max_new and out[-1] not in eos_ids:
|
| 1214 |
+
pos0 = len(prompt) + len(out) - 1
|
| 1215 |
+
budget = min(policy.max_draft, max_new - len(out))
|
| 1216 |
+
t_cycle = time.perf_counter()
|
| 1217 |
+
probe = False
|
| 1218 |
+
if controller is not None:
|
| 1219 |
+
taus, gamma, ar = controller.policy()
|
| 1220 |
+
if ar and not controller.probe:
|
| 1221 |
+
self._set_pos([pos0])
|
| 1222 |
+
self.ids.fill_(out[-1])
|
| 1223 |
+
out.append(int(self._g_ar()))
|
| 1224 |
+
self._emit(st, on_commit, out[-1:], ["model"])
|
| 1225 |
+
st.cycles += 1
|
| 1226 |
+
st.ar_steps += 1
|
| 1227 |
+
controller.update(onl.Cycle([], [], 0, 0, 1, 1.0, ar=True))
|
| 1228 |
+
account(1.0, 1, False)
|
| 1229 |
+
if stop is not None and stop(out[-1]): # same rule as the tree branch
|
| 1230 |
+
stopped = True
|
| 1231 |
+
continue
|
| 1232 |
+
probe = bool(ar)
|
| 1233 |
+
if not gamma_only: # gamma_only: the policy keeps its heads and a_hat thresholds
|
| 1234 |
+
tau_h = predictor.thresholds(taus) # exit iff H = -conf <= tau
|
| 1235 |
+
policy.thresholds = {k: (float("inf") if probe else float(-tau_h[k])) for k in range(K)}
|
| 1236 |
+
policy.head_order = list(range(K))
|
| 1237 |
+
if hasattr(policy, "cum_gamma"):
|
| 1238 |
+
base = getattr(policy, "_gamma_base", None)
|
| 1239 |
+
if base is None:
|
| 1240 |
+
base = policy._gamma_base = policy.cum_gamma
|
| 1241 |
+
if isinstance(base, dict): # per-head gamma_k: gamma_k <- max(gamma_k^pol, gamma_safety)
|
| 1242 |
+
policy.cum_gamma = {k: max(v, float(gamma)) for k, v in base.items()}
|
| 1243 |
+
else:
|
| 1244 |
+
policy.cum_gamma = float(gamma)
|
| 1245 |
+
if hasattr(policy, "start_block"):
|
| 1246 |
+
policy.start_block()
|
| 1247 |
+
used_drafter = False
|
| 1248 |
+
if drafter is not None:
|
| 1249 |
+
prop = index.propose(pld.get("m", 15)) if index is not None else []
|
| 1250 |
+
if not prop:
|
| 1251 |
+
prop, used_drafter = drafter.propose(prompt + out, budget), True
|
| 1252 |
+
drafts, c_verify = draft(policy, out[-1], pos0, budget, eos_ids, st, proposal=prop, pld_mode="fallback")
|
| 1253 |
+
if used_drafter: # draft-model tokens went through the copy path: count them apart
|
| 1254 |
+
nd = sum(1 for k in self._draft_heads if k < 0)
|
| 1255 |
+
st.exits["pld"] = st.exits.get("pld", 0) - nd
|
| 1256 |
+
st.exits["draft"] = st.exits.get("draft", 0) + nd
|
| 1257 |
+
elif index is not None:
|
| 1258 |
+
drafts, c_verify = draft(policy, out[-1], pos0, budget, eos_ids, st,
|
| 1259 |
+
proposal=index.propose(pld.get("m", 15)), pld_mode=pld.get("mode", "fallback"))
|
| 1260 |
+
else:
|
| 1261 |
+
drafts, c_verify = draft(policy, out[-1], pos0, budget, eos_ids, st)
|
| 1262 |
+
n = len(drafts) + 1
|
| 1263 |
+
self._set_pos(list(range(pos0, pos0 + n)))
|
| 1264 |
+
self.buf[:n].copy_(self.hrows[:n])
|
| 1265 |
+
if self._sampling:
|
| 1266 |
+
committed = self._sample_verify(c_verify, drafts, st)
|
| 1267 |
+
else:
|
| 1268 |
+
if accept_rule is None:
|
| 1269 |
+
target = self._g_verify(c_verify, n).tolist()
|
| 1270 |
+
tk = None
|
| 1271 |
+
else:
|
| 1272 |
+
rows = self._g_verify_topk(c_verify, n).tolist()
|
| 1273 |
+
target, tk = [r[0] for r in rows], [r[1:] for r in rows]
|
| 1274 |
+
st.cycles += 1
|
| 1275 |
+
st.drafted += len(drafts)
|
| 1276 |
+
st.empty_drafts += int(not drafts)
|
| 1277 |
+
committed = []
|
| 1278 |
+
for i, d in enumerate(drafts):
|
| 1279 |
+
ok = target[i] == d
|
| 1280 |
+
if not ok and accept_rule is not None:
|
| 1281 |
+
if accept_rule[0] == "topk":
|
| 1282 |
+
ok = d in tk[i][: accept_rule[1]]
|
| 1283 |
+
elif accept_rule[0] == "equiv":
|
| 1284 |
+
ok = accept_rule[1][d] == accept_rule[1][target[i]]
|
| 1285 |
+
if not ok:
|
| 1286 |
+
break
|
| 1287 |
+
committed.append(d)
|
| 1288 |
+
st.accepted += len(committed)
|
| 1289 |
+
if index is not None and not (drafter is not None and used_drafter):
|
| 1290 |
+
st.pld_drafted += sum(1 for k in self._draft_heads if k < 0)
|
| 1291 |
+
st.pld_accepted += sum(1 for k in self._draft_heads[:len(committed)] if k < 0)
|
| 1292 |
+
st.rejected_cycles += int(len(committed) < len(drafts))
|
| 1293 |
+
if hasattr(policy, "update"):
|
| 1294 |
+
policy.update(len(drafts), len(committed))
|
| 1295 |
+
if drafter is not None and used_drafter:
|
| 1296 |
+
drafter.commit(len(committed))
|
| 1297 |
+
committed.append(target[len(committed)]) # correction or bonus token
|
| 1298 |
+
if controller is not None:
|
| 1299 |
+
n_acc = st_acc = len(committed) - 1
|
| 1300 |
+
per_token = [(k, i < n_acc) for i, k in enumerate(self._draft_heads)]
|
| 1301 |
+
stop_pos = None # (a_hat, labels) at the stop position (not the `stop` rule)
|
| 1302 |
+
if (predictor is not None and n_acc == len(drafts) and self._stop_info is not None
|
| 1303 |
+
and len(self._stop_info) == K):
|
| 1304 |
+
confs = np.array([self._stop_info[k][0] for k in range(K)])
|
| 1305 |
+
toks = [self._stop_info[k][1] for k in range(K)]
|
| 1306 |
+
ahat = predictor.predict(-confs[None, :])[0]
|
| 1307 |
+
stop_pos = (ahat, np.array([float(t == target[len(drafts)]) for t in toks]))
|
| 1308 |
+
cost = ((time.perf_counter() - t_cycle) / ar_step_seconds) if ar_step_seconds else 1.0
|
| 1309 |
+
cyc = onl.cycle_from_hook(per_token, emitted=len(committed), cost=cost, stop=stop_pos)
|
| 1310 |
+
cyc.ar = probe
|
| 1311 |
+
controller.update(cyc)
|
| 1312 |
+
st.lab_n += len(cyc.labels)
|
| 1313 |
+
st.lab_s += float(sum(cyc.labels))
|
| 1314 |
+
account(cost, len(committed), True)
|
| 1315 |
+
srcs = [("draft" if used_drafter else "copy") if k < 0 else "head"
|
| 1316 |
+
for k in self._draft_heads[:len(committed) - 1]] + ["model"]
|
| 1317 |
+
stopped = self._commit(out, committed, srcs, index, stop, eos_ids, max_new, st, on_commit) or stopped
|
| 1318 |
+
self._sync()
|
| 1319 |
+
st.seconds = time.perf_counter() - t0
|
| 1320 |
+
st.new_tokens = len(out)
|
| 1321 |
+
st.layer_rows = self.stats["seg_rows"] - seg_rows0
|
| 1322 |
+
self._sampling = False
|
| 1323 |
+
if controller is not None:
|
| 1324 |
+
st.lam_end = _lam(controller)
|
| 1325 |
+
st.n_jumps = int(getattr(getattr(controller, "aci", controller), "n_jumps", -1))
|
| 1326 |
+
st.gamma_end = float(getattr(controller, "gamma", float("nan")))
|
| 1327 |
+
return out, st
|
dssd/heads.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Early-exit heads: full-vocabulary heads and pruned heads with a reject class.
|
| 2 |
+
|
| 3 |
+
A pruned head k predicts over its vocabulary V_k plus one extra output, the
|
| 4 |
+
reject class (⊥). Its training target is the full-model distribution collapsed
|
| 5 |
+
onto V_k ∪ {⊥}: p̄(v) = p(v) for v ∈ V_k and p̄(⊥) = p(V \\ V_k).
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class RMSNorm(nn.Module):
|
| 16 |
+
def __init__(self, dim: int, eps: float):
|
| 17 |
+
super().__init__()
|
| 18 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 19 |
+
self.eps = eps
|
| 20 |
+
|
| 21 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 22 |
+
dtype = x.dtype
|
| 23 |
+
x = x.float()
|
| 24 |
+
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 25 |
+
return self.weight * x.to(dtype)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class Adapter(nn.Module):
|
| 29 |
+
"""Residual low-rank MLP before an early head (Kangaroo/EAGLE-style, no attention,
|
| 30 |
+
hence no extra KV cache): h + up(silu(down(norm(h)))), identity at initialisation."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, hidden_size: int, rank: int, eps: float):
|
| 33 |
+
super().__init__()
|
| 34 |
+
self.norm = RMSNorm(hidden_size, eps)
|
| 35 |
+
self.down = nn.Linear(hidden_size, rank, bias=False)
|
| 36 |
+
self.up = nn.Linear(rank, hidden_size, bias=False)
|
| 37 |
+
nn.init.zeros_(self.up.weight)
|
| 38 |
+
|
| 39 |
+
def forward(self, h: torch.Tensor) -> torch.Tensor:
|
| 40 |
+
return h + self.up(F.silu(self.down(self.norm(h))))
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class ExitHead(nn.Module):
|
| 44 |
+
"""Norm + linear projection onto `vocab_ids` (+ optional reject class).
|
| 45 |
+
|
| 46 |
+
vocab_ids=None means the full vocabulary (output index == token id).
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(self, hidden_size: int, vocab_size: int, eps: float,
|
| 50 |
+
vocab_ids: torch.Tensor | None = None, reject: bool = False, adapter_rank: int = 0):
|
| 51 |
+
super().__init__()
|
| 52 |
+
self.adapter = Adapter(hidden_size, adapter_rank, eps) if adapter_rank > 0 else None
|
| 53 |
+
self.norm = RMSNorm(hidden_size, eps)
|
| 54 |
+
self.vocab_size = vocab_size
|
| 55 |
+
n_out = vocab_size if vocab_ids is None else int(vocab_ids.numel())
|
| 56 |
+
self.reject = reject
|
| 57 |
+
self.proj = nn.Linear(hidden_size, n_out + int(reject), bias=False)
|
| 58 |
+
# The ⊥ logit gets its own scalar bias (initialised to the log of the
|
| 59 |
+
# mean out-of-vocabulary mass when pruning).
|
| 60 |
+
self.reject_bias = nn.Parameter(torch.zeros(1)) if reject else None
|
| 61 |
+
if vocab_ids is None:
|
| 62 |
+
self.register_buffer("vocab_ids", None)
|
| 63 |
+
else:
|
| 64 |
+
self.register_buffer("vocab_ids", vocab_ids.long().clone())
|
| 65 |
+
|
| 66 |
+
@property
|
| 67 |
+
def n_vocab(self) -> int:
|
| 68 |
+
return self.proj.out_features - int(self.reject)
|
| 69 |
+
|
| 70 |
+
def forward(self, h: torch.Tensor) -> torch.Tensor:
|
| 71 |
+
"""Logits over V_k (∪ ⊥ as the last column), float32."""
|
| 72 |
+
if self.adapter is not None:
|
| 73 |
+
h = self.adapter(h)
|
| 74 |
+
logits = self.proj(self.norm(h)).float()
|
| 75 |
+
if self.reject:
|
| 76 |
+
logits = torch.cat([logits[..., :-1], logits[..., -1:] + self.reject_bias.float()], dim=-1)
|
| 77 |
+
return logits
|
| 78 |
+
|
| 79 |
+
def to_token_ids(self, idx: torch.Tensor) -> torch.Tensor:
|
| 80 |
+
# gather (not advanced indexing) so that it stays capturable in CUDA/HIP graphs
|
| 81 |
+
if self.vocab_ids is None:
|
| 82 |
+
return idx
|
| 83 |
+
return self.vocab_ids.gather(0, idx.reshape(-1)).reshape(idx.shape)
|
| 84 |
+
|
| 85 |
+
def scatter_full(self, probs_k: torch.Tensor) -> torch.Tensor:
|
| 86 |
+
"""Map probabilities over V_k (no ⊥ column) to the full vocabulary."""
|
| 87 |
+
if self.vocab_ids is None:
|
| 88 |
+
return probs_k
|
| 89 |
+
out = probs_k.new_zeros(*probs_k.shape[:-1], self.vocab_size)
|
| 90 |
+
out[..., self.vocab_ids] = probs_k
|
| 91 |
+
return out
|
| 92 |
+
|
| 93 |
+
@classmethod
|
| 94 |
+
def from_full(cls, full: "ExitHead", vocab_ids: torch.Tensor, reject: bool = True,
|
| 95 |
+
reject_bias: float = 0.0, adapter_rank: int = 0) -> "ExitHead":
|
| 96 |
+
"""Prune a full head to vocab_ids, appending a zero-weight ⊥ row (optional adapter)."""
|
| 97 |
+
assert full.vocab_ids is None
|
| 98 |
+
d = full.proj.in_features
|
| 99 |
+
head = cls(d, full.vocab_size, full.norm.eps, vocab_ids, reject, adapter_rank)
|
| 100 |
+
with torch.no_grad():
|
| 101 |
+
head.norm.weight.copy_(full.norm.weight)
|
| 102 |
+
head.proj.weight[: vocab_ids.numel()].copy_(full.proj.weight[vocab_ids])
|
| 103 |
+
if reject:
|
| 104 |
+
head.proj.weight[-1].zero_()
|
| 105 |
+
head.reject_bias.fill_(reject_bias)
|
| 106 |
+
return head
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def collapse_target(p_full: torch.Tensor, vocab_ids: torch.Tensor) -> torch.Tensor:
|
| 110 |
+
"""p̄ over V_k ∪ {⊥} from full-vocabulary probabilities (…, V)."""
|
| 111 |
+
kept = p_full[..., vocab_ids]
|
| 112 |
+
rest = (1.0 - kept.sum(-1, keepdim=True)).clamp_min(0.0)
|
| 113 |
+
return torch.cat([kept, rest], dim=-1)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def head_draft_distribution(logits: torch.Tensor, reject: bool, temperature: float):
|
| 117 |
+
"""From head logits return (q over V_k renormalised, reject prob, confidence).
|
| 118 |
+
|
| 119 |
+
The confidence is the max probability over V_k ∪ {⊥} (⊥ included in the
|
| 120 |
+
softmax), so a head that puts its mass on ⊥ is not confident. At T=0 the
|
| 121 |
+
returned q is one-hot on the argmax over V_k.
|
| 122 |
+
"""
|
| 123 |
+
probs = F.softmax(logits, dim=-1)
|
| 124 |
+
if reject:
|
| 125 |
+
p_rej = probs[..., -1]
|
| 126 |
+
conf, arg = probs.max(-1)
|
| 127 |
+
is_rej = arg == probs.shape[-1] - 1
|
| 128 |
+
conf = torch.where(is_rej, torch.zeros_like(conf), conf)
|
| 129 |
+
body_logits = logits[..., :-1]
|
| 130 |
+
else:
|
| 131 |
+
p_rej = torch.zeros(probs.shape[:-1], device=probs.device)
|
| 132 |
+
conf = probs.max(-1).values
|
| 133 |
+
body_logits = logits
|
| 134 |
+
if temperature == 0.0:
|
| 135 |
+
q = F.one_hot(body_logits.argmax(-1), body_logits.shape[-1]).float()
|
| 136 |
+
else:
|
| 137 |
+
q = F.softmax(body_logits / temperature, dim=-1)
|
| 138 |
+
return q, p_rej, conf
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def entropy(logits: torch.Tensor) -> torch.Tensor:
|
| 142 |
+
logp = F.log_softmax(logits, dim=-1)
|
| 143 |
+
return -(logp.exp() * logp).sum(-1)
|
dssd/hub.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hub format of a DSSD head set: `heads.safetensors` + `dssd_config.json`.
|
| 2 |
+
|
| 3 |
+
heads.safetensors, for every head k (bf16 except the token ids):
|
| 4 |
+
h{k}.norm.weight [d], h{k}.proj.weight [B_k + 1, d] (last row = reject class ⊥),
|
| 5 |
+
h{k}.reject_bias [1], h{k}.vocab_ids [B_k] (int32).
|
| 6 |
+
dssd_config.json: format_version, base_model, base_revision, head_layers, budgets,
|
| 7 |
+
reject_class, dtype, default_domain, domains {name: {policy, stop}}, provenance.
|
| 8 |
+
A policy uses the bench format (scripts/compose_final.py): type "threshold", thresholds,
|
| 9 |
+
cum_gamma, acc_maps {k: [knots, values]}, max_draft, tree.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
|
| 20 |
+
from .heads import ExitHead
|
| 21 |
+
|
| 22 |
+
FORMAT_VERSION = 1
|
| 23 |
+
HEADS_FILE = "heads.safetensors"
|
| 24 |
+
CONFIG_FILE = "dssd_config.json"
|
| 25 |
+
BASE_DIMS = ("hidden_size", "vocab_size", "num_hidden_layers")
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def save_heads(out_dir: str | Path, heads: list[ExitHead], layers: list[int], config: dict,
|
| 29 |
+
model_config=None) -> None:
|
| 30 |
+
"""Write heads.safetensors and dssd_config.json (format, layers, budgets and reject class
|
| 31 |
+
are filled in here; every other key comes from `config`). With the base model's config, its
|
| 32 |
+
dimensions are recorded and checked at load time."""
|
| 33 |
+
from safetensors.torch import save_file
|
| 34 |
+
out = Path(out_dir)
|
| 35 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 36 |
+
tensors = {}
|
| 37 |
+
for k, h in enumerate(heads):
|
| 38 |
+
if h.adapter is not None or h.vocab_ids is None or not h.reject:
|
| 39 |
+
raise ValueError("only pruned heads with a reject class and no adapter are supported")
|
| 40 |
+
tensors[f"h{k}.norm.weight"] = h.norm.weight.detach().to(torch.bfloat16).contiguous()
|
| 41 |
+
tensors[f"h{k}.proj.weight"] = h.proj.weight.detach().to(torch.bfloat16).contiguous()
|
| 42 |
+
tensors[f"h{k}.reject_bias"] = h.reject_bias.detach().to(torch.bfloat16).contiguous()
|
| 43 |
+
tensors[f"h{k}.vocab_ids"] = h.vocab_ids.to(torch.int32).contiguous()
|
| 44 |
+
save_file({n: t.cpu() for n, t in tensors.items()}, str(out / HEADS_FILE))
|
| 45 |
+
conf = {"format_version": FORMAT_VERSION, **config, "head_layers": [int(x) for x in layers],
|
| 46 |
+
"budgets": [h.n_vocab for h in heads], "reject_class": True, "dtype": "bfloat16"}
|
| 47 |
+
if model_config is not None:
|
| 48 |
+
conf["base_dims"] = {k: int(getattr(model_config, k)) for k in BASE_DIMS}
|
| 49 |
+
(out / CONFIG_FILE).write_text(json.dumps(conf, indent=1, ensure_ascii=False))
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _resolve(path_or_repo: str, revision: str | None, token: str | None) -> Path:
|
| 53 |
+
if os.path.isdir(path_or_repo):
|
| 54 |
+
return Path(path_or_repo)
|
| 55 |
+
from huggingface_hub import snapshot_download
|
| 56 |
+
return Path(snapshot_download(path_or_repo, revision=revision, token=token,
|
| 57 |
+
allow_patterns=[HEADS_FILE, CONFIG_FILE]))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def read_config(path_or_repo: str, revision: str | None = None, token: str | None = None) -> dict:
|
| 61 |
+
return json.loads((_resolve(path_or_repo, revision, token) / CONFIG_FILE).read_text())
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def load_heads(path_or_repo: str, model_config, device="cpu", dtype=torch.bfloat16,
|
| 65 |
+
revision: str | None = None, token: str | None = None) -> tuple[list[ExitHead], list[int], dict]:
|
| 66 |
+
"""Heads, their layers and the config from a local directory or a Hub repo. Checks that
|
| 67 |
+
they fit the base model (hidden size, vocabulary, number of layers)."""
|
| 68 |
+
from safetensors.torch import load_file
|
| 69 |
+
root = _resolve(path_or_repo, revision, token)
|
| 70 |
+
conf = json.loads((root / CONFIG_FILE).read_text())
|
| 71 |
+
if conf.get("format_version") != FORMAT_VERSION:
|
| 72 |
+
raise ValueError(f"unsupported format_version {conf.get('format_version')!r} (expected {FORMAT_VERSION})")
|
| 73 |
+
for k, v in conf.get("base_dims", {}).items(): # heads trained on another base model
|
| 74 |
+
if int(getattr(model_config, k)) != v:
|
| 75 |
+
raise ValueError(f"base model {k} {getattr(model_config, k)} != {v} recorded with the heads")
|
| 76 |
+
layers = [int(x) for x in conf["head_layers"]]
|
| 77 |
+
n_layers = model_config.num_hidden_layers
|
| 78 |
+
if any(not 0 < x < n_layers for x in layers):
|
| 79 |
+
raise ValueError(f"head layer out of range for a {n_layers}-layer base model: {layers}")
|
| 80 |
+
t = load_file(str(root / HEADS_FILE))
|
| 81 |
+
heads = []
|
| 82 |
+
for k in range(len(layers)):
|
| 83 |
+
w, ids = t[f"h{k}.proj.weight"], t[f"h{k}.vocab_ids"].long()
|
| 84 |
+
if w.shape[1] != model_config.hidden_size:
|
| 85 |
+
raise ValueError(f"head {k}: hidden_size {w.shape[1]} != base model {model_config.hidden_size}")
|
| 86 |
+
if int(ids.max()) >= model_config.vocab_size:
|
| 87 |
+
raise ValueError(f"head {k}: token id {int(ids.max())} >= base model vocab_size {model_config.vocab_size}")
|
| 88 |
+
h = ExitHead(model_config.hidden_size, model_config.vocab_size, model_config.rms_norm_eps,
|
| 89 |
+
vocab_ids=ids, reject=True)
|
| 90 |
+
h.load_state_dict({"norm.weight": t[f"h{k}.norm.weight"], "proj.weight": w,
|
| 91 |
+
"reject_bias": t[f"h{k}.reject_bias"], "vocab_ids": ids})
|
| 92 |
+
heads.append(h.to(device=device, dtype=dtype).eval())
|
| 93 |
+
return heads, layers, conf
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def heads_from_pt(path: str, model_config) -> tuple[list[ExitHead], list[int]]:
|
| 97 |
+
"""Pruned heads from a training checkpoint {'heads': [state_dict], 'meta': {'layers'}}
|
| 98 |
+
(scripts/train_pruned.py, merge_pruned_heads.py), dtype preserved."""
|
| 99 |
+
ck = torch.load(path, map_location="cpu", weights_only=False)
|
| 100 |
+
layers = [int(x) for x in ck["meta"]["layers"]]
|
| 101 |
+
heads = []
|
| 102 |
+
for k in range(len(layers)):
|
| 103 |
+
sd = ck["heads"][k]
|
| 104 |
+
if any(n.startswith("adapter.") for n in sd) or sd.get("vocab_ids") is None or "reject_bias" not in sd:
|
| 105 |
+
raise ValueError(f"head {k}: only pruned heads with a reject class and no adapter are supported")
|
| 106 |
+
n_out = sd["proj.weight"].shape[0] - 1
|
| 107 |
+
h = ExitHead(model_config.hidden_size, model_config.vocab_size, model_config.rms_norm_eps,
|
| 108 |
+
vocab_ids=sd["vocab_ids"][:n_out], reject=True)
|
| 109 |
+
h.load_state_dict(sd)
|
| 110 |
+
heads.append(h.to(sd["proj.weight"].dtype).eval())
|
| 111 |
+
return heads, layers
|
dssd/kernels.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Row-invariant decode attention (Triton), usable inside CUDA/HIP graphs.
|
| 2 |
+
|
| 3 |
+
One program per (query row, head). Row i attends to the logical keys t in [0, pos[i]]
|
| 4 |
+
of its layer; `pos` is read from device memory (graph-capturable). Optional tree
|
| 5 |
+
addressing: keys t < cl[i] are read at cache index t, keys t >= cl[i] at
|
| 6 |
+
anc[i, t - cl[i]] (the slots of the row's own path in a draft tree; its last entry
|
| 7 |
+
is the row's own slot). The summation order (blocks of BLOCK_N logical keys) does not
|
| 8 |
+
depend on the addresses, so a chain (cl = pos + 1) is bitwise unchanged and a tree
|
| 9 |
+
node gets exactly the AR arithmetic of its own path.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
import triton
|
| 16 |
+
import triton.language as tl
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@triton.jit
|
| 20 |
+
def _row_attention_kernel(Q, K, V, O, pos_ptr, cl_ptr, anc_ptr, sm_scale,
|
| 21 |
+
stride_qn, stride_qh, stride_kh, stride_kt,
|
| 22 |
+
stride_on, stride_oh,
|
| 23 |
+
GROUP: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_N: tl.constexpr,
|
| 24 |
+
MAXD: tl.constexpr):
|
| 25 |
+
i = tl.program_id(0)
|
| 26 |
+
h = tl.program_id(1)
|
| 27 |
+
kvh = h // GROUP
|
| 28 |
+
length = tl.load(pos_ptr + i).to(tl.int32) + 1
|
| 29 |
+
cl = tl.load(cl_ptr + i).to(tl.int32)
|
| 30 |
+
d = tl.arange(0, HEAD_DIM)
|
| 31 |
+
q = tl.load(Q + i * stride_qn + h * stride_qh + d).to(tl.float32) * sm_scale
|
| 32 |
+
m_i = tl.full((), -1.0e30, tl.float32)
|
| 33 |
+
l_i = tl.full((), 0.0, tl.float32)
|
| 34 |
+
acc = tl.zeros((HEAD_DIM,), tl.float32)
|
| 35 |
+
k_base = K + kvh * stride_kh
|
| 36 |
+
v_base = V + kvh * stride_kh
|
| 37 |
+
for s in range(0, length, BLOCK_N):
|
| 38 |
+
t = s + tl.arange(0, BLOCK_N)
|
| 39 |
+
mask = t < length
|
| 40 |
+
e = t - cl
|
| 41 |
+
tree = mask & (e >= 0)
|
| 42 |
+
a = tl.where(tree, tl.load(anc_ptr + i * MAXD + e, mask=tree, other=0).to(tl.int32), t)
|
| 43 |
+
k = tl.load(k_base + a[:, None] * stride_kt + d[None, :], mask=mask[:, None], other=0.0).to(tl.float32)
|
| 44 |
+
scores = tl.sum(k * q[None, :], axis=1)
|
| 45 |
+
scores = tl.where(mask, scores, -1.0e30)
|
| 46 |
+
m_new = tl.maximum(m_i, tl.max(scores, axis=0))
|
| 47 |
+
alpha = tl.exp(m_i - m_new)
|
| 48 |
+
p = tl.exp(scores - m_new)
|
| 49 |
+
p = tl.where(mask, p, 0.0)
|
| 50 |
+
v = tl.load(v_base + a[:, None] * stride_kt + d[None, :], mask=mask[:, None], other=0.0).to(tl.float32)
|
| 51 |
+
l_i = l_i * alpha + tl.sum(p, axis=0)
|
| 52 |
+
acc = acc * alpha + tl.sum(p[:, None] * v, axis=0)
|
| 53 |
+
m_i = m_new
|
| 54 |
+
out = acc / l_i
|
| 55 |
+
tl.store(O + i * stride_on + h * stride_oh + d, out.to(O.dtype.element_ty))
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def row_attention(q: torch.Tensor, k_cache: torch.Tensor, v_cache: torch.Tensor,
|
| 59 |
+
pos: torch.Tensor, scale: float, out: torch.Tensor | None = None,
|
| 60 |
+
block_n: int = 64, cl: torch.Tensor | None = None,
|
| 61 |
+
anc: torch.Tensor | None = None) -> torch.Tensor:
|
| 62 |
+
"""q: (n, H, hd); k_cache/v_cache: (Hkv, max_len, hd); pos: int tensor (>= n,) on device.
|
| 63 |
+
|
| 64 |
+
Returns (n, H, hd): row i attends to positions [0, pos[i]].
|
| 65 |
+
"""
|
| 66 |
+
n, H, hd = q.shape
|
| 67 |
+
Hkv = k_cache.shape[0]
|
| 68 |
+
assert k_cache.stride(-1) == 1 and q.stride(-1) == 1
|
| 69 |
+
if out is None:
|
| 70 |
+
out = torch.empty_like(q)
|
| 71 |
+
if cl is None: # plain chains: every key at its own position
|
| 72 |
+
cl, anc = pos + 1, pos[:, None]
|
| 73 |
+
_row_attention_kernel[(n, H)](
|
| 74 |
+
q, k_cache, v_cache, out, pos, cl, anc, scale,
|
| 75 |
+
q.stride(0), q.stride(1), k_cache.stride(0), k_cache.stride(1),
|
| 76 |
+
out.stride(0), out.stride(1),
|
| 77 |
+
GROUP=H // Hkv, HEAD_DIM=hd, BLOCK_N=block_n, MAXD=anc.shape[1],
|
| 78 |
+
)
|
| 79 |
+
return out
|
dssd/layered.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Layer-addressable decoder with a static per-layer KV cache.
|
| 2 |
+
|
| 3 |
+
The wrapper drives the submodules of a Hugging Face dense decoder (Qwen3, Llama)
|
| 4 |
+
one layer range at a time. Every layer owns a KV buffer of size `max_len`; a
|
| 5 |
+
token's K/V are written at its absolute position, so layers may hold different
|
| 6 |
+
sets of positions while drafting (early exits) and verification can overwrite
|
| 7 |
+
approximate entries in place.
|
| 8 |
+
|
| 9 |
+
Attention for a chunk of `n` new rows at positions [s, s+n) reads K/V
|
| 10 |
+
positions [0, s+n) of the same layer, with a causal mask inside the chunk.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from dataclasses import dataclass
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn.functional as F
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
| 22 |
+
x1, x2 = x.chunk(2, dim=-1)
|
| 23 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@dataclass
|
| 27 |
+
class ModelSpec:
|
| 28 |
+
num_layers: int
|
| 29 |
+
hidden_size: int
|
| 30 |
+
vocab_size: int
|
| 31 |
+
num_heads: int
|
| 32 |
+
num_kv_heads: int
|
| 33 |
+
head_dim: int
|
| 34 |
+
has_qk_norm: bool
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def use_row_invariant_blas() -> None:
|
| 38 |
+
"""Select rocBLAS on ROCm: row-invariant for 1..64 rows on MI210 (job 893629),
|
| 39 |
+
unlike hipBLASLt (o_proj breaks at 17 rows, q_proj at 24). No-op elsewhere."""
|
| 40 |
+
if torch.cuda.is_available() and torch.version.hip:
|
| 41 |
+
torch.backends.cuda.preferred_blas_library("cublas") # maps to rocBLAS on ROCm
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class LayeredDecoder:
|
| 45 |
+
"""Run a frozen HF causal LM layer by layer with a positional KV cache."""
|
| 46 |
+
|
| 47 |
+
def __init__(self, hf_model, max_len: int, canonical_attention: bool = True,
|
| 48 |
+
max_rows: int = 16):
|
| 49 |
+
self.hf = hf_model
|
| 50 |
+
# Canonical attention: every query row uses the one-query SDPA call over its
|
| 51 |
+
# exact prefix, as in one-token AR. On MI210/bf16 the batched masked call
|
| 52 |
+
# is not row-invariant while all projections and norms are (job 893599).
|
| 53 |
+
self.canonical_attention = canonical_attention
|
| 54 |
+
# GEMMs were verified row-invariant up to 16 rows (job 893599); larger
|
| 55 |
+
# chunks are processed in causal sub-chunks of at most `max_rows` rows.
|
| 56 |
+
self.max_rows = max_rows
|
| 57 |
+
base = hf_model.model
|
| 58 |
+
cfg = hf_model.config
|
| 59 |
+
self.embed_tokens = base.embed_tokens
|
| 60 |
+
self.layers = base.layers
|
| 61 |
+
self.norm = base.norm
|
| 62 |
+
self.lm_head = hf_model.lm_head
|
| 63 |
+
self.rotary_emb = base.rotary_emb
|
| 64 |
+
attn0 = self.layers[0].self_attn
|
| 65 |
+
self.spec = ModelSpec(
|
| 66 |
+
num_layers=len(self.layers),
|
| 67 |
+
hidden_size=cfg.hidden_size,
|
| 68 |
+
vocab_size=cfg.vocab_size,
|
| 69 |
+
num_heads=cfg.num_attention_heads,
|
| 70 |
+
num_kv_heads=cfg.num_key_value_heads,
|
| 71 |
+
head_dim=attn0.head_dim,
|
| 72 |
+
has_qk_norm=hasattr(attn0, "q_norm"),
|
| 73 |
+
)
|
| 74 |
+
self.scaling = attn0.scaling
|
| 75 |
+
self.max_len = max_len
|
| 76 |
+
p = next(hf_model.parameters())
|
| 77 |
+
self.device, self.dtype = p.device, p.dtype
|
| 78 |
+
shape = (self.spec.num_layers, self.spec.num_kv_heads, max_len, self.spec.head_dim)
|
| 79 |
+
self.k_cache = torch.zeros(shape, device=self.device, dtype=self.dtype)
|
| 80 |
+
self.v_cache = torch.zeros(shape, device=self.device, dtype=self.dtype)
|
| 81 |
+
# Rotary tables for every position, computed once.
|
| 82 |
+
pos = torch.arange(max_len, device=self.device).unsqueeze(0)
|
| 83 |
+
dummy = torch.zeros(1, 1, self.spec.head_dim, device=self.device, dtype=self.dtype)
|
| 84 |
+
cos, sin = self.rotary_emb(dummy, pos)
|
| 85 |
+
self.cos, self.sin = cos[0], sin[0] # (max_len, head_dim)
|
| 86 |
+
self.counters = {"layer_rows": 0, "kv_fill_rows": 0, "vocab_proj_rows": 0}
|
| 87 |
+
|
| 88 |
+
# ------------------------------------------------------------------ helpers
|
| 89 |
+
def reset_counters(self) -> None:
|
| 90 |
+
for k in self.counters:
|
| 91 |
+
self.counters[k] = 0
|
| 92 |
+
|
| 93 |
+
def embed(self, ids: torch.Tensor) -> torch.Tensor:
|
| 94 |
+
"""ids: (n,) -> (n, d)"""
|
| 95 |
+
return self.embed_tokens(ids)
|
| 96 |
+
|
| 97 |
+
def _rope(self, x: torch.Tensor, pos: torch.Tensor) -> torch.Tensor:
|
| 98 |
+
# x: (heads, n, head_dim); pos: (n,) absolute positions
|
| 99 |
+
cos = self.cos.index_select(0, pos).unsqueeze(0)
|
| 100 |
+
sin = self.sin.index_select(0, pos).unsqueeze(0)
|
| 101 |
+
return x * cos + _rotate_half(x) * sin
|
| 102 |
+
|
| 103 |
+
def _kv(self, attn, x_normed: torch.Tensor, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 104 |
+
n = x_normed.shape[0]
|
| 105 |
+
hd = self.spec.head_dim
|
| 106 |
+
k = attn.k_proj(x_normed).view(n, -1, hd)
|
| 107 |
+
if self.spec.has_qk_norm:
|
| 108 |
+
k = attn.k_norm(k)
|
| 109 |
+
k = self._rope(k.transpose(0, 1), pos)
|
| 110 |
+
v = attn.v_proj(x_normed).view(n, -1, hd).transpose(0, 1)
|
| 111 |
+
return k, v
|
| 112 |
+
|
| 113 |
+
def _positions(self, start: int, n: int, positions) -> tuple[torch.Tensor, list[int]]:
|
| 114 |
+
if positions is None:
|
| 115 |
+
plist = list(range(start, start + n))
|
| 116 |
+
else:
|
| 117 |
+
plist = [int(p) for p in positions]
|
| 118 |
+
assert len(plist) == n and all(a < b for a, b in zip(plist, plist[1:])), "ascending positions"
|
| 119 |
+
return torch.tensor(plist, device=self.device), plist
|
| 120 |
+
|
| 121 |
+
# ------------------------------------------------------------------ layers
|
| 122 |
+
def run_layers(self, h: torch.Tensor, start: int, first: int, last: int,
|
| 123 |
+
positions=None) -> torch.Tensor:
|
| 124 |
+
"""Apply layers [first, last) to hidden rows h (n, d).
|
| 125 |
+
|
| 126 |
+
Rows sit at positions [start, start+n), or at the given ascending
|
| 127 |
+
`positions` (rows of a draft block caught up to a deeper layer). K/V of
|
| 128 |
+
every processed layer are written at those positions; row i attends to
|
| 129 |
+
positions [0, pos_i] of the same layer.
|
| 130 |
+
"""
|
| 131 |
+
n = h.shape[0]
|
| 132 |
+
pos, plist = self._positions(start, n, positions)
|
| 133 |
+
if self.canonical_attention and n > self.max_rows:
|
| 134 |
+
m = self.max_rows # causal: earlier rows never depend on later rows
|
| 135 |
+
return torch.cat([self.run_layers(h[i:i + m], 0, first, last, plist[i:i + m])
|
| 136 |
+
for i in range(0, n, m)])
|
| 137 |
+
hd = self.spec.head_dim
|
| 138 |
+
end = plist[-1] + 1
|
| 139 |
+
mask = None
|
| 140 |
+
if n > 1 and not self.canonical_attention:
|
| 141 |
+
mask = torch.arange(0, end, device=h.device).unsqueeze(0) <= pos.unsqueeze(1) # (n, end)
|
| 142 |
+
for li in range(first, last):
|
| 143 |
+
layer = self.layers[li]
|
| 144 |
+
attn = layer.self_attn
|
| 145 |
+
x = layer.input_layernorm(h)
|
| 146 |
+
q = attn.q_proj(x).view(n, -1, hd)
|
| 147 |
+
if self.spec.has_qk_norm:
|
| 148 |
+
q = attn.q_norm(q)
|
| 149 |
+
q = self._rope(q.transpose(0, 1), pos)
|
| 150 |
+
k, v = self._kv(attn, x, pos)
|
| 151 |
+
self.k_cache[li].index_copy_(1, pos, k)
|
| 152 |
+
self.v_cache[li].index_copy_(1, pos, v)
|
| 153 |
+
if n == 1 or not self.canonical_attention:
|
| 154 |
+
o = F.scaled_dot_product_attention(
|
| 155 |
+
q.unsqueeze(0),
|
| 156 |
+
self.k_cache[li, :, :end].unsqueeze(0),
|
| 157 |
+
self.v_cache[li, :, :end].unsqueeze(0),
|
| 158 |
+
attn_mask=mask,
|
| 159 |
+
scale=self.scaling,
|
| 160 |
+
enable_gqa=True,
|
| 161 |
+
)[0]
|
| 162 |
+
else:
|
| 163 |
+
o = torch.cat([F.scaled_dot_product_attention(
|
| 164 |
+
q[:, i:i + 1].unsqueeze(0),
|
| 165 |
+
self.k_cache[li, :, :p + 1].unsqueeze(0),
|
| 166 |
+
self.v_cache[li, :, :p + 1].unsqueeze(0),
|
| 167 |
+
scale=self.scaling,
|
| 168 |
+
enable_gqa=True,
|
| 169 |
+
)[0] for i, p in enumerate(plist)], dim=1)
|
| 170 |
+
o = attn.o_proj(o.transpose(0, 1).reshape(n, -1))
|
| 171 |
+
h = h + o
|
| 172 |
+
h = h + layer.mlp(layer.post_attention_layernorm(h))
|
| 173 |
+
self.counters["layer_rows"] += n * (last - first)
|
| 174 |
+
return h
|
| 175 |
+
|
| 176 |
+
def fill_kv_from_hidden(self, h: torch.Tensor, start: int, first: int, last: int) -> None:
|
| 177 |
+
"""CALM-style state copy (regime B): K/V of layers [first, last) computed from h itself.
|
| 178 |
+
|
| 179 |
+
Used for tokens that exited early, so that deeper draft tokens can attend
|
| 180 |
+
to them. Only affects draft quality, never verified outputs.
|
| 181 |
+
"""
|
| 182 |
+
pos, _ = self._positions(start, h.shape[0], None)
|
| 183 |
+
for li in range(first, last):
|
| 184 |
+
layer = self.layers[li]
|
| 185 |
+
k, v = self._kv(layer.self_attn, layer.input_layernorm(h), pos)
|
| 186 |
+
self.k_cache[li].index_copy_(1, pos, k)
|
| 187 |
+
self.v_cache[li].index_copy_(1, pos, v)
|
| 188 |
+
self.counters["kv_fill_rows"] += h.shape[0] * max(0, last - first)
|
| 189 |
+
|
| 190 |
+
def final_logits(self, h: torch.Tensor) -> torch.Tensor:
|
| 191 |
+
self.counters["vocab_proj_rows"] += h.shape[0]
|
| 192 |
+
return self.lm_head(self.norm(h)).float()
|
| 193 |
+
|
| 194 |
+
# ------------------------------------------------------------------ full passes
|
| 195 |
+
def prefill(self, ids: torch.Tensor) -> torch.Tensor:
|
| 196 |
+
"""Process the whole prompt; returns last-position logits (V,).
|
| 197 |
+
|
| 198 |
+
The prompt is shared by every decoding method, so it uses the fast batched
|
| 199 |
+
attention (row invariance is only needed for generated rows).
|
| 200 |
+
"""
|
| 201 |
+
canon = self.canonical_attention
|
| 202 |
+
self.canonical_attention = False
|
| 203 |
+
try:
|
| 204 |
+
h = self.run_layers(self.embed(ids), 0, 0, self.spec.num_layers)
|
| 205 |
+
finally:
|
| 206 |
+
self.canonical_attention = canon
|
| 207 |
+
return self.final_logits(h[-1:])[0]
|
| 208 |
+
|
| 209 |
+
def full_step(self, ids: torch.Tensor, start: int) -> torch.Tensor:
|
| 210 |
+
"""All layers on rows ids at positions [start, ...); returns logits (n, V)."""
|
| 211 |
+
h = self.run_layers(self.embed(ids), start, 0, self.spec.num_layers)
|
| 212 |
+
return self.final_logits(h)
|
dssd/pld.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prompt lookup decoding (PLD): drafts copied from the context (prompt + output).
|
| 2 |
+
|
| 3 |
+
At each cycle, the longest suffix of the context of length n_max..n_min that occurred
|
| 4 |
+
earlier is matched to its MOST RECENT previous occurrence, and the m tokens that followed
|
| 5 |
+
it are proposed as drafts. The drafts are verified like any other (exact, lossless).
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class NgramIndex:
|
| 10 |
+
"""Incremental index: for n in [n_min, n_max], the last END position of every n-gram,
|
| 11 |
+
excluding the gram that ends at the current last token (so a match is always earlier)."""
|
| 12 |
+
|
| 13 |
+
def __init__(self, n_max: int = 4, n_min: int = 1):
|
| 14 |
+
self.n_max, self.n_min = n_max, n_min
|
| 15 |
+
self.seq: list[int] = []
|
| 16 |
+
self.last = {n: {} for n in range(n_min, n_max + 1)}
|
| 17 |
+
|
| 18 |
+
def extend(self, toks) -> None:
|
| 19 |
+
for t in toks:
|
| 20 |
+
self.seq.append(t)
|
| 21 |
+
e = len(self.seq) - 2 # index grams ending one before the new last token
|
| 22 |
+
if e < 0:
|
| 23 |
+
continue
|
| 24 |
+
for n in range(self.n_min, self.n_max + 1):
|
| 25 |
+
if e - n + 1 >= 0:
|
| 26 |
+
self.last[n][tuple(self.seq[e - n + 1:e + 1])] = e
|
| 27 |
+
|
| 28 |
+
def propose(self, m: int) -> list[int]:
|
| 29 |
+
"""Up to m copied tokens; self.last_n = length of the matched suffix (0 if none)."""
|
| 30 |
+
s = self.seq
|
| 31 |
+
self.last_n = 0
|
| 32 |
+
for n in range(self.n_max, self.n_min - 1, -1):
|
| 33 |
+
if len(s) < n:
|
| 34 |
+
continue
|
| 35 |
+
e = self.last[n].get(tuple(s[-n:]))
|
| 36 |
+
if e is not None:
|
| 37 |
+
self.last_n = n
|
| 38 |
+
return s[e + 1:e + 1 + m]
|
| 39 |
+
return []
|
dssd/stops.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Benchmark stop rules, applied identically to AR and speculative decoding.
|
| 2 |
+
|
| 3 |
+
Same criteria as scripts/bench_speed.StopRule (chantiers/06-freins-vitesse.md, T3): GSM8K
|
| 4 |
+
stops after the '#### <answer>' line or at the next few-shot question; HumanEval at the
|
| 5 |
+
usual function-end sequences.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import re
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class StopRule:
|
| 14 |
+
SEQS = {"gsm8k": ("\n\nQuestion:",), "humaneval": ("\ndef ", "\nclass ", "\nif __name__", "\nprint(", "\n#")}
|
| 15 |
+
|
| 16 |
+
def __init__(self, tok, bench):
|
| 17 |
+
self.tok, self.bench, self.text = tok, bench, ""
|
| 18 |
+
|
| 19 |
+
def __call__(self, t):
|
| 20 |
+
self.text += self.tok.decode([t])
|
| 21 |
+
tail = self.text[-48:]
|
| 22 |
+
if self.bench == "gsm8k" and re.search(r"####[^\n]*\d[^\n]*\n", tail):
|
| 23 |
+
return True
|
| 24 |
+
return any(s in tail for s in self.SEQS.get(self.bench, ()))
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def make_stop(tok, kind: str | None) -> StopRule | None:
|
| 28 |
+
"""A fresh stop rule (they are stateful: one per generation), or None."""
|
| 29 |
+
if kind is None:
|
| 30 |
+
return None
|
| 31 |
+
if kind not in StopRule.SEQS:
|
| 32 |
+
raise ValueError(f"unknown stop rule {kind!r}; available: {sorted(StopRule.SEQS)} or None")
|
| 33 |
+
return StopRule(tok, kind)
|
requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch==2.9.1
|
| 2 |
+
transformers==5.19.0
|
| 3 |
+
safetensors
|
| 4 |
+
huggingface_hub
|
| 5 |
+
numpy
|
| 6 |
+
scipy
|
scripts/vendor_dssd.sh
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Copy the dssd inference package into this Space (single source: BranchyLLM, branch dssd-v2/dssd-pub).
|
| 3 |
+
# Usage: scripts/vendor_dssd.sh [path/to/BranchyLLM checkout]
|
| 4 |
+
set -euo pipefail
|
| 5 |
+
SRC=${1:-../BranchyLLM-pub}
|
| 6 |
+
DST=$(cd "$(dirname "$0")/.." && pwd)/dssd
|
| 7 |
+
rm -rf "$DST"
|
| 8 |
+
mkdir -p "$DST/calibration"
|
| 9 |
+
for f in __init__ api decode fast heads hub kernels layered pld stops; do cp "$SRC/dssd/$f.py" "$DST/"; done
|
| 10 |
+
cp "$SRC/dssd/calibration/__init__.py" "$SRC/dssd/calibration/online.py" "$SRC/dssd/calibration/calib.py" "$DST/calibration/"
|
| 11 |
+
echo "Vendored from BranchyLLM $(git -C "$SRC" rev-parse --short HEAD) ($(git -C "$SRC" branch --show-current)); do not edit here." > "$DST/VENDORED.txt"
|
| 12 |
+
cat "$DST/VENDORED.txt"
|