File size: 9,456 Bytes
74679d4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
"""Layer-addressable decoder with a static per-layer KV cache.

The wrapper drives the submodules of a Hugging Face dense decoder (Qwen3, Llama)
one layer range at a time. Every layer owns a KV buffer of size `max_len`; a
token's K/V are written at its absolute position, so layers may hold different
sets of positions while drafting (early exits) and verification can overwrite
approximate entries in place.

Attention for a chunk of `n` new rows at positions [s, s+n) reads K/V
positions [0, s+n) of the same layer, with a causal mask inside the chunk.
"""

from __future__ import annotations

from dataclasses import dataclass

import torch
import torch.nn.functional as F


def _rotate_half(x: torch.Tensor) -> torch.Tensor:
    x1, x2 = x.chunk(2, dim=-1)
    return torch.cat((-x2, x1), dim=-1)


@dataclass
class ModelSpec:
    num_layers: int
    hidden_size: int
    vocab_size: int
    num_heads: int
    num_kv_heads: int
    head_dim: int
    has_qk_norm: bool


def use_row_invariant_blas() -> None:
    """Select rocBLAS on ROCm: row-invariant for 1..64 rows on MI210 (job 893629),
    unlike hipBLASLt (o_proj breaks at 17 rows, q_proj at 24). No-op elsewhere."""
    if torch.cuda.is_available() and torch.version.hip:
        torch.backends.cuda.preferred_blas_library("cublas")  # maps to rocBLAS on ROCm


class LayeredDecoder:
    """Run a frozen HF causal LM layer by layer with a positional KV cache."""

    def __init__(self, hf_model, max_len: int, canonical_attention: bool = True,
                 max_rows: int = 16):
        self.hf = hf_model
        # Canonical attention: every query row uses the one-query SDPA call over its
        # exact prefix, as in one-token AR. On MI210/bf16 the batched masked call
        # is not row-invariant while all projections and norms are (job 893599).
        self.canonical_attention = canonical_attention
        # GEMMs were verified row-invariant up to 16 rows (job 893599); larger
        # chunks are processed in causal sub-chunks of at most `max_rows` rows.
        self.max_rows = max_rows
        base = hf_model.model
        cfg = hf_model.config
        self.embed_tokens = base.embed_tokens
        self.layers = base.layers
        self.norm = base.norm
        self.lm_head = hf_model.lm_head
        self.rotary_emb = base.rotary_emb
        attn0 = self.layers[0].self_attn
        self.spec = ModelSpec(
            num_layers=len(self.layers),
            hidden_size=cfg.hidden_size,
            vocab_size=cfg.vocab_size,
            num_heads=cfg.num_attention_heads,
            num_kv_heads=cfg.num_key_value_heads,
            head_dim=attn0.head_dim,
            has_qk_norm=hasattr(attn0, "q_norm"),
        )
        self.scaling = attn0.scaling
        self.max_len = max_len
        p = next(hf_model.parameters())
        self.device, self.dtype = p.device, p.dtype
        shape = (self.spec.num_layers, self.spec.num_kv_heads, max_len, self.spec.head_dim)
        self.k_cache = torch.zeros(shape, device=self.device, dtype=self.dtype)
        self.v_cache = torch.zeros(shape, device=self.device, dtype=self.dtype)
        # Rotary tables for every position, computed once.
        pos = torch.arange(max_len, device=self.device).unsqueeze(0)
        dummy = torch.zeros(1, 1, self.spec.head_dim, device=self.device, dtype=self.dtype)
        cos, sin = self.rotary_emb(dummy, pos)
        self.cos, self.sin = cos[0], sin[0]  # (max_len, head_dim)
        self.counters = {"layer_rows": 0, "kv_fill_rows": 0, "vocab_proj_rows": 0}

    # ------------------------------------------------------------------ helpers
    def reset_counters(self) -> None:
        for k in self.counters:
            self.counters[k] = 0

    def embed(self, ids: torch.Tensor) -> torch.Tensor:
        """ids: (n,) -> (n, d)"""
        return self.embed_tokens(ids)

    def _rope(self, x: torch.Tensor, pos: torch.Tensor) -> torch.Tensor:
        # x: (heads, n, head_dim); pos: (n,) absolute positions
        cos = self.cos.index_select(0, pos).unsqueeze(0)
        sin = self.sin.index_select(0, pos).unsqueeze(0)
        return x * cos + _rotate_half(x) * sin

    def _kv(self, attn, x_normed: torch.Tensor, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
        n = x_normed.shape[0]
        hd = self.spec.head_dim
        k = attn.k_proj(x_normed).view(n, -1, hd)
        if self.spec.has_qk_norm:
            k = attn.k_norm(k)
        k = self._rope(k.transpose(0, 1), pos)
        v = attn.v_proj(x_normed).view(n, -1, hd).transpose(0, 1)
        return k, v

    def _positions(self, start: int, n: int, positions) -> tuple[torch.Tensor, list[int]]:
        if positions is None:
            plist = list(range(start, start + n))
        else:
            plist = [int(p) for p in positions]
            assert len(plist) == n and all(a < b for a, b in zip(plist, plist[1:])), "ascending positions"
        return torch.tensor(plist, device=self.device), plist

    # ------------------------------------------------------------------ layers
    def run_layers(self, h: torch.Tensor, start: int, first: int, last: int,
                   positions=None) -> torch.Tensor:
        """Apply layers [first, last) to hidden rows h (n, d).

        Rows sit at positions [start, start+n), or at the given ascending
        `positions` (rows of a draft block caught up to a deeper layer). K/V of
        every processed layer are written at those positions; row i attends to
        positions [0, pos_i] of the same layer.
        """
        n = h.shape[0]
        pos, plist = self._positions(start, n, positions)
        if self.canonical_attention and n > self.max_rows:
            m = self.max_rows  # causal: earlier rows never depend on later rows
            return torch.cat([self.run_layers(h[i:i + m], 0, first, last, plist[i:i + m])
                              for i in range(0, n, m)])
        hd = self.spec.head_dim
        end = plist[-1] + 1
        mask = None
        if n > 1 and not self.canonical_attention:
            mask = torch.arange(0, end, device=h.device).unsqueeze(0) <= pos.unsqueeze(1)  # (n, end)
        for li in range(first, last):
            layer = self.layers[li]
            attn = layer.self_attn
            x = layer.input_layernorm(h)
            q = attn.q_proj(x).view(n, -1, hd)
            if self.spec.has_qk_norm:
                q = attn.q_norm(q)
            q = self._rope(q.transpose(0, 1), pos)
            k, v = self._kv(attn, x, pos)
            self.k_cache[li].index_copy_(1, pos, k)
            self.v_cache[li].index_copy_(1, pos, v)
            if n == 1 or not self.canonical_attention:
                o = F.scaled_dot_product_attention(
                    q.unsqueeze(0),
                    self.k_cache[li, :, :end].unsqueeze(0),
                    self.v_cache[li, :, :end].unsqueeze(0),
                    attn_mask=mask,
                    scale=self.scaling,
                    enable_gqa=True,
                )[0]
            else:
                o = torch.cat([F.scaled_dot_product_attention(
                    q[:, i:i + 1].unsqueeze(0),
                    self.k_cache[li, :, :p + 1].unsqueeze(0),
                    self.v_cache[li, :, :p + 1].unsqueeze(0),
                    scale=self.scaling,
                    enable_gqa=True,
                )[0] for i, p in enumerate(plist)], dim=1)
            o = attn.o_proj(o.transpose(0, 1).reshape(n, -1))
            h = h + o
            h = h + layer.mlp(layer.post_attention_layernorm(h))
        self.counters["layer_rows"] += n * (last - first)
        return h

    def fill_kv_from_hidden(self, h: torch.Tensor, start: int, first: int, last: int) -> None:
        """CALM-style state copy (regime B): K/V of layers [first, last) computed from h itself.

        Used for tokens that exited early, so that deeper draft tokens can attend
        to them. Only affects draft quality, never verified outputs.
        """
        pos, _ = self._positions(start, h.shape[0], None)
        for li in range(first, last):
            layer = self.layers[li]
            k, v = self._kv(layer.self_attn, layer.input_layernorm(h), pos)
            self.k_cache[li].index_copy_(1, pos, k)
            self.v_cache[li].index_copy_(1, pos, v)
        self.counters["kv_fill_rows"] += h.shape[0] * max(0, last - first)

    def final_logits(self, h: torch.Tensor) -> torch.Tensor:
        self.counters["vocab_proj_rows"] += h.shape[0]
        return self.lm_head(self.norm(h)).float()

    # ------------------------------------------------------------------ full passes
    def prefill(self, ids: torch.Tensor) -> torch.Tensor:
        """Process the whole prompt; returns last-position logits (V,).

        The prompt is shared by every decoding method, so it uses the fast batched
        attention (row invariance is only needed for generated rows).
        """
        canon = self.canonical_attention
        self.canonical_attention = False
        try:
            h = self.run_layers(self.embed(ids), 0, 0, self.spec.num_layers)
        finally:
            self.canonical_attention = canon
        return self.final_logits(h[-1:])[0]

    def full_step(self, ids: torch.Tensor, start: int) -> torch.Tensor:
        """All layers on rows ids at positions [start, ...); returns logits (n, V)."""
        h = self.run_layers(self.embed(ids), start, 0, self.spec.num_layers)
        return self.final_logits(h)