File size: 5,864 Bytes
74679d4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
"""Early-exit heads: full-vocabulary heads and pruned heads with a reject class.

A pruned head k predicts over its vocabulary V_k plus one extra output, the
reject class (⊥). Its training target is the full-model distribution collapsed
onto V_k ∪ {⊥}: p̄(v) = p(v) for v ∈ V_k and p̄(⊥) = p(V \\ V_k).
"""

from __future__ import annotations

import torch
import torch.nn as nn
import torch.nn.functional as F


class RMSNorm(nn.Module):
    def __init__(self, dim: int, eps: float):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(dim))
        self.eps = eps

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        dtype = x.dtype
        x = x.float()
        x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
        return self.weight * x.to(dtype)


class Adapter(nn.Module):
    """Residual low-rank MLP before an early head (Kangaroo/EAGLE-style, no attention,
    hence no extra KV cache): h + up(silu(down(norm(h)))), identity at initialisation."""

    def __init__(self, hidden_size: int, rank: int, eps: float):
        super().__init__()
        self.norm = RMSNorm(hidden_size, eps)
        self.down = nn.Linear(hidden_size, rank, bias=False)
        self.up = nn.Linear(rank, hidden_size, bias=False)
        nn.init.zeros_(self.up.weight)

    def forward(self, h: torch.Tensor) -> torch.Tensor:
        return h + self.up(F.silu(self.down(self.norm(h))))


class ExitHead(nn.Module):
    """Norm + linear projection onto `vocab_ids` (+ optional reject class).

    vocab_ids=None means the full vocabulary (output index == token id).
    """

    def __init__(self, hidden_size: int, vocab_size: int, eps: float,
                 vocab_ids: torch.Tensor | None = None, reject: bool = False, adapter_rank: int = 0):
        super().__init__()
        self.adapter = Adapter(hidden_size, adapter_rank, eps) if adapter_rank > 0 else None
        self.norm = RMSNorm(hidden_size, eps)
        self.vocab_size = vocab_size
        n_out = vocab_size if vocab_ids is None else int(vocab_ids.numel())
        self.reject = reject
        self.proj = nn.Linear(hidden_size, n_out + int(reject), bias=False)
        # The ⊥ logit gets its own scalar bias (initialised to the log of the
        # mean out-of-vocabulary mass when pruning).
        self.reject_bias = nn.Parameter(torch.zeros(1)) if reject else None
        if vocab_ids is None:
            self.register_buffer("vocab_ids", None)
        else:
            self.register_buffer("vocab_ids", vocab_ids.long().clone())

    @property
    def n_vocab(self) -> int:
        return self.proj.out_features - int(self.reject)

    def forward(self, h: torch.Tensor) -> torch.Tensor:
        """Logits over V_k (∪ ⊥ as the last column), float32."""
        if self.adapter is not None:
            h = self.adapter(h)
        logits = self.proj(self.norm(h)).float()
        if self.reject:
            logits = torch.cat([logits[..., :-1], logits[..., -1:] + self.reject_bias.float()], dim=-1)
        return logits

    def to_token_ids(self, idx: torch.Tensor) -> torch.Tensor:
        # gather (not advanced indexing) so that it stays capturable in CUDA/HIP graphs
        if self.vocab_ids is None:
            return idx
        return self.vocab_ids.gather(0, idx.reshape(-1)).reshape(idx.shape)

    def scatter_full(self, probs_k: torch.Tensor) -> torch.Tensor:
        """Map probabilities over V_k (no ⊥ column) to the full vocabulary."""
        if self.vocab_ids is None:
            return probs_k
        out = probs_k.new_zeros(*probs_k.shape[:-1], self.vocab_size)
        out[..., self.vocab_ids] = probs_k
        return out

    @classmethod
    def from_full(cls, full: "ExitHead", vocab_ids: torch.Tensor, reject: bool = True,
                  reject_bias: float = 0.0, adapter_rank: int = 0) -> "ExitHead":
        """Prune a full head to vocab_ids, appending a zero-weight ⊥ row (optional adapter)."""
        assert full.vocab_ids is None
        d = full.proj.in_features
        head = cls(d, full.vocab_size, full.norm.eps, vocab_ids, reject, adapter_rank)
        with torch.no_grad():
            head.norm.weight.copy_(full.norm.weight)
            head.proj.weight[: vocab_ids.numel()].copy_(full.proj.weight[vocab_ids])
            if reject:
                head.proj.weight[-1].zero_()
                head.reject_bias.fill_(reject_bias)
        return head


def collapse_target(p_full: torch.Tensor, vocab_ids: torch.Tensor) -> torch.Tensor:
    """p̄ over V_k ∪ {⊥} from full-vocabulary probabilities (…, V)."""
    kept = p_full[..., vocab_ids]
    rest = (1.0 - kept.sum(-1, keepdim=True)).clamp_min(0.0)
    return torch.cat([kept, rest], dim=-1)


def head_draft_distribution(logits: torch.Tensor, reject: bool, temperature: float):
    """From head logits return (q over V_k renormalised, reject prob, confidence).

    The confidence is the max probability over V_k ∪ {⊥} (⊥ included in the
    softmax), so a head that puts its mass on ⊥ is not confident. At T=0 the
    returned q is one-hot on the argmax over V_k.
    """
    probs = F.softmax(logits, dim=-1)
    if reject:
        p_rej = probs[..., -1]
        conf, arg = probs.max(-1)
        is_rej = arg == probs.shape[-1] - 1
        conf = torch.where(is_rej, torch.zeros_like(conf), conf)
        body_logits = logits[..., :-1]
    else:
        p_rej = torch.zeros(probs.shape[:-1], device=probs.device)
        conf = probs.max(-1).values
        body_logits = logits
    if temperature == 0.0:
        q = F.one_hot(body_logits.argmax(-1), body_logits.shape[-1]).float()
    else:
        q = F.softmax(body_logits / temperature, dim=-1)
    return q, p_rej, conf


def entropy(logits: torch.Tensor) -> torch.Tensor:
    logp = F.log_softmax(logits, dim=-1)
    return -(logp.exp() * logp).sum(-1)