File size: 11,551 Bytes
c8a9c5d
 
0efa6d3
c8a9c5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
"""PPAI / PP-H (PacPerlar-Hydrogen) — реализация модели.

Архитектура PPAI, спроектированная под 100M всего / 12.4M активных параметров:
  * top-1 MoE (20 роутед-экспертов) + shared-эксперт в каждом слое;
  * n-gram хэш-эмбеддинги (окна 2/3/4 токена) на входе;
  * value-residual из нулевого слоя, per-head гейт на выходе внимания;
  * RMSNorm без обучаемых смещений, qk-norm, RoPE, logit softcap.

Отличие от исходника: dtype выбирается по compute capability (на T4 fp16,
на Ampere+ bf16) прямо в коде, без внешних патчей.
"""
from __future__ import annotations

import math
from typing import Optional

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import GenerationMixin, PreTrainedModel
from transformers.cache_utils import DynamicCache
from transformers.modeling_outputs import CausalLMOutputWithPast

try:  # при загрузке через HF (trust_remote_code) модуль лежит в пакете
    from .configuration_ppai import PPAIConfig
except ImportError:  # при прямом запуске скриптом
    from configuration_ppai import PPAIConfig


def _rms(x, w, eps):
    return F.rms_norm(x, (x.shape[-1],), w.to(x.dtype), eps)


def _qk_norm(x, w, eps):
    xf = x.float()
    return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * w.float()


def _rope(x, cos, sin, dtype):  # x: (B,T,H,hd) fp32
    h = x.shape[-1] // 2
    x1, x2 = x[..., :h], x[..., h:]
    c, s = cos[None, :, None, :], sin[None, :, None, :]
    return torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], -1).to(dtype)


def _ngram_hash(ids, n, seed, nb):
    """Детерминированный причинный хэш последних n токенов (включая текущий)."""
    h = torch.full_like(ids, seed + 7919 * n)
    for k in range(n):
        xk = F.pad(ids, (k, 0))[:, : ids.shape[1]]
        h = (h * 1000003 + xk + 1) & 0x7FFFFFFF
        h = h ^ (h >> 13)
    h = (h * 40503) & 0x7FFFFFFF
    return (h >> 4) % nb


class PPAIBlock(nn.Module):
    def __init__(self, c: PPAIConfig, idx: int):
        super().__init__()
        self.c = c
        self.idx = idx
        d, hq, hkv, hd = c.d_model, c.n_heads, c.n_kv_heads, c.head_dim
        self.ln1 = nn.Parameter(torch.ones(d))
        self.ln2 = nn.Parameter(torch.ones(d))
        self.wq = nn.Parameter(torch.empty(hq * hd, d))
        self.wk = nn.Parameter(torch.empty(hkv * hd, d))
        self.wv = nn.Parameter(torch.empty(hkv * hd, d))
        self.wo = nn.Parameter(torch.empty(d, hq * hd))
        self.q_norm = nn.Parameter(torch.ones(hq, hd))
        self.k_norm = nn.Parameter(torch.ones(hkv, hd))
        self.router_w = nn.Parameter(torch.empty(c.num_experts, d))
        self.router_b = nn.Parameter(torch.zeros(c.num_experts))
        self.s13 = nn.Parameter(torch.empty(2 * c.ffn_shared_dim, d))
        self.s2 = nn.Parameter(torch.empty(d, c.ffn_shared_dim))
        self.w13 = nn.Parameter(torch.empty(c.num_experts, 2 * c.ffn_dim, d))   # fused gate|up
        self.w2 = nn.Parameter(torch.empty(c.num_experts, d, c.ffn_dim))
        self.lam = nn.Parameter(torch.tensor([1.0, 0.0]))                       # x <- lam0*x + lam1*x0
        self.ag_w = nn.Parameter(torch.zeros(hq, c.gate_dim))                    # per-head gate
        if idx > 0:
            self.vl = nn.Parameter(torch.tensor([0.5, 0.5]))                     # value residual mix
        else:
            self.vl = None

    def moe(self, m):
        """Dropless top-1 MoE. m: (N,d)."""
        logits = F.linear(m.float(), self.router_w.float(), self.router_b.float())
        probs = logits.softmax(-1)
        g, idx = probs.max(-1)
        out = torch.zeros_like(m)
        ffn = self.c.ffn_dim
        for e in idx.unique().tolist():
            sel = (idx == e).nonzero(as_tuple=True)[0]
            h = F.linear(m[sel], self.w13[e].to(m.dtype))
            y = F.linear(F.silu(h[:, :ffn]) * h[:, ffn:], self.w2[e].to(m.dtype))
            out.index_copy_(0, sel, y)
        return out * g.to(out.dtype).unsqueeze(-1)

    def forward(self, x, x0, v1, cos, sin, cache, past_len):
        c = self.c
        B, T, d = x.shape
        dt = x.dtype
        hq, hkv, hd = c.n_heads, c.n_kv_heads, c.head_dim
        x = self.lam[0].to(dt) * x + self.lam[1].to(dt) * x0
        a = _rms(x, self.ln1, c.rms_norm_eps)
        qkv = F.linear(a, torch.cat([self.wq, self.wk, self.wv], 0).to(dt))
        q, k, v = qkv.split([hq * hd, hkv * hd, hkv * hd], -1)
        q = q.reshape(B, T, hq, hd)
        k = k.reshape(B, T, hkv, hd)
        v = v.reshape(B, T, hkv, hd)
        v_raw = v
        if self.vl is not None:
            v = self.vl[0].to(dt) * v + self.vl[1].to(dt) * v1
        q = _rope(_qk_norm(q, self.q_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2)
        k = _rope(_qk_norm(k, self.k_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2)
        v = v.transpose(1, 2)
        if cache is not None:
            k, v = cache.update(k, v, self.idx)
        rep = hq // hkv
        k = k.repeat_interleave(rep, dim=1)
        v = v.repeat_interleave(rep, dim=1)
        if T == 1:
            o = F.scaled_dot_product_attention(q, k, v)
        elif past_len == 0:
            o = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        else:
            Tk = k.shape[2]
            mask = torch.ones(T, Tk, dtype=torch.bool, device=q.device).tril(diagonal=Tk - T)
            o = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
        gate = 2.0 * torch.sigmoid(F.linear(a[..., : c.gate_dim], self.ag_w.to(dt)))
        o = o.transpose(1, 2) * gate.unsqueeze(-1)
        x = x + F.linear(o.reshape(B, T, hq * hd), self.wo.to(dt))
        m = _rms(x, self.ln2, c.rms_norm_eps)
        sg, su = F.linear(m, self.s13.to(dt)).chunk(2, -1)
        s = F.linear(F.silu(sg) * su, self.s2.to(dt))
        e = self.moe(m.reshape(B * T, d)).view(B, T, d)
        return x + s + e, v_raw


class PPAIPreTrainedModel(PreTrainedModel):
    config_class = PPAIConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = False
    _no_split_modules = ["PPAIBlock"]

    @torch.no_grad()
    def _init_weights(self, module):
        c = self.config
        std, ostd = 0.02, 0.02 / math.sqrt(2 * c.n_layers)
        if isinstance(module, nn.Embedding):
            module.weight.normal_(0, std)
        elif isinstance(module, PPAIBlock):
            for p in (module.wq, module.wk, module.wv, module.s13, module.w13):
                p.normal_(0, std)
            for p in (module.wo, module.s2, module.w2):
                p.normal_(0, ostd)
            module.router_w.normal_(0, 0.01)
            module.router_b.zero_()
            module.ln1.fill_(1.0)
            module.ln2.fill_(1.0)
            module.q_norm.fill_(1.0)
            module.k_norm.fill_(1.0)
            module.lam.copy_(torch.tensor([1.0, 0.0]))
            module.ag_w.zero_()
            if module.vl is not None:
                module.vl.copy_(torch.tensor([0.5, 0.5]))
        elif isinstance(module, PPAIModel):
            module.ng_emb.normal_(0, std)
            module.ng_proj.normal_(0, std)
            module.ng_gate.fill_(0.1)
            module.final_norm.fill_(1.0)


class PPAIModel(PPAIPreTrainedModel):
    def __init__(self, c: PPAIConfig):
        super().__init__(c)
        self.tok_emb = nn.Embedding(c.vocab_size, c.d_model)          # tied with LM head
        self.ng_emb = nn.Parameter(torch.empty(len(c.ngram_sizes), c.ngram_buckets, c.ngram_dim))
        self.ng_proj = nn.Parameter(torch.empty(c.d_model, len(c.ngram_sizes) * c.ngram_dim))
        self.ng_gate = nn.Parameter(torch.full((c.d_model,), 0.1))
        self.final_norm = nn.Parameter(torch.ones(c.d_model))
        self.blocks = nn.ModuleList(PPAIBlock(c, i) for i in range(c.n_layers))
        self.post_init()

    def forward(self, input_ids, past_key_values=None, use_cache=False):
        c = self.config
        B, T = input_ids.shape
        dev = input_ids.device
        # bf16 только там, где он аппаратный; на T4 (sm_75) — fp16
        if dev.type == "cuda" and torch.cuda.get_device_capability(dev)[0] >= 8:
            compute_dtype = torch.bfloat16
        elif dev.type == "cuda":
            compute_dtype = torch.float16
        else:
            compute_dtype = torch.float32

        ids = input_ids.long()
        if use_cache and past_key_values is None:
            past_key_values = DynamicCache()
        cache = past_key_values if use_cache else None
        past_len = cache.get_seq_length() if cache is not None else 0

        # n-gram признаки требуют до 3 предыдущих токенов
        hist = getattr(cache, "_ppai_hist", None) if cache is not None else None
        full = ids if hist is None else torch.cat([hist.to(dev), ids], 1)
        feats = [F.embedding(_ngram_hash(full, n, c.ngram_seed, c.ngram_buckets)[:, -T:],
                             self.ng_emb[i]).float()
                 for i, n in enumerate(c.ngram_sizes)]
        if cache is not None:
            cache._ppai_hist = full[:, -3:].clone()
        x = self.tok_emb(ids).float() + self.ng_gate.float() * F.linear(torch.cat(feats, -1), self.ng_proj.float())
        x = x.to(compute_dtype)
        x0 = x

        pos = torch.arange(past_len, past_len + T, device=dev).float()
        inv = 1.0 / (c.rope_theta ** (torch.arange(0, c.head_dim, 2, device=dev).float() / c.head_dim))
        fr = torch.outer(pos, inv)
        cos, sin = fr.cos(), fr.sin()

        v1 = None
        for i, blk in enumerate(self.blocks):
            x, v_raw = blk(x, x0, v1, cos, sin, cache, past_len)
            if i == 0:
                v1 = v_raw
        return _rms(x, self.final_norm, c.rms_norm_eps), past_key_values


class PPAIForCausalLM(PPAIPreTrainedModel, GenerationMixin):
    def __init__(self, config: PPAIConfig):
        super().__init__(config)
        self.model = PPAIModel(config)
        self.post_init()

    def get_input_embeddings(self):
        return self.model.tok_emb

    def set_input_embeddings(self, value):
        self.model.tok_emb = value

    def get_output_embeddings(self):
        return None

    def forward(self, input_ids=None, attention_mask=None, past_key_values=None, labels=None,
                use_cache=None, logits_to_keep=0, return_dict=None, **kwargs):
        use_cache = self.config.use_cache if use_cache is None else use_cache
        logits_to_keep = kwargs.pop("num_logits_to_keep", logits_to_keep) or 0
        h, pkv = self.model(input_ids, past_key_values=past_key_values, use_cache=use_cache)
        if isinstance(logits_to_keep, int) and logits_to_keep > 0:
            h = h[:, -logits_to_keep:]
        logits = F.linear(h, self.model.tok_emb.weight.to(h.dtype)).float()
        cap = self.config.logit_softcap
        if cap and cap > 0:
            logits = cap * torch.tanh(logits / cap)
        loss = None
        if labels is not None:
            loss = F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]),
                                   labels[:, 1:].reshape(-1), ignore_index=-100)
        return CausalLMOutputWithPast(loss=loss, logits=logits,
                                      past_key_values=pkv if use_cache else None)