"""PPAI / PP-H (PacPerlar-Hydrogen) — реализация модели. Архитектура PPAI, спроектированная под 100M всего / 12.4M активных параметров: * top-1 MoE (20 роутед-экспертов) + shared-эксперт в каждом слое; * n-gram хэш-эмбеддинги (окна 2/3/4 токена) на входе; * value-residual из нулевого слоя, per-head гейт на выходе внимания; * RMSNorm без обучаемых смещений, qk-norm, RoPE, logit softcap. Отличие от исходника: dtype выбирается по compute capability (на T4 fp16, на Ampere+ bf16) прямо в коде, без внешних патчей. """ from __future__ import annotations import math from typing import Optional import torch import torch.nn as nn import torch.nn.functional as F from transformers import GenerationMixin, PreTrainedModel from transformers.cache_utils import DynamicCache from transformers.modeling_outputs import CausalLMOutputWithPast try: # при загрузке через HF (trust_remote_code) модуль лежит в пакете from .configuration_ppai import PPAIConfig except ImportError: # при прямом запуске скриптом from configuration_ppai import PPAIConfig def _rms(x, w, eps): return F.rms_norm(x, (x.shape[-1],), w.to(x.dtype), eps) def _qk_norm(x, w, eps): xf = x.float() return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * w.float() def _rope(x, cos, sin, dtype): # x: (B,T,H,hd) fp32 h = x.shape[-1] // 2 x1, x2 = x[..., :h], x[..., h:] c, s = cos[None, :, None, :], sin[None, :, None, :] return torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], -1).to(dtype) def _ngram_hash(ids, n, seed, nb): """Детерминированный причинный хэш последних n токенов (включая текущий).""" h = torch.full_like(ids, seed + 7919 * n) for k in range(n): xk = F.pad(ids, (k, 0))[:, : ids.shape[1]] h = (h * 1000003 + xk + 1) & 0x7FFFFFFF h = h ^ (h >> 13) h = (h * 40503) & 0x7FFFFFFF return (h >> 4) % nb class PPAIBlock(nn.Module): def __init__(self, c: PPAIConfig, idx: int): super().__init__() self.c = c self.idx = idx d, hq, hkv, hd = c.d_model, c.n_heads, c.n_kv_heads, c.head_dim self.ln1 = nn.Parameter(torch.ones(d)) self.ln2 = nn.Parameter(torch.ones(d)) self.wq = nn.Parameter(torch.empty(hq * hd, d)) self.wk = nn.Parameter(torch.empty(hkv * hd, d)) self.wv = nn.Parameter(torch.empty(hkv * hd, d)) self.wo = nn.Parameter(torch.empty(d, hq * hd)) self.q_norm = nn.Parameter(torch.ones(hq, hd)) self.k_norm = nn.Parameter(torch.ones(hkv, hd)) self.router_w = nn.Parameter(torch.empty(c.num_experts, d)) self.router_b = nn.Parameter(torch.zeros(c.num_experts)) self.s13 = nn.Parameter(torch.empty(2 * c.ffn_shared_dim, d)) self.s2 = nn.Parameter(torch.empty(d, c.ffn_shared_dim)) self.w13 = nn.Parameter(torch.empty(c.num_experts, 2 * c.ffn_dim, d)) # fused gate|up self.w2 = nn.Parameter(torch.empty(c.num_experts, d, c.ffn_dim)) self.lam = nn.Parameter(torch.tensor([1.0, 0.0])) # x <- lam0*x + lam1*x0 self.ag_w = nn.Parameter(torch.zeros(hq, c.gate_dim)) # per-head gate if idx > 0: self.vl = nn.Parameter(torch.tensor([0.5, 0.5])) # value residual mix else: self.vl = None def moe(self, m): """Dropless top-1 MoE. m: (N,d).""" logits = F.linear(m.float(), self.router_w.float(), self.router_b.float()) probs = logits.softmax(-1) g, idx = probs.max(-1) out = torch.zeros_like(m) ffn = self.c.ffn_dim for e in idx.unique().tolist(): sel = (idx == e).nonzero(as_tuple=True)[0] h = F.linear(m[sel], self.w13[e].to(m.dtype)) y = F.linear(F.silu(h[:, :ffn]) * h[:, ffn:], self.w2[e].to(m.dtype)) out.index_copy_(0, sel, y) return out * g.to(out.dtype).unsqueeze(-1) def forward(self, x, x0, v1, cos, sin, cache, past_len): c = self.c B, T, d = x.shape dt = x.dtype hq, hkv, hd = c.n_heads, c.n_kv_heads, c.head_dim x = self.lam[0].to(dt) * x + self.lam[1].to(dt) * x0 a = _rms(x, self.ln1, c.rms_norm_eps) qkv = F.linear(a, torch.cat([self.wq, self.wk, self.wv], 0).to(dt)) q, k, v = qkv.split([hq * hd, hkv * hd, hkv * hd], -1) q = q.reshape(B, T, hq, hd) k = k.reshape(B, T, hkv, hd) v = v.reshape(B, T, hkv, hd) v_raw = v if self.vl is not None: v = self.vl[0].to(dt) * v + self.vl[1].to(dt) * v1 q = _rope(_qk_norm(q, self.q_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2) k = _rope(_qk_norm(k, self.k_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2) v = v.transpose(1, 2) if cache is not None: k, v = cache.update(k, v, self.idx) rep = hq // hkv k = k.repeat_interleave(rep, dim=1) v = v.repeat_interleave(rep, dim=1) if T == 1: o = F.scaled_dot_product_attention(q, k, v) elif past_len == 0: o = F.scaled_dot_product_attention(q, k, v, is_causal=True) else: Tk = k.shape[2] mask = torch.ones(T, Tk, dtype=torch.bool, device=q.device).tril(diagonal=Tk - T) o = F.scaled_dot_product_attention(q, k, v, attn_mask=mask) gate = 2.0 * torch.sigmoid(F.linear(a[..., : c.gate_dim], self.ag_w.to(dt))) o = o.transpose(1, 2) * gate.unsqueeze(-1) x = x + F.linear(o.reshape(B, T, hq * hd), self.wo.to(dt)) m = _rms(x, self.ln2, c.rms_norm_eps) sg, su = F.linear(m, self.s13.to(dt)).chunk(2, -1) s = F.linear(F.silu(sg) * su, self.s2.to(dt)) e = self.moe(m.reshape(B * T, d)).view(B, T, d) return x + s + e, v_raw class PPAIPreTrainedModel(PreTrainedModel): config_class = PPAIConfig base_model_prefix = "model" supports_gradient_checkpointing = False _no_split_modules = ["PPAIBlock"] @torch.no_grad() def _init_weights(self, module): c = self.config std, ostd = 0.02, 0.02 / math.sqrt(2 * c.n_layers) if isinstance(module, nn.Embedding): module.weight.normal_(0, std) elif isinstance(module, PPAIBlock): for p in (module.wq, module.wk, module.wv, module.s13, module.w13): p.normal_(0, std) for p in (module.wo, module.s2, module.w2): p.normal_(0, ostd) module.router_w.normal_(0, 0.01) module.router_b.zero_() module.ln1.fill_(1.0) module.ln2.fill_(1.0) module.q_norm.fill_(1.0) module.k_norm.fill_(1.0) module.lam.copy_(torch.tensor([1.0, 0.0])) module.ag_w.zero_() if module.vl is not None: module.vl.copy_(torch.tensor([0.5, 0.5])) elif isinstance(module, PPAIModel): module.ng_emb.normal_(0, std) module.ng_proj.normal_(0, std) module.ng_gate.fill_(0.1) module.final_norm.fill_(1.0) class PPAIModel(PPAIPreTrainedModel): def __init__(self, c: PPAIConfig): super().__init__(c) self.tok_emb = nn.Embedding(c.vocab_size, c.d_model) # tied with LM head self.ng_emb = nn.Parameter(torch.empty(len(c.ngram_sizes), c.ngram_buckets, c.ngram_dim)) self.ng_proj = nn.Parameter(torch.empty(c.d_model, len(c.ngram_sizes) * c.ngram_dim)) self.ng_gate = nn.Parameter(torch.full((c.d_model,), 0.1)) self.final_norm = nn.Parameter(torch.ones(c.d_model)) self.blocks = nn.ModuleList(PPAIBlock(c, i) for i in range(c.n_layers)) self.post_init() def forward(self, input_ids, past_key_values=None, use_cache=False): c = self.config B, T = input_ids.shape dev = input_ids.device # bf16 только там, где он аппаратный; на T4 (sm_75) — fp16 if dev.type == "cuda" and torch.cuda.get_device_capability(dev)[0] >= 8: compute_dtype = torch.bfloat16 elif dev.type == "cuda": compute_dtype = torch.float16 else: compute_dtype = torch.float32 ids = input_ids.long() if use_cache and past_key_values is None: past_key_values = DynamicCache() cache = past_key_values if use_cache else None past_len = cache.get_seq_length() if cache is not None else 0 # n-gram признаки требуют до 3 предыдущих токенов hist = getattr(cache, "_ppai_hist", None) if cache is not None else None full = ids if hist is None else torch.cat([hist.to(dev), ids], 1) feats = [F.embedding(_ngram_hash(full, n, c.ngram_seed, c.ngram_buckets)[:, -T:], self.ng_emb[i]).float() for i, n in enumerate(c.ngram_sizes)] if cache is not None: cache._ppai_hist = full[:, -3:].clone() x = self.tok_emb(ids).float() + self.ng_gate.float() * F.linear(torch.cat(feats, -1), self.ng_proj.float()) x = x.to(compute_dtype) x0 = x pos = torch.arange(past_len, past_len + T, device=dev).float() inv = 1.0 / (c.rope_theta ** (torch.arange(0, c.head_dim, 2, device=dev).float() / c.head_dim)) fr = torch.outer(pos, inv) cos, sin = fr.cos(), fr.sin() v1 = None for i, blk in enumerate(self.blocks): x, v_raw = blk(x, x0, v1, cos, sin, cache, past_len) if i == 0: v1 = v_raw return _rms(x, self.final_norm, c.rms_norm_eps), past_key_values class PPAIForCausalLM(PPAIPreTrainedModel, GenerationMixin): def __init__(self, config: PPAIConfig): super().__init__(config) self.model = PPAIModel(config) self.post_init() def get_input_embeddings(self): return self.model.tok_emb def set_input_embeddings(self, value): self.model.tok_emb = value def get_output_embeddings(self): return None def forward(self, input_ids=None, attention_mask=None, past_key_values=None, labels=None, use_cache=None, logits_to_keep=0, return_dict=None, **kwargs): use_cache = self.config.use_cache if use_cache is None else use_cache logits_to_keep = kwargs.pop("num_logits_to_keep", logits_to_keep) or 0 h, pkv = self.model(input_ids, past_key_values=past_key_values, use_cache=use_cache) if isinstance(logits_to_keep, int) and logits_to_keep > 0: h = h[:, -logits_to_keep:] logits = F.linear(h, self.model.tok_emb.weight.to(h.dtype)).float() cap = self.config.logit_softcap if cap and cap > 0: logits = cap * torch.tanh(logits / cap) loss = None if labels is not None: loss = F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]), labels[:, 1:].reshape(-1), ignore_index=-100) return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=pkv if use_cache else None)