PP-Hydrogen / modeling_ppai.py
AdminHugging's picture
clean: без упоминаний сторонних проектов в карточке
0efa6d3 verified
Raw History Blame Contribute Delete
11.6 kB
"""PPAI / PP-H (PacPerlar-Hydrogen) — реализация модели.
Архитектура PPAI, спроектированная под 100M всего / 12.4M активных параметров:
* top-1 MoE (20 роутед-экспертов) + shared-эксперт в каждом слое;
* n-gram хэш-эмбеддинги (окна 2/3/4 токена) на входе;
* value-residual из нулевого слоя, per-head гейт на выходе внимания;
* RMSNorm без обучаемых смещений, qk-norm, RoPE, logit softcap.
Отличие от исходника: dtype выбирается по compute capability (на T4 fp16,
на Ampere+ bf16) прямо в коде, без внешних патчей.
"""
from __future__ import annotations
import math
from typing import Optional
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import GenerationMixin, PreTrainedModel
from transformers.cache_utils import DynamicCache
from transformers.modeling_outputs import CausalLMOutputWithPast
try: # при загрузке через HF (trust_remote_code) модуль лежит в пакете
from .configuration_ppai import PPAIConfig
except ImportError: # при прямом запуске скриптом
from configuration_ppai import PPAIConfig
def _rms(x, w, eps):
return F.rms_norm(x, (x.shape[-1],), w.to(x.dtype), eps)
def _qk_norm(x, w, eps):
xf = x.float()
return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * w.float()
def _rope(x, cos, sin, dtype): # x: (B,T,H,hd) fp32
h = x.shape[-1] // 2
x1, x2 = x[..., :h], x[..., h:]
c, s = cos[None, :, None, :], sin[None, :, None, :]
return torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], -1).to(dtype)
def _ngram_hash(ids, n, seed, nb):
"""Детерминированный причинный хэш последних n токенов (включая текущий)."""
h = torch.full_like(ids, seed + 7919 * n)
for k in range(n):
xk = F.pad(ids, (k, 0))[:, : ids.shape[1]]
h = (h * 1000003 + xk + 1) & 0x7FFFFFFF
h = h ^ (h >> 13)
h = (h * 40503) & 0x7FFFFFFF
return (h >> 4) % nb
class PPAIBlock(nn.Module):
def __init__(self, c: PPAIConfig, idx: int):
super().__init__()
self.c = c
self.idx = idx
d, hq, hkv, hd = c.d_model, c.n_heads, c.n_kv_heads, c.head_dim
self.ln1 = nn.Parameter(torch.ones(d))
self.ln2 = nn.Parameter(torch.ones(d))
self.wq = nn.Parameter(torch.empty(hq * hd, d))
self.wk = nn.Parameter(torch.empty(hkv * hd, d))
self.wv = nn.Parameter(torch.empty(hkv * hd, d))
self.wo = nn.Parameter(torch.empty(d, hq * hd))
self.q_norm = nn.Parameter(torch.ones(hq, hd))
self.k_norm = nn.Parameter(torch.ones(hkv, hd))
self.router_w = nn.Parameter(torch.empty(c.num_experts, d))
self.router_b = nn.Parameter(torch.zeros(c.num_experts))
self.s13 = nn.Parameter(torch.empty(2 * c.ffn_shared_dim, d))
self.s2 = nn.Parameter(torch.empty(d, c.ffn_shared_dim))
self.w13 = nn.Parameter(torch.empty(c.num_experts, 2 * c.ffn_dim, d)) # fused gate|up
self.w2 = nn.Parameter(torch.empty(c.num_experts, d, c.ffn_dim))
self.lam = nn.Parameter(torch.tensor([1.0, 0.0])) # x <- lam0*x + lam1*x0
self.ag_w = nn.Parameter(torch.zeros(hq, c.gate_dim)) # per-head gate
if idx > 0:
self.vl = nn.Parameter(torch.tensor([0.5, 0.5])) # value residual mix
else:
self.vl = None
def moe(self, m):
"""Dropless top-1 MoE. m: (N,d)."""
logits = F.linear(m.float(), self.router_w.float(), self.router_b.float())
probs = logits.softmax(-1)
g, idx = probs.max(-1)
out = torch.zeros_like(m)
ffn = self.c.ffn_dim
for e in idx.unique().tolist():
sel = (idx == e).nonzero(as_tuple=True)[0]
h = F.linear(m[sel], self.w13[e].to(m.dtype))
y = F.linear(F.silu(h[:, :ffn]) * h[:, ffn:], self.w2[e].to(m.dtype))
out.index_copy_(0, sel, y)
return out * g.to(out.dtype).unsqueeze(-1)
def forward(self, x, x0, v1, cos, sin, cache, past_len):
c = self.c
B, T, d = x.shape
dt = x.dtype
hq, hkv, hd = c.n_heads, c.n_kv_heads, c.head_dim
x = self.lam[0].to(dt) * x + self.lam[1].to(dt) * x0
a = _rms(x, self.ln1, c.rms_norm_eps)
qkv = F.linear(a, torch.cat([self.wq, self.wk, self.wv], 0).to(dt))
q, k, v = qkv.split([hq * hd, hkv * hd, hkv * hd], -1)
q = q.reshape(B, T, hq, hd)
k = k.reshape(B, T, hkv, hd)
v = v.reshape(B, T, hkv, hd)
v_raw = v
if self.vl is not None:
v = self.vl[0].to(dt) * v + self.vl[1].to(dt) * v1
q = _rope(_qk_norm(q, self.q_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2)
k = _rope(_qk_norm(k, self.k_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2)
v = v.transpose(1, 2)
if cache is not None:
k, v = cache.update(k, v, self.idx)
rep = hq // hkv
k = k.repeat_interleave(rep, dim=1)
v = v.repeat_interleave(rep, dim=1)
if T == 1:
o = F.scaled_dot_product_attention(q, k, v)
elif past_len == 0:
o = F.scaled_dot_product_attention(q, k, v, is_causal=True)
else:
Tk = k.shape[2]
mask = torch.ones(T, Tk, dtype=torch.bool, device=q.device).tril(diagonal=Tk - T)
o = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
gate = 2.0 * torch.sigmoid(F.linear(a[..., : c.gate_dim], self.ag_w.to(dt)))
o = o.transpose(1, 2) * gate.unsqueeze(-1)
x = x + F.linear(o.reshape(B, T, hq * hd), self.wo.to(dt))
m = _rms(x, self.ln2, c.rms_norm_eps)
sg, su = F.linear(m, self.s13.to(dt)).chunk(2, -1)
s = F.linear(F.silu(sg) * su, self.s2.to(dt))
e = self.moe(m.reshape(B * T, d)).view(B, T, d)
return x + s + e, v_raw
class PPAIPreTrainedModel(PreTrainedModel):
config_class = PPAIConfig
base_model_prefix = "model"
supports_gradient_checkpointing = False
_no_split_modules = ["PPAIBlock"]
@torch.no_grad()
def _init_weights(self, module):
c = self.config
std, ostd = 0.02, 0.02 / math.sqrt(2 * c.n_layers)
if isinstance(module, nn.Embedding):
module.weight.normal_(0, std)
elif isinstance(module, PPAIBlock):
for p in (module.wq, module.wk, module.wv, module.s13, module.w13):
p.normal_(0, std)
for p in (module.wo, module.s2, module.w2):
p.normal_(0, ostd)
module.router_w.normal_(0, 0.01)
module.router_b.zero_()
module.ln1.fill_(1.0)
module.ln2.fill_(1.0)
module.q_norm.fill_(1.0)
module.k_norm.fill_(1.0)
module.lam.copy_(torch.tensor([1.0, 0.0]))
module.ag_w.zero_()
if module.vl is not None:
module.vl.copy_(torch.tensor([0.5, 0.5]))
elif isinstance(module, PPAIModel):
module.ng_emb.normal_(0, std)
module.ng_proj.normal_(0, std)
module.ng_gate.fill_(0.1)
module.final_norm.fill_(1.0)
class PPAIModel(PPAIPreTrainedModel):
def __init__(self, c: PPAIConfig):
super().__init__(c)
self.tok_emb = nn.Embedding(c.vocab_size, c.d_model) # tied with LM head
self.ng_emb = nn.Parameter(torch.empty(len(c.ngram_sizes), c.ngram_buckets, c.ngram_dim))
self.ng_proj = nn.Parameter(torch.empty(c.d_model, len(c.ngram_sizes) * c.ngram_dim))
self.ng_gate = nn.Parameter(torch.full((c.d_model,), 0.1))
self.final_norm = nn.Parameter(torch.ones(c.d_model))
self.blocks = nn.ModuleList(PPAIBlock(c, i) for i in range(c.n_layers))
self.post_init()
def forward(self, input_ids, past_key_values=None, use_cache=False):
c = self.config
B, T = input_ids.shape
dev = input_ids.device
# bf16 только там, где он аппаратный; на T4 (sm_75) — fp16
if dev.type == "cuda" and torch.cuda.get_device_capability(dev)[0] >= 8:
compute_dtype = torch.bfloat16
elif dev.type == "cuda":
compute_dtype = torch.float16
else:
compute_dtype = torch.float32
ids = input_ids.long()
if use_cache and past_key_values is None:
past_key_values = DynamicCache()
cache = past_key_values if use_cache else None
past_len = cache.get_seq_length() if cache is not None else 0
# n-gram признаки требуют до 3 предыдущих токенов
hist = getattr(cache, "_ppai_hist", None) if cache is not None else None
full = ids if hist is None else torch.cat([hist.to(dev), ids], 1)
feats = [F.embedding(_ngram_hash(full, n, c.ngram_seed, c.ngram_buckets)[:, -T:],
self.ng_emb[i]).float()
for i, n in enumerate(c.ngram_sizes)]
if cache is not None:
cache._ppai_hist = full[:, -3:].clone()
x = self.tok_emb(ids).float() + self.ng_gate.float() * F.linear(torch.cat(feats, -1), self.ng_proj.float())
x = x.to(compute_dtype)
x0 = x
pos = torch.arange(past_len, past_len + T, device=dev).float()
inv = 1.0 / (c.rope_theta ** (torch.arange(0, c.head_dim, 2, device=dev).float() / c.head_dim))
fr = torch.outer(pos, inv)
cos, sin = fr.cos(), fr.sin()
v1 = None
for i, blk in enumerate(self.blocks):
x, v_raw = blk(x, x0, v1, cos, sin, cache, past_len)
if i == 0:
v1 = v_raw
return _rms(x, self.final_norm, c.rms_norm_eps), past_key_values
class PPAIForCausalLM(PPAIPreTrainedModel, GenerationMixin):
def __init__(self, config: PPAIConfig):
super().__init__(config)
self.model = PPAIModel(config)
self.post_init()
def get_input_embeddings(self):
return self.model.tok_emb
def set_input_embeddings(self, value):
self.model.tok_emb = value
def get_output_embeddings(self):
return None
def forward(self, input_ids=None, attention_mask=None, past_key_values=None, labels=None,
use_cache=None, logits_to_keep=0, return_dict=None, **kwargs):
use_cache = self.config.use_cache if use_cache is None else use_cache
logits_to_keep = kwargs.pop("num_logits_to_keep", logits_to_keep) or 0
h, pkv = self.model(input_ids, past_key_values=past_key_values, use_cache=use_cache)
if isinstance(logits_to_keep, int) and logits_to_keep > 0:
h = h[:, -logits_to_keep:]
logits = F.linear(h, self.model.tok_emb.weight.to(h.dtype)).float()
cap = self.config.logit_softcap
if cap and cap > 0:
logits = cap * torch.tanh(logits / cap)
loss = None
if labels is not None:
loss = F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]),
labels[:, 1:].reshape(-1), ignore_index=-100)
return CausalLMOutputWithPast(loss=loss, logits=logits,
past_key_values=pkv if use_cache else None)