Text Generation
Transformers
Safetensors
Russian
English
ppai
pacperlar
mixture-of-experts
russian
code
small-language-model
custom_code
Instructions to use AdminHugging/PP-Hydrogen with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use AdminHugging/PP-Hydrogen with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="AdminHugging/PP-Hydrogen", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("AdminHugging/PP-Hydrogen", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use AdminHugging/PP-Hydrogen with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "AdminHugging/PP-Hydrogen" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "AdminHugging/PP-Hydrogen", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/AdminHugging/PP-Hydrogen
- SGLang
How to use AdminHugging/PP-Hydrogen with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "AdminHugging/PP-Hydrogen" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "AdminHugging/PP-Hydrogen", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "AdminHugging/PP-Hydrogen" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "AdminHugging/PP-Hydrogen", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use AdminHugging/PP-Hydrogen with Docker Model Runner:
docker model run hf.co/AdminHugging/PP-Hydrogen
Download modeling_ppai.py from AdminHugging/PP-Hydrogen: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/AdminHugging/PP-Hydrogen/resolve/main/modeling_ppai.py
- Command line
-
hf download hf://AdminHugging/PP-Hydrogen/modeling_ppai.py
-
curl -L -o modeling_ppai.py https://huggingface.co/AdminHugging/PP-Hydrogen/resolve/main/modeling_ppai.py
11.6 kB
| """PPAI / PP-H (PacPerlar-Hydrogen) — реализация модели. | |
| Архитектура PPAI, спроектированная под 100M всего / 12.4M активных параметров: | |
| * top-1 MoE (20 роутед-экспертов) + shared-эксперт в каждом слое; | |
| * n-gram хэш-эмбеддинги (окна 2/3/4 токена) на входе; | |
| * value-residual из нулевого слоя, per-head гейт на выходе внимания; | |
| * RMSNorm без обучаемых смещений, qk-norm, RoPE, logit softcap. | |
| Отличие от исходника: dtype выбирается по compute capability (на T4 fp16, | |
| на Ampere+ bf16) прямо в коде, без внешних патчей. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import Optional | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from transformers import GenerationMixin, PreTrainedModel | |
| from transformers.cache_utils import DynamicCache | |
| from transformers.modeling_outputs import CausalLMOutputWithPast | |
| try: # при загрузке через HF (trust_remote_code) модуль лежит в пакете | |
| from .configuration_ppai import PPAIConfig | |
| except ImportError: # при прямом запуске скриптом | |
| from configuration_ppai import PPAIConfig | |
| def _rms(x, w, eps): | |
| return F.rms_norm(x, (x.shape[-1],), w.to(x.dtype), eps) | |
| def _qk_norm(x, w, eps): | |
| xf = x.float() | |
| return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * w.float() | |
| def _rope(x, cos, sin, dtype): # x: (B,T,H,hd) fp32 | |
| h = x.shape[-1] // 2 | |
| x1, x2 = x[..., :h], x[..., h:] | |
| c, s = cos[None, :, None, :], sin[None, :, None, :] | |
| return torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], -1).to(dtype) | |
| def _ngram_hash(ids, n, seed, nb): | |
| """Детерминированный причинный хэш последних n токенов (включая текущий).""" | |
| h = torch.full_like(ids, seed + 7919 * n) | |
| for k in range(n): | |
| xk = F.pad(ids, (k, 0))[:, : ids.shape[1]] | |
| h = (h * 1000003 + xk + 1) & 0x7FFFFFFF | |
| h = h ^ (h >> 13) | |
| h = (h * 40503) & 0x7FFFFFFF | |
| return (h >> 4) % nb | |
| class PPAIBlock(nn.Module): | |
| def __init__(self, c: PPAIConfig, idx: int): | |
| super().__init__() | |
| self.c = c | |
| self.idx = idx | |
| d, hq, hkv, hd = c.d_model, c.n_heads, c.n_kv_heads, c.head_dim | |
| self.ln1 = nn.Parameter(torch.ones(d)) | |
| self.ln2 = nn.Parameter(torch.ones(d)) | |
| self.wq = nn.Parameter(torch.empty(hq * hd, d)) | |
| self.wk = nn.Parameter(torch.empty(hkv * hd, d)) | |
| self.wv = nn.Parameter(torch.empty(hkv * hd, d)) | |
| self.wo = nn.Parameter(torch.empty(d, hq * hd)) | |
| self.q_norm = nn.Parameter(torch.ones(hq, hd)) | |
| self.k_norm = nn.Parameter(torch.ones(hkv, hd)) | |
| self.router_w = nn.Parameter(torch.empty(c.num_experts, d)) | |
| self.router_b = nn.Parameter(torch.zeros(c.num_experts)) | |
| self.s13 = nn.Parameter(torch.empty(2 * c.ffn_shared_dim, d)) | |
| self.s2 = nn.Parameter(torch.empty(d, c.ffn_shared_dim)) | |
| self.w13 = nn.Parameter(torch.empty(c.num_experts, 2 * c.ffn_dim, d)) # fused gate|up | |
| self.w2 = nn.Parameter(torch.empty(c.num_experts, d, c.ffn_dim)) | |
| self.lam = nn.Parameter(torch.tensor([1.0, 0.0])) # x <- lam0*x + lam1*x0 | |
| self.ag_w = nn.Parameter(torch.zeros(hq, c.gate_dim)) # per-head gate | |
| if idx > 0: | |
| self.vl = nn.Parameter(torch.tensor([0.5, 0.5])) # value residual mix | |
| else: | |
| self.vl = None | |
| def moe(self, m): | |
| """Dropless top-1 MoE. m: (N,d).""" | |
| logits = F.linear(m.float(), self.router_w.float(), self.router_b.float()) | |
| probs = logits.softmax(-1) | |
| g, idx = probs.max(-1) | |
| out = torch.zeros_like(m) | |
| ffn = self.c.ffn_dim | |
| for e in idx.unique().tolist(): | |
| sel = (idx == e).nonzero(as_tuple=True)[0] | |
| h = F.linear(m[sel], self.w13[e].to(m.dtype)) | |
| y = F.linear(F.silu(h[:, :ffn]) * h[:, ffn:], self.w2[e].to(m.dtype)) | |
| out.index_copy_(0, sel, y) | |
| return out * g.to(out.dtype).unsqueeze(-1) | |
| def forward(self, x, x0, v1, cos, sin, cache, past_len): | |
| c = self.c | |
| B, T, d = x.shape | |
| dt = x.dtype | |
| hq, hkv, hd = c.n_heads, c.n_kv_heads, c.head_dim | |
| x = self.lam[0].to(dt) * x + self.lam[1].to(dt) * x0 | |
| a = _rms(x, self.ln1, c.rms_norm_eps) | |
| qkv = F.linear(a, torch.cat([self.wq, self.wk, self.wv], 0).to(dt)) | |
| q, k, v = qkv.split([hq * hd, hkv * hd, hkv * hd], -1) | |
| q = q.reshape(B, T, hq, hd) | |
| k = k.reshape(B, T, hkv, hd) | |
| v = v.reshape(B, T, hkv, hd) | |
| v_raw = v | |
| if self.vl is not None: | |
| v = self.vl[0].to(dt) * v + self.vl[1].to(dt) * v1 | |
| q = _rope(_qk_norm(q, self.q_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2) | |
| k = _rope(_qk_norm(k, self.k_norm, c.rms_norm_eps), cos, sin, dt).transpose(1, 2) | |
| v = v.transpose(1, 2) | |
| if cache is not None: | |
| k, v = cache.update(k, v, self.idx) | |
| rep = hq // hkv | |
| k = k.repeat_interleave(rep, dim=1) | |
| v = v.repeat_interleave(rep, dim=1) | |
| if T == 1: | |
| o = F.scaled_dot_product_attention(q, k, v) | |
| elif past_len == 0: | |
| o = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| else: | |
| Tk = k.shape[2] | |
| mask = torch.ones(T, Tk, dtype=torch.bool, device=q.device).tril(diagonal=Tk - T) | |
| o = F.scaled_dot_product_attention(q, k, v, attn_mask=mask) | |
| gate = 2.0 * torch.sigmoid(F.linear(a[..., : c.gate_dim], self.ag_w.to(dt))) | |
| o = o.transpose(1, 2) * gate.unsqueeze(-1) | |
| x = x + F.linear(o.reshape(B, T, hq * hd), self.wo.to(dt)) | |
| m = _rms(x, self.ln2, c.rms_norm_eps) | |
| sg, su = F.linear(m, self.s13.to(dt)).chunk(2, -1) | |
| s = F.linear(F.silu(sg) * su, self.s2.to(dt)) | |
| e = self.moe(m.reshape(B * T, d)).view(B, T, d) | |
| return x + s + e, v_raw | |
| class PPAIPreTrainedModel(PreTrainedModel): | |
| config_class = PPAIConfig | |
| base_model_prefix = "model" | |
| supports_gradient_checkpointing = False | |
| _no_split_modules = ["PPAIBlock"] | |
| def _init_weights(self, module): | |
| c = self.config | |
| std, ostd = 0.02, 0.02 / math.sqrt(2 * c.n_layers) | |
| if isinstance(module, nn.Embedding): | |
| module.weight.normal_(0, std) | |
| elif isinstance(module, PPAIBlock): | |
| for p in (module.wq, module.wk, module.wv, module.s13, module.w13): | |
| p.normal_(0, std) | |
| for p in (module.wo, module.s2, module.w2): | |
| p.normal_(0, ostd) | |
| module.router_w.normal_(0, 0.01) | |
| module.router_b.zero_() | |
| module.ln1.fill_(1.0) | |
| module.ln2.fill_(1.0) | |
| module.q_norm.fill_(1.0) | |
| module.k_norm.fill_(1.0) | |
| module.lam.copy_(torch.tensor([1.0, 0.0])) | |
| module.ag_w.zero_() | |
| if module.vl is not None: | |
| module.vl.copy_(torch.tensor([0.5, 0.5])) | |
| elif isinstance(module, PPAIModel): | |
| module.ng_emb.normal_(0, std) | |
| module.ng_proj.normal_(0, std) | |
| module.ng_gate.fill_(0.1) | |
| module.final_norm.fill_(1.0) | |
| class PPAIModel(PPAIPreTrainedModel): | |
| def __init__(self, c: PPAIConfig): | |
| super().__init__(c) | |
| self.tok_emb = nn.Embedding(c.vocab_size, c.d_model) # tied with LM head | |
| self.ng_emb = nn.Parameter(torch.empty(len(c.ngram_sizes), c.ngram_buckets, c.ngram_dim)) | |
| self.ng_proj = nn.Parameter(torch.empty(c.d_model, len(c.ngram_sizes) * c.ngram_dim)) | |
| self.ng_gate = nn.Parameter(torch.full((c.d_model,), 0.1)) | |
| self.final_norm = nn.Parameter(torch.ones(c.d_model)) | |
| self.blocks = nn.ModuleList(PPAIBlock(c, i) for i in range(c.n_layers)) | |
| self.post_init() | |
| def forward(self, input_ids, past_key_values=None, use_cache=False): | |
| c = self.config | |
| B, T = input_ids.shape | |
| dev = input_ids.device | |
| # bf16 только там, где он аппаратный; на T4 (sm_75) — fp16 | |
| if dev.type == "cuda" and torch.cuda.get_device_capability(dev)[0] >= 8: | |
| compute_dtype = torch.bfloat16 | |
| elif dev.type == "cuda": | |
| compute_dtype = torch.float16 | |
| else: | |
| compute_dtype = torch.float32 | |
| ids = input_ids.long() | |
| if use_cache and past_key_values is None: | |
| past_key_values = DynamicCache() | |
| cache = past_key_values if use_cache else None | |
| past_len = cache.get_seq_length() if cache is not None else 0 | |
| # n-gram признаки требуют до 3 предыдущих токенов | |
| hist = getattr(cache, "_ppai_hist", None) if cache is not None else None | |
| full = ids if hist is None else torch.cat([hist.to(dev), ids], 1) | |
| feats = [F.embedding(_ngram_hash(full, n, c.ngram_seed, c.ngram_buckets)[:, -T:], | |
| self.ng_emb[i]).float() | |
| for i, n in enumerate(c.ngram_sizes)] | |
| if cache is not None: | |
| cache._ppai_hist = full[:, -3:].clone() | |
| x = self.tok_emb(ids).float() + self.ng_gate.float() * F.linear(torch.cat(feats, -1), self.ng_proj.float()) | |
| x = x.to(compute_dtype) | |
| x0 = x | |
| pos = torch.arange(past_len, past_len + T, device=dev).float() | |
| inv = 1.0 / (c.rope_theta ** (torch.arange(0, c.head_dim, 2, device=dev).float() / c.head_dim)) | |
| fr = torch.outer(pos, inv) | |
| cos, sin = fr.cos(), fr.sin() | |
| v1 = None | |
| for i, blk in enumerate(self.blocks): | |
| x, v_raw = blk(x, x0, v1, cos, sin, cache, past_len) | |
| if i == 0: | |
| v1 = v_raw | |
| return _rms(x, self.final_norm, c.rms_norm_eps), past_key_values | |
| class PPAIForCausalLM(PPAIPreTrainedModel, GenerationMixin): | |
| def __init__(self, config: PPAIConfig): | |
| super().__init__(config) | |
| self.model = PPAIModel(config) | |
| self.post_init() | |
| def get_input_embeddings(self): | |
| return self.model.tok_emb | |
| def set_input_embeddings(self, value): | |
| self.model.tok_emb = value | |
| def get_output_embeddings(self): | |
| return None | |
| def forward(self, input_ids=None, attention_mask=None, past_key_values=None, labels=None, | |
| use_cache=None, logits_to_keep=0, return_dict=None, **kwargs): | |
| use_cache = self.config.use_cache if use_cache is None else use_cache | |
| logits_to_keep = kwargs.pop("num_logits_to_keep", logits_to_keep) or 0 | |
| h, pkv = self.model(input_ids, past_key_values=past_key_values, use_cache=use_cache) | |
| if isinstance(logits_to_keep, int) and logits_to_keep > 0: | |
| h = h[:, -logits_to_keep:] | |
| logits = F.linear(h, self.model.tok_emb.weight.to(h.dtype)).float() | |
| cap = self.config.logit_softcap | |
| if cap and cap > 0: | |
| logits = cap * torch.tanh(logits / cap) | |
| loss = None | |
| if labels is not None: | |
| loss = F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]), | |
| labels[:, 1:].reshape(-1), ignore_index=-100) | |
| return CausalLMOutputWithPast(loss=loss, logits=logits, | |
| past_key_values=pkv if use_cache else None) | |