Download modeling_onner.py from ReskiGaming/onner: direct link, hf CLI and curl.
- Browser
- Download file 7.22 kB
-
https://huggingface.co/ReskiGaming/onner/resolve/main/modeling_onner.py
- Command line
-
hf download hf://ReskiGaming/onner/modeling_onner.py
-
curl -L -o modeling_onner.py https://huggingface.co/ReskiGaming/onner/resolve/main/modeling_onner.py
7.22 kB
| import math | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class OnnerReasearchConfig: | |
| architecture_name: str = "OnnerReasearch" | |
| model_name: str = "onner" | |
| dataset_name: str = "indonesian-nlp/wikipedia-id" | |
| vocab_size: int = 28000 | |
| tokenizer_train_samples: int = 50000 | |
| tokenizer_path: str = "onner_tokenizer.json" | |
| cache_docs: int = 50000 | |
| full_token_cache_path: str = "onner_tokens_full.pt" | |
| train_token_cache_path: str = "onner_tokens_train.pt" | |
| val_token_cache_path: str = "onner_tokens_val.pt" | |
| val_ratio: float = 0.02 | |
| seq_len: int = 512 | |
| d_model: int = 768 | |
| n_layers: int = 16 | |
| n_heads: int = 12 | |
| n_kv_heads: int = 4 | |
| ffn_hidden: int = 2048 | |
| dropout: float = 0.0 | |
| rope_base: float = 10000.0 | |
| mtp_n: int = 4 | |
| mtp_loss_weights: tuple = (1.0, 0.3, 0.2, 0.1) | |
| micro_batch_size: int = 8 | |
| grad_accum_steps: int = 8 | |
| max_steps: int = 500 | |
| lr: float = 3e-4 | |
| min_lr: float = 3e-5 | |
| warmup_steps: int = 50 | |
| weight_decay: float = 0.1 | |
| grad_clip: float = 1.0 | |
| eval_interval: int = 100 | |
| eval_steps: int = 20 | |
| save_interval: int = 500 | |
| checkpoint_dir: str = "onner_checkpoints" | |
| dataloader_workers: int = 0 | |
| seed: int = 42 | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim, eps=1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x): | |
| normed = x * torch.rsqrt( | |
| x.pow(2).mean(dim=-1, keepdim=True) + self.eps | |
| ) | |
| return normed * self.weight | |
| def build_rope_cache(seq_len, head_dim, device, base=10000.0): | |
| inv_freq = 1.0 / ( | |
| base ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim) | |
| ) | |
| t = torch.arange(seq_len, device=device).float() | |
| freqs = torch.outer(t, inv_freq) | |
| cos = freqs.cos()[None, None, :, :] | |
| sin = freqs.sin()[None, None, :, :] | |
| return cos, sin | |
| def apply_rope(x, cos, sin): | |
| x_even = x[..., 0::2] | |
| x_odd = x[..., 1::2] | |
| x_rot_even = x_even * cos - x_odd * sin | |
| x_rot_odd = x_even * sin + x_odd * cos | |
| return torch.stack((x_rot_even, x_rot_odd), dim=-1).flatten(-2) | |
| class OnnerReasearchGQAAttention(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| assert cfg.d_model % cfg.n_heads == 0 | |
| assert cfg.n_heads % cfg.n_kv_heads == 0 | |
| self.cfg = cfg | |
| self.head_dim = cfg.d_model // cfg.n_heads | |
| self.n_heads = cfg.n_heads | |
| self.n_kv_heads = cfg.n_kv_heads | |
| self.repeat_factor = cfg.n_heads // cfg.n_kv_heads | |
| self.q_proj = nn.Linear( | |
| cfg.d_model, | |
| cfg.n_heads * self.head_dim, | |
| bias=False, | |
| ) | |
| self.k_proj = nn.Linear( | |
| cfg.d_model, | |
| cfg.n_kv_heads * self.head_dim, | |
| bias=False, | |
| ) | |
| self.v_proj = nn.Linear( | |
| cfg.d_model, | |
| cfg.n_kv_heads * self.head_dim, | |
| bias=False, | |
| ) | |
| self.o_proj = nn.Linear( | |
| cfg.n_heads * self.head_dim, | |
| cfg.d_model, | |
| bias=False, | |
| ) | |
| self.dropout = cfg.dropout | |
| def forward(self, x): | |
| B, T, C = x.shape | |
| q = self.q_proj(x) | |
| k = self.k_proj(x) | |
| v = self.v_proj(x) | |
| q = q.view(B, T, self.n_heads, self.head_dim).transpose(1, 2) | |
| k = k.view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2) | |
| v = v.view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2) | |
| cos, sin = build_rope_cache( | |
| T, | |
| self.head_dim, | |
| x.device, | |
| base=self.cfg.rope_base, | |
| ) | |
| q = apply_rope(q, cos, sin) | |
| k = apply_rope(k, cos, sin) | |
| if self.repeat_factor > 1: | |
| k = k.repeat_interleave(self.repeat_factor, dim=1) | |
| v = v.repeat_interleave(self.repeat_factor, dim=1) | |
| y = F.scaled_dot_product_attention( | |
| q, | |
| k, | |
| v, | |
| attn_mask=None, | |
| dropout_p=self.dropout if self.training else 0.0, | |
| is_causal=True, | |
| ) | |
| y = y.transpose(1, 2).contiguous().view(B, T, C) | |
| return self.o_proj(y) | |
| class OnnerReasearchSwiGLU(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.up_proj = nn.Linear( | |
| cfg.d_model, | |
| cfg.ffn_hidden, | |
| bias=False, | |
| ) | |
| self.gate_proj = nn.Linear( | |
| cfg.d_model, | |
| cfg.ffn_hidden, | |
| bias=False, | |
| ) | |
| self.down_proj = nn.Linear( | |
| cfg.ffn_hidden, | |
| cfg.d_model, | |
| bias=False, | |
| ) | |
| def forward(self, x): | |
| return self.down_proj( | |
| F.silu(self.gate_proj(x)) * self.up_proj(x) | |
| ) | |
| class OnnerReasearchBlock(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.norm1 = RMSNorm(cfg.d_model) | |
| self.attn = OnnerReasearchGQAAttention(cfg) | |
| self.norm2 = RMSNorm(cfg.d_model) | |
| self.ffn = OnnerReasearchSwiGLU(cfg) | |
| def forward(self, x): | |
| x = x + self.attn(self.norm1(x)) | |
| x = x + self.ffn(self.norm2(x)) | |
| return x | |
| class OnnerMTPAdapter(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.norm = RMSNorm(dim) | |
| self.proj = nn.Linear(dim, dim, bias=False) | |
| def forward(self, x): | |
| return self.proj(self.norm(x)) | |
| class OnnerReasearchModel(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.architecture_name = cfg.architecture_name | |
| self.model_name = cfg.model_name | |
| self.token_emb = nn.Embedding( | |
| cfg.vocab_size, | |
| cfg.d_model, | |
| ) | |
| self.blocks = nn.ModuleList([ | |
| OnnerReasearchBlock(cfg) | |
| for _ in range(cfg.n_layers) | |
| ]) | |
| self.final_norm = RMSNorm(cfg.d_model) | |
| self.lm_head = nn.Linear( | |
| cfg.d_model, | |
| cfg.vocab_size, | |
| bias=False, | |
| ) | |
| self.lm_head.weight = self.token_emb.weight | |
| adapters = [nn.Identity()] | |
| for _ in range(cfg.mtp_n - 1): | |
| adapters.append(OnnerMTPAdapter(cfg.d_model)) | |
| self.mtp_adapters = nn.ModuleList(adapters) | |
| def forward(self, input_ids): | |
| x = self.token_emb(input_ids) | |
| for block in self.blocks: | |
| x = block(x) | |
| h = self.final_norm(x) | |
| logits_list = [] | |
| for adapter in self.mtp_adapters: | |
| h_i = adapter(h) | |
| logits = self.lm_head(h_i) | |
| logits_list.append(logits) | |
| return logits_list | |
| def sample_next_token(logits, temperature=0.8, top_k=50): | |
| logits = logits / max(temperature, 1e-6) | |
| if top_k is not None: | |
| values, _ = torch.topk(logits, top_k) | |
| min_value = values[:, -1].unsqueeze(-1) | |
| logits = torch.where( | |
| logits < min_value, | |
| torch.full_like(logits, float("-inf")), | |
| logits, | |
| ) | |
| probs = F.softmax(logits, dim=-1) | |
| return torch.multinomial(probs, num_samples=1) | |