import math from dataclasses import dataclass import torch import torch.nn as nn import torch.nn.functional as F @dataclass class OnnerReasearchConfig: architecture_name: str = "OnnerReasearch" model_name: str = "onner" dataset_name: str = "indonesian-nlp/wikipedia-id" vocab_size: int = 28000 tokenizer_train_samples: int = 50000 tokenizer_path: str = "onner_tokenizer.json" cache_docs: int = 50000 full_token_cache_path: str = "onner_tokens_full.pt" train_token_cache_path: str = "onner_tokens_train.pt" val_token_cache_path: str = "onner_tokens_val.pt" val_ratio: float = 0.02 seq_len: int = 512 d_model: int = 768 n_layers: int = 16 n_heads: int = 12 n_kv_heads: int = 4 ffn_hidden: int = 2048 dropout: float = 0.0 rope_base: float = 10000.0 mtp_n: int = 4 mtp_loss_weights: tuple = (1.0, 0.3, 0.2, 0.1) micro_batch_size: int = 8 grad_accum_steps: int = 8 max_steps: int = 500 lr: float = 3e-4 min_lr: float = 3e-5 warmup_steps: int = 50 weight_decay: float = 0.1 grad_clip: float = 1.0 eval_interval: int = 100 eval_steps: int = 20 save_interval: int = 500 checkpoint_dir: str = "onner_checkpoints" dataloader_workers: int = 0 seed: int = 42 class RMSNorm(nn.Module): def __init__(self, dim, eps=1e-6): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x): normed = x * torch.rsqrt( x.pow(2).mean(dim=-1, keepdim=True) + self.eps ) return normed * self.weight def build_rope_cache(seq_len, head_dim, device, base=10000.0): inv_freq = 1.0 / ( base ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim) ) t = torch.arange(seq_len, device=device).float() freqs = torch.outer(t, inv_freq) cos = freqs.cos()[None, None, :, :] sin = freqs.sin()[None, None, :, :] return cos, sin def apply_rope(x, cos, sin): x_even = x[..., 0::2] x_odd = x[..., 1::2] x_rot_even = x_even * cos - x_odd * sin x_rot_odd = x_even * sin + x_odd * cos return torch.stack((x_rot_even, x_rot_odd), dim=-1).flatten(-2) class OnnerReasearchGQAAttention(nn.Module): def __init__(self, cfg): super().__init__() assert cfg.d_model % cfg.n_heads == 0 assert cfg.n_heads % cfg.n_kv_heads == 0 self.cfg = cfg self.head_dim = cfg.d_model // cfg.n_heads self.n_heads = cfg.n_heads self.n_kv_heads = cfg.n_kv_heads self.repeat_factor = cfg.n_heads // cfg.n_kv_heads self.q_proj = nn.Linear( cfg.d_model, cfg.n_heads * self.head_dim, bias=False, ) self.k_proj = nn.Linear( cfg.d_model, cfg.n_kv_heads * self.head_dim, bias=False, ) self.v_proj = nn.Linear( cfg.d_model, cfg.n_kv_heads * self.head_dim, bias=False, ) self.o_proj = nn.Linear( cfg.n_heads * self.head_dim, cfg.d_model, bias=False, ) self.dropout = cfg.dropout def forward(self, x): B, T, C = x.shape q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) q = q.view(B, T, self.n_heads, self.head_dim).transpose(1, 2) k = k.view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2) v = v.view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2) cos, sin = build_rope_cache( T, self.head_dim, x.device, base=self.cfg.rope_base, ) q = apply_rope(q, cos, sin) k = apply_rope(k, cos, sin) if self.repeat_factor > 1: k = k.repeat_interleave(self.repeat_factor, dim=1) v = v.repeat_interleave(self.repeat_factor, dim=1) y = F.scaled_dot_product_attention( q, k, v, attn_mask=None, dropout_p=self.dropout if self.training else 0.0, is_causal=True, ) y = y.transpose(1, 2).contiguous().view(B, T, C) return self.o_proj(y) class OnnerReasearchSwiGLU(nn.Module): def __init__(self, cfg): super().__init__() self.up_proj = nn.Linear( cfg.d_model, cfg.ffn_hidden, bias=False, ) self.gate_proj = nn.Linear( cfg.d_model, cfg.ffn_hidden, bias=False, ) self.down_proj = nn.Linear( cfg.ffn_hidden, cfg.d_model, bias=False, ) def forward(self, x): return self.down_proj( F.silu(self.gate_proj(x)) * self.up_proj(x) ) class OnnerReasearchBlock(nn.Module): def __init__(self, cfg): super().__init__() self.norm1 = RMSNorm(cfg.d_model) self.attn = OnnerReasearchGQAAttention(cfg) self.norm2 = RMSNorm(cfg.d_model) self.ffn = OnnerReasearchSwiGLU(cfg) def forward(self, x): x = x + self.attn(self.norm1(x)) x = x + self.ffn(self.norm2(x)) return x class OnnerMTPAdapter(nn.Module): def __init__(self, dim): super().__init__() self.norm = RMSNorm(dim) self.proj = nn.Linear(dim, dim, bias=False) def forward(self, x): return self.proj(self.norm(x)) class OnnerReasearchModel(nn.Module): def __init__(self, cfg): super().__init__() self.cfg = cfg self.architecture_name = cfg.architecture_name self.model_name = cfg.model_name self.token_emb = nn.Embedding( cfg.vocab_size, cfg.d_model, ) self.blocks = nn.ModuleList([ OnnerReasearchBlock(cfg) for _ in range(cfg.n_layers) ]) self.final_norm = RMSNorm(cfg.d_model) self.lm_head = nn.Linear( cfg.d_model, cfg.vocab_size, bias=False, ) self.lm_head.weight = self.token_emb.weight adapters = [nn.Identity()] for _ in range(cfg.mtp_n - 1): adapters.append(OnnerMTPAdapter(cfg.d_model)) self.mtp_adapters = nn.ModuleList(adapters) def forward(self, input_ids): x = self.token_emb(input_ids) for block in self.blocks: x = block(x) h = self.final_norm(x) logits_list = [] for adapter in self.mtp_adapters: h_i = adapter(h) logits = self.lm_head(h_i) logits_list.append(logits) return logits_list @torch.no_grad() def sample_next_token(logits, temperature=0.8, top_k=50): logits = logits / max(temperature, 1e-6) if top_k is not None: values, _ = torch.topk(logits, top_k) min_value = values[:, -1].unsqueeze(-1) logits = torch.where( logits < min_value, torch.full_like(logits, float("-inf")), logits, ) probs = F.softmax(logits, dim=-1) return torch.multinomial(probs, num_samples=1)