onner / modeling_onner.py
ReskiGaming's picture
Upload onner model with OnnerReasearch architecture
d9c1356 verified
Raw History Blame Contribute Delete
7.22 kB
import math
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class OnnerReasearchConfig:
architecture_name: str = "OnnerReasearch"
model_name: str = "onner"
dataset_name: str = "indonesian-nlp/wikipedia-id"
vocab_size: int = 28000
tokenizer_train_samples: int = 50000
tokenizer_path: str = "onner_tokenizer.json"
cache_docs: int = 50000
full_token_cache_path: str = "onner_tokens_full.pt"
train_token_cache_path: str = "onner_tokens_train.pt"
val_token_cache_path: str = "onner_tokens_val.pt"
val_ratio: float = 0.02
seq_len: int = 512
d_model: int = 768
n_layers: int = 16
n_heads: int = 12
n_kv_heads: int = 4
ffn_hidden: int = 2048
dropout: float = 0.0
rope_base: float = 10000.0
mtp_n: int = 4
mtp_loss_weights: tuple = (1.0, 0.3, 0.2, 0.1)
micro_batch_size: int = 8
grad_accum_steps: int = 8
max_steps: int = 500
lr: float = 3e-4
min_lr: float = 3e-5
warmup_steps: int = 50
weight_decay: float = 0.1
grad_clip: float = 1.0
eval_interval: int = 100
eval_steps: int = 20
save_interval: int = 500
checkpoint_dir: str = "onner_checkpoints"
dataloader_workers: int = 0
seed: int = 42
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
normed = x * torch.rsqrt(
x.pow(2).mean(dim=-1, keepdim=True) + self.eps
)
return normed * self.weight
def build_rope_cache(seq_len, head_dim, device, base=10000.0):
inv_freq = 1.0 / (
base ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim)
)
t = torch.arange(seq_len, device=device).float()
freqs = torch.outer(t, inv_freq)
cos = freqs.cos()[None, None, :, :]
sin = freqs.sin()[None, None, :, :]
return cos, sin
def apply_rope(x, cos, sin):
x_even = x[..., 0::2]
x_odd = x[..., 1::2]
x_rot_even = x_even * cos - x_odd * sin
x_rot_odd = x_even * sin + x_odd * cos
return torch.stack((x_rot_even, x_rot_odd), dim=-1).flatten(-2)
class OnnerReasearchGQAAttention(nn.Module):
def __init__(self, cfg):
super().__init__()
assert cfg.d_model % cfg.n_heads == 0
assert cfg.n_heads % cfg.n_kv_heads == 0
self.cfg = cfg
self.head_dim = cfg.d_model // cfg.n_heads
self.n_heads = cfg.n_heads
self.n_kv_heads = cfg.n_kv_heads
self.repeat_factor = cfg.n_heads // cfg.n_kv_heads
self.q_proj = nn.Linear(
cfg.d_model,
cfg.n_heads * self.head_dim,
bias=False,
)
self.k_proj = nn.Linear(
cfg.d_model,
cfg.n_kv_heads * self.head_dim,
bias=False,
)
self.v_proj = nn.Linear(
cfg.d_model,
cfg.n_kv_heads * self.head_dim,
bias=False,
)
self.o_proj = nn.Linear(
cfg.n_heads * self.head_dim,
cfg.d_model,
bias=False,
)
self.dropout = cfg.dropout
def forward(self, x):
B, T, C = x.shape
q = self.q_proj(x)
k = self.k_proj(x)
v = self.v_proj(x)
q = q.view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
k = k.view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
v = v.view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
cos, sin = build_rope_cache(
T,
self.head_dim,
x.device,
base=self.cfg.rope_base,
)
q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
if self.repeat_factor > 1:
k = k.repeat_interleave(self.repeat_factor, dim=1)
v = v.repeat_interleave(self.repeat_factor, dim=1)
y = F.scaled_dot_product_attention(
q,
k,
v,
attn_mask=None,
dropout_p=self.dropout if self.training else 0.0,
is_causal=True,
)
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.o_proj(y)
class OnnerReasearchSwiGLU(nn.Module):
def __init__(self, cfg):
super().__init__()
self.up_proj = nn.Linear(
cfg.d_model,
cfg.ffn_hidden,
bias=False,
)
self.gate_proj = nn.Linear(
cfg.d_model,
cfg.ffn_hidden,
bias=False,
)
self.down_proj = nn.Linear(
cfg.ffn_hidden,
cfg.d_model,
bias=False,
)
def forward(self, x):
return self.down_proj(
F.silu(self.gate_proj(x)) * self.up_proj(x)
)
class OnnerReasearchBlock(nn.Module):
def __init__(self, cfg):
super().__init__()
self.norm1 = RMSNorm(cfg.d_model)
self.attn = OnnerReasearchGQAAttention(cfg)
self.norm2 = RMSNorm(cfg.d_model)
self.ffn = OnnerReasearchSwiGLU(cfg)
def forward(self, x):
x = x + self.attn(self.norm1(x))
x = x + self.ffn(self.norm2(x))
return x
class OnnerMTPAdapter(nn.Module):
def __init__(self, dim):
super().__init__()
self.norm = RMSNorm(dim)
self.proj = nn.Linear(dim, dim, bias=False)
def forward(self, x):
return self.proj(self.norm(x))
class OnnerReasearchModel(nn.Module):
def __init__(self, cfg):
super().__init__()
self.cfg = cfg
self.architecture_name = cfg.architecture_name
self.model_name = cfg.model_name
self.token_emb = nn.Embedding(
cfg.vocab_size,
cfg.d_model,
)
self.blocks = nn.ModuleList([
OnnerReasearchBlock(cfg)
for _ in range(cfg.n_layers)
])
self.final_norm = RMSNorm(cfg.d_model)
self.lm_head = nn.Linear(
cfg.d_model,
cfg.vocab_size,
bias=False,
)
self.lm_head.weight = self.token_emb.weight
adapters = [nn.Identity()]
for _ in range(cfg.mtp_n - 1):
adapters.append(OnnerMTPAdapter(cfg.d_model))
self.mtp_adapters = nn.ModuleList(adapters)
def forward(self, input_ids):
x = self.token_emb(input_ids)
for block in self.blocks:
x = block(x)
h = self.final_norm(x)
logits_list = []
for adapter in self.mtp_adapters:
h_i = adapter(h)
logits = self.lm_head(h_i)
logits_list.append(logits)
return logits_list
@torch.no_grad()
def sample_next_token(logits, temperature=0.8, top_k=50):
logits = logits / max(temperature, 1e-6)
if top_k is not None:
values, _ = torch.topk(logits, top_k)
min_value = values[:, -1].unsqueeze(-1)
logits = torch.where(
logits < min_value,
torch.full_like(logits, float("-inf")),
logits,
)
probs = F.softmax(logits, dim=-1)
return torch.multinomial(probs, num_samples=1)