AgentContentVietnamese-110m / scripts /pretrain_model.py
Huypero's picture
Upload verified final Vietnamese 110M pretrained model
982c9f8 verified
Raw History Blame Contribute Delete
5.36 kB
"""Small dense causal LM: pre-RMSNorm, RoPE, MHA, SwiGLU, tied embeddings."""
import math
import torch
from torch import nn
from torch.nn import functional as F
from torch.utils.checkpoint import checkpoint
class RMSNorm(nn.Module):
def __init__(self, size, eps):
super().__init__()
self.weight = nn.Parameter(torch.ones(size))
self.eps = eps
def forward(self, x):
normalized = x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + self.eps)
return normalized.to(x.dtype) * self.weight
class Attention(nn.Module):
def __init__(self, config):
super().__init__()
width = config['hidden_size']
self.heads = config['num_heads']
self.head_dim = width // self.heads
self.qkv = nn.Linear(width, 3 * width, bias=False)
self.output = nn.Linear(width, width, bias=False)
frequencies = config['rope_theta'] ** (-torch.arange(0, self.head_dim, 2).float() / self.head_dim)
angles = torch.outer(torch.arange(config['context_length']).float(), frequencies)
self.register_buffer('cos', angles.cos(), persistent=False)
self.register_buffer('sin', angles.sin(), persistent=False)
def rotate(self, x):
cos = self.cos[:x.shape[-2]].to(x.dtype)
sin = self.sin[:x.shape[-2]].to(x.dtype)
even, odd = x[..., 0::2], x[..., 1::2]
return torch.stack((even * cos - odd * sin, even * sin + odd * cos), dim=-1).flatten(-2)
def forward(self, x):
batch, length, width = x.shape
q, k, v = self.qkv(x).view(batch, length, 3, self.heads, self.head_dim).unbind(2)
q, k, v = (part.transpose(1, 2) for part in (q, k, v))
attention = F.scaled_dot_product_attention(self.rotate(q), self.rotate(k), v,
dropout_p=0.0, is_causal=True)
return self.output(attention.transpose(1, 2).contiguous().view(batch, length, width))
class Block(nn.Module):
def __init__(self, config):
super().__init__()
width, intermediate = config['hidden_size'], config['intermediate_size']
self.attention_norm = RMSNorm(width, config['rms_norm_eps'])
self.ffn_norm = RMSNorm(width, config['rms_norm_eps'])
self.attention = Attention(config)
self.gate = nn.Linear(width, intermediate, bias=False)
self.up = nn.Linear(width, intermediate, bias=False)
self.down = nn.Linear(intermediate, width, bias=False)
def forward(self, x):
x = x + self.attention(self.attention_norm(x))
normalized = self.ffn_norm(x)
return x + self.down(F.silu(self.gate(normalized)) * self.up(normalized))
class CausalLM(nn.Module):
def __init__(self, config):
super().__init__()
self.config = dict(config)
width, heads = config['hidden_size'], config['num_heads']
if width % heads or (width // heads) % 2:
raise ValueError('hidden_size must divide num_heads with even head dimension')
self.embedding = nn.Embedding(config['vocab_size'], width)
self.blocks = nn.ModuleList(Block(config) for _ in range(config['num_layers']))
self.norm = RMSNorm(width, config['rms_norm_eps'])
self.apply(self._initialize)
for block in self.blocks:
for layer in (block.attention.output, block.down):
nn.init.normal_(layer.weight, std=0.02 / math.sqrt(2 * config['num_layers']))
@staticmethod
def _initialize(module):
if isinstance(module, (nn.Linear, nn.Embedding)):
nn.init.normal_(module.weight, std=0.02)
def hidden(self, input_ids):
if input_ids.ndim != 2 or not 0 < input_ids.shape[1] <= self.config['context_length']:
raise ValueError('Expected [batch, sequence] within context_length')
x = self.embedding(input_ids)
for block in self.blocks:
if self.training and self.config['gradient_checkpointing']:
x = checkpoint(block, x, use_reentrant=False)
else:
x = block(x)
return self.norm(x)
def forward(self, input_ids):
return F.linear(self.hidden(input_ids), self.embedding.weight)
def loss_sum(self, input_ids, targets, chunk_size=256):
"""Targets are ALREADY shifted. Recompute chunked head to bound logits memory."""
if targets.shape != input_ids.shape or chunk_size < 1:
raise ValueError('Invalid targets shape or loss chunk size')
hidden = self.hidden(input_ids).reshape(-1, self.config['hidden_size'])
targets = targets.reshape(-1)
def chunk_loss(x, y):
return F.cross_entropy(F.linear(x, self.embedding.weight).float(), y,
ignore_index=-100, reduction='sum')
total = hidden.new_zeros((), dtype=torch.float32)
for start in range(0, len(targets), chunk_size):
x, y = hidden[start:start + chunk_size], targets[start:start + chunk_size]
if self.training and torch.is_grad_enabled():
total = total + checkpoint(chunk_loss, x, y, use_reentrant=False)
else:
total = total + chunk_loss(x, y)
return total