Download scripts/pretrain_model.py from Huypero/AgentContentVietnamese-110m: direct link, hf CLI and curl.
- Browser
- Download file 5.36 kB
-
https://huggingface.co/Huypero/AgentContentVietnamese-110m/resolve/main/scripts/pretrain_model.py
- Command line
-
hf download hf://Huypero/AgentContentVietnamese-110m/scripts/pretrain_model.py
-
curl -L -o pretrain_model.py https://huggingface.co/Huypero/AgentContentVietnamese-110m/resolve/main/scripts/pretrain_model.py
5.36 kB
| """Small dense causal LM: pre-RMSNorm, RoPE, MHA, SwiGLU, tied embeddings.""" | |
| import math | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from torch.utils.checkpoint import checkpoint | |
| class RMSNorm(nn.Module): | |
| def __init__(self, size, eps): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(size)) | |
| self.eps = eps | |
| def forward(self, x): | |
| normalized = x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + self.eps) | |
| return normalized.to(x.dtype) * self.weight | |
| class Attention(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| width = config['hidden_size'] | |
| self.heads = config['num_heads'] | |
| self.head_dim = width // self.heads | |
| self.qkv = nn.Linear(width, 3 * width, bias=False) | |
| self.output = nn.Linear(width, width, bias=False) | |
| frequencies = config['rope_theta'] ** (-torch.arange(0, self.head_dim, 2).float() / self.head_dim) | |
| angles = torch.outer(torch.arange(config['context_length']).float(), frequencies) | |
| self.register_buffer('cos', angles.cos(), persistent=False) | |
| self.register_buffer('sin', angles.sin(), persistent=False) | |
| def rotate(self, x): | |
| cos = self.cos[:x.shape[-2]].to(x.dtype) | |
| sin = self.sin[:x.shape[-2]].to(x.dtype) | |
| even, odd = x[..., 0::2], x[..., 1::2] | |
| return torch.stack((even * cos - odd * sin, even * sin + odd * cos), dim=-1).flatten(-2) | |
| def forward(self, x): | |
| batch, length, width = x.shape | |
| q, k, v = self.qkv(x).view(batch, length, 3, self.heads, self.head_dim).unbind(2) | |
| q, k, v = (part.transpose(1, 2) for part in (q, k, v)) | |
| attention = F.scaled_dot_product_attention(self.rotate(q), self.rotate(k), v, | |
| dropout_p=0.0, is_causal=True) | |
| return self.output(attention.transpose(1, 2).contiguous().view(batch, length, width)) | |
| class Block(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| width, intermediate = config['hidden_size'], config['intermediate_size'] | |
| self.attention_norm = RMSNorm(width, config['rms_norm_eps']) | |
| self.ffn_norm = RMSNorm(width, config['rms_norm_eps']) | |
| self.attention = Attention(config) | |
| self.gate = nn.Linear(width, intermediate, bias=False) | |
| self.up = nn.Linear(width, intermediate, bias=False) | |
| self.down = nn.Linear(intermediate, width, bias=False) | |
| def forward(self, x): | |
| x = x + self.attention(self.attention_norm(x)) | |
| normalized = self.ffn_norm(x) | |
| return x + self.down(F.silu(self.gate(normalized)) * self.up(normalized)) | |
| class CausalLM(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.config = dict(config) | |
| width, heads = config['hidden_size'], config['num_heads'] | |
| if width % heads or (width // heads) % 2: | |
| raise ValueError('hidden_size must divide num_heads with even head dimension') | |
| self.embedding = nn.Embedding(config['vocab_size'], width) | |
| self.blocks = nn.ModuleList(Block(config) for _ in range(config['num_layers'])) | |
| self.norm = RMSNorm(width, config['rms_norm_eps']) | |
| self.apply(self._initialize) | |
| for block in self.blocks: | |
| for layer in (block.attention.output, block.down): | |
| nn.init.normal_(layer.weight, std=0.02 / math.sqrt(2 * config['num_layers'])) | |
| def _initialize(module): | |
| if isinstance(module, (nn.Linear, nn.Embedding)): | |
| nn.init.normal_(module.weight, std=0.02) | |
| def hidden(self, input_ids): | |
| if input_ids.ndim != 2 or not 0 < input_ids.shape[1] <= self.config['context_length']: | |
| raise ValueError('Expected [batch, sequence] within context_length') | |
| x = self.embedding(input_ids) | |
| for block in self.blocks: | |
| if self.training and self.config['gradient_checkpointing']: | |
| x = checkpoint(block, x, use_reentrant=False) | |
| else: | |
| x = block(x) | |
| return self.norm(x) | |
| def forward(self, input_ids): | |
| return F.linear(self.hidden(input_ids), self.embedding.weight) | |
| def loss_sum(self, input_ids, targets, chunk_size=256): | |
| """Targets are ALREADY shifted. Recompute chunked head to bound logits memory.""" | |
| if targets.shape != input_ids.shape or chunk_size < 1: | |
| raise ValueError('Invalid targets shape or loss chunk size') | |
| hidden = self.hidden(input_ids).reshape(-1, self.config['hidden_size']) | |
| targets = targets.reshape(-1) | |
| def chunk_loss(x, y): | |
| return F.cross_entropy(F.linear(x, self.embedding.weight).float(), y, | |
| ignore_index=-100, reduction='sum') | |
| total = hidden.new_zeros((), dtype=torch.float32) | |
| for start in range(0, len(targets), chunk_size): | |
| x, y = hidden[start:start + chunk_size], targets[start:start + chunk_size] | |
| if self.training and torch.is_grad_enabled(): | |
| total = total + checkpoint(chunk_loss, x, y, use_reentrant=False) | |
| else: | |
| total = total + chunk_loss(x, y) | |
| return total | |