AMAIA / model.py
Adam25613's picture
Upload 7 files
ca0bb3f verified
Raw History Blame Contribute Delete
3.8 kB
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class CausalSelfAttention(nn.Module):
def __init__(self, cfg):
super().__init__()
self.n_heads = cfg.n_heads
self.d_head = cfg.d_model // cfg.n_heads
self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model, bias=False)
self.proj = nn.Linear(cfg.d_model, cfg.d_model, bias=False)
self.dropout = nn.Dropout(cfg.dropout)
def forward(self, x):
B, T, C = x.shape
q, k, v = self.qkv(x).split(C, dim=2)
q = q.view(B, T, self.n_heads, self.d_head).transpose(1, 2)
k = k.view(B, T, self.n_heads, self.d_head).transpose(1, 2)
v = v.view(B, T, self.n_heads, self.d_head).transpose(1, 2)
y = F.scaled_dot_product_attention(q, k, v, is_causal=True,
dropout_p=self.dropout.p if self.training else 0.0)
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.dropout(self.proj(y))
class FFN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.fc1 = nn.Linear(cfg.d_model, 4 * cfg.d_model)
self.fc2 = nn.Linear(4 * cfg.d_model, cfg.d_model)
self.dropout = nn.Dropout(cfg.dropout)
def forward(self, x):
return self.dropout(self.fc2(F.gelu(self.fc1(x))))
class Block(nn.Module):
def __init__(self, cfg):
super().__init__()
self.ln1 = nn.LayerNorm(cfg.d_model)
self.attn = CausalSelfAttention(cfg)
self.ln2 = nn.LayerNorm(cfg.d_model)
self.ffn = FFN(cfg)
def forward(self, x):
x = x + self.attn(self.ln1(x))
x = x + self.ffn(self.ln2(x))
return x
class Amaia(nn.Module):
def __init__(self, cfg):
super().__init__()
self.cfg = cfg
self.tok_emb = nn.Embedding(cfg.vocab_size, cfg.d_model)
self.pos_emb = nn.Embedding(cfg.max_seq, cfg.d_model)
self.drop = nn.Dropout(cfg.dropout)
self.blocks = nn.Sequential(*[Block(cfg) for _ in range(cfg.n_layers)])
self.ln_f = nn.LayerNorm(cfg.d_model)
self.head = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False)
self.tok_emb.weight = self.head.weight
self.apply(self._init_weights)
def _init_weights(self, m):
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, std=0.02)
if m.bias is not None:
nn.init.zeros_(m.bias)
elif isinstance(m, nn.Embedding):
nn.init.normal_(m.weight, std=0.02)
def forward(self, idx, targets=None):
B, T = idx.shape
pos = torch.arange(T, device=idx.device)
x = self.drop(self.tok_emb(idx) + self.pos_emb(pos))
x = self.blocks(x)
x = self.ln_f(x)
logits = self.head(x)
loss = None
if targets is not None:
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
return logits, loss
def n_params(self):
return sum(p.numel() for p in self.parameters())
@torch.no_grad()
def generate(self, idx, max_new_tokens, temperature=0.8, top_p=0.92):
for _ in range(max_new_tokens):
idx_cond = idx[:, -self.cfg.max_seq:]
logits, _ = self(idx_cond)
logits = logits[:, -1, :] / temperature
probs = F.softmax(logits, dim=-1)
sorted_probs, sorted_idx = torch.sort(probs, descending=True)
cumsum = torch.cumsum(sorted_probs, dim=-1)
mask = cumsum - sorted_probs > top_p
sorted_probs[mask] = 0.0
sorted_probs /= sorted_probs.sum()
next_token = sorted_idx.gather(-1, torch.multinomial(sorted_probs, 1))
idx = torch.cat([idx, next_token], dim=1)
return idx