Merlin-9M / merlin9m.py
Monster-Code's picture
Initial upload: Merlin-9M checkpoint (step 12502, 8.85M params)
b447e73 verified
Raw History Blame
7.23 kB
"""
Merlin-9M inference module.
A 8.85M-param decoder-only LM. Self-contained: no transformers needed to load it
(only for the GPT-2 tokenizer in the benchmark script).
Usage:
from merlin9m import load_model, generate
model, cfg, tok = load_model("Monster-Code/Merlin-9M")
print(generate(model, cfg, tok, "The capital of France is"))
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
# ============================================================
# MODEL
# ============================================================
class RMSNorm(nn.Module):
def __init__(self, d, eps=1e-5):
super().__init__()
self.w = nn.Parameter(torch.ones(d))
self.eps = eps
def forward(self, x):
dt = x.dtype
x = x.float()
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
return self.w * x.to(dt)
def build_rope(dim, max_len, device, base=10000.0):
inv = 1.0 / (base ** (torch.arange(0, dim, 2, device=device).float() / dim))
t = torch.arange(max_len, device=device).float()
f = torch.outer(t, inv)
f = torch.cat((f, f), dim=-1)
return torch.cos(f), torch.sin(f)
def rope_apply(x, cos, sin):
half = x.shape[-1] // 2
x1, x2 = x[..., :half], x[..., half:]
rx = torch.cat((-x2, x1), dim=-1)
return x * cos + rx * sin
class Block(nn.Module):
def __init__(self, d, nhead, ffn):
super().__init__()
self.d = d
self.nhead = nhead
self.hd = d // nhead
self.n1 = RMSNorm(d)
self.qkv = nn.Linear(d, 3 * d, bias=False)
self.o = nn.Linear(d, d, bias=False)
self.qk = RMSNorm(self.hd)
self.n2 = RMSNorm(d)
self.w1 = nn.Linear(d, ffn, bias=False)
self.w3 = nn.Linear(d, ffn, bias=False)
self.w2 = nn.Linear(ffn, d, bias=False)
def forward(self, x, cos, sin, scale):
b, t, d = x.shape
h = self.n1(x)
q, k, v = self.qkv(h).split(d, dim=-1)
q = q.view(b, t, self.nhead, self.hd).transpose(1, 2)
k = k.view(b, t, self.nhead, self.hd).transpose(1, 2)
v = v.view(b, t, self.nhead, self.hd).transpose(1, 2)
q = self.qk(q)
k = self.qk(k)
q = rope_apply(q, cos, sin)
k = rope_apply(k, cos, sin)
o = F.scaled_dot_product_attention(q, k, v, is_causal=True)
o = o.transpose(1, 2).reshape(b, t, d)
x = x + scale * self.o(o)
h = self.n2(x)
x = x + scale * self.w2(F.silu(self.w1(h)) * self.w3(h))
return x
class Brasa9M(nn.Module):
"""12 unique blocks, 18 effective through a doubled 3..8 core."""
def __init__(self, cfg):
super().__init__()
d = cfg["d_model"]
self.cfg = cfg
self.emb = nn.Embedding(cfg["vocab"], d)
self.blocks = nn.ModuleList(
[Block(d, cfg["n_heads"], cfg["ffn"]) for _ in range(cfg["n_layers"])]
)
self.norm = RMSNorm(d)
self.mtp = nn.Linear(d, d, bias=False)
self.scale_first = nn.Parameter(torch.ones(1))
self.scale_loop = nn.Parameter(torch.ones(1) * 0.85)
cos, sin = build_rope(d // cfg["n_heads"], cfg["ctx"], "cpu")
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
def trunk(self, idx):
c = self.cfg
ctx = idx.shape[1]
cos = self.rope_cos[:ctx].to(idx.device).view(1, 1, ctx, -1)
sin = self.rope_sin[:ctx].to(idx.device).view(1, 1, ctx, -1)
x = self.emb(idx)
n = c["n_layers"]
pre = 3
core_end = 9
s1 = self.scale_first
for i in range(pre):
x = self.blocks[i](x, cos, sin, s1)
for _ in range(2):
for i in range(pre, core_end):
x = self.blocks[i](x, cos, sin, s1 if _ == 0 else self.scale_loop)
for i in range(core_end, n):
x = self.blocks[i](x, cos, sin, s1)
return self.norm(x)
def head(self, h):
logits = F.linear(h, self.emb.weight)
return self.cfg["softcap"] * torch.tanh(logits / self.cfg["softcap"])
def forward(self, idx, targets=None, targets2=None):
h = self.trunk(idx)
logits = self.head(h)
if targets is None:
return logits
logits = logits[:, : targets.shape[1]]
h = h[:, : targets.shape[1]]
loss1 = F.cross_entropy(
logits.reshape(-1, logits.size(-1)).float(), targets.reshape(-1),
ignore_index=-100,
)
lse = torch.logsumexp(logits.float(), dim=-1)
zl = self.cfg["z_loss"] * (lse ** 2).mean()
loss2 = torch.tensor(0.0, device=idx.device)
if targets2 is not None and self.cfg["mtp_weight"] > 0:
h2 = self.head(self.mtp(h))
h2 = h2[:, : targets2.shape[1]]
loss2 = F.cross_entropy(
h2.reshape(-1, h2.size(-1)).float(), targets2.reshape(-1),
ignore_index=-100,
)
loss = loss1 + self.cfg["mtp_weight"] * loss2 + zl
return loss, loss1.detach(), loss2.detach(), zl.detach()
# ============================================================
# LOADER
# ============================================================
def load_model(repo_id="Monster-Code/Merlin-9M", ckpt="brasa9m_last.pt",
device=None, token=None):
"""Load config + weights from a HF repo. Returns (model, cfg, tokenizer)."""
from huggingface_hub import hf_hub_download
from transformers import AutoTokenizer
path = hf_hub_download(
repo_id=repo_id, filename=ckpt, repo_type="model", token=token
)
ck = torch.load(path, map_location="cpu", weights_only=False)
cfg = dict(ck["cfg"])
model = Brasa9M(cfg)
model.load_state_dict(ck["model"])
model.eval()
if device is None:
device = "cuda" if torch.cuda.is_available() else "cpu"
model = model.to(device)
# GPT-2 tokenizer. The model was trained on GPT-2's first 8192 ids.
tok = AutoTokenizer.from_pretrained("gpt2", token=token)
return model, cfg, tok, ck.get("step")
@torch.no_grad()
def generate(model, cfg, tok, prompt, max_new_tokens=40, temperature=0.8, top_k=50,
device=None):
if device is None:
device = next(model.parameters()).device
ids = torch.tensor([tok(prompt, add_special_tokens=False)["input_ids"]],
dtype=torch.long, device=device)
ids = ids.clamp(max=cfg["vocab"] - 1)
if ids.shape[1] < 2:
ids = ids.repeat(1, 2)
out = []
for _ in range(max_new_tokens):
inp = ids[:, -cfg["ctx"]:].clamp(max=cfg["vocab"] - 1)
logits = model(inp)[:, -1, :] / max(temperature, 1e-6)
if top_k:
k = min(top_k, logits.size(-1))
vals, idx = torch.topk(logits, k)
probs = F.softmax(vals, dim=-1)
pick = torch.multinomial(probs, 1)
nxt = idx[0, pick].item()
else:
nxt = logits.argmax(-1)[0].item()
out.append(nxt)
ids = torch.cat([ids, torch.tensor([[nxt]], device=device)], dim=1)
text = tok.decode(out, skip_special_tokens=True)
return text