""" Merlin-9M inference module. A 8.85M-param decoder-only LM. Self-contained: no transformers needed to load it (only for the GPT-2 tokenizer in the benchmark script). Usage: from merlin9m import load_model, generate model, cfg, tok = load_model("Monster-Code/Merlin-9M") print(generate(model, cfg, tok, "The capital of France is")) """ import torch import torch.nn as nn import torch.nn.functional as F # ============================================================ # MODEL # ============================================================ class RMSNorm(nn.Module): def __init__(self, d, eps=1e-5): super().__init__() self.w = nn.Parameter(torch.ones(d)) self.eps = eps def forward(self, x): dt = x.dtype x = x.float() x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) return self.w * x.to(dt) def build_rope(dim, max_len, device, base=10000.0): inv = 1.0 / (base ** (torch.arange(0, dim, 2, device=device).float() / dim)) t = torch.arange(max_len, device=device).float() f = torch.outer(t, inv) f = torch.cat((f, f), dim=-1) return torch.cos(f), torch.sin(f) def rope_apply(x, cos, sin): half = x.shape[-1] // 2 x1, x2 = x[..., :half], x[..., half:] rx = torch.cat((-x2, x1), dim=-1) return x * cos + rx * sin class Block(nn.Module): def __init__(self, d, nhead, ffn): super().__init__() self.d = d self.nhead = nhead self.hd = d // nhead self.n1 = RMSNorm(d) self.qkv = nn.Linear(d, 3 * d, bias=False) self.o = nn.Linear(d, d, bias=False) self.qk = RMSNorm(self.hd) self.n2 = RMSNorm(d) self.w1 = nn.Linear(d, ffn, bias=False) self.w3 = nn.Linear(d, ffn, bias=False) self.w2 = nn.Linear(ffn, d, bias=False) def forward(self, x, cos, sin, scale): b, t, d = x.shape h = self.n1(x) q, k, v = self.qkv(h).split(d, dim=-1) q = q.view(b, t, self.nhead, self.hd).transpose(1, 2) k = k.view(b, t, self.nhead, self.hd).transpose(1, 2) v = v.view(b, t, self.nhead, self.hd).transpose(1, 2) q = self.qk(q) k = self.qk(k) q = rope_apply(q, cos, sin) k = rope_apply(k, cos, sin) o = F.scaled_dot_product_attention(q, k, v, is_causal=True) o = o.transpose(1, 2).reshape(b, t, d) x = x + scale * self.o(o) h = self.n2(x) x = x + scale * self.w2(F.silu(self.w1(h)) * self.w3(h)) return x class Brasa9M(nn.Module): """12 unique blocks, 18 effective through a doubled 3..8 core.""" def __init__(self, cfg): super().__init__() d = cfg["d_model"] self.cfg = cfg self.emb = nn.Embedding(cfg["vocab"], d) self.blocks = nn.ModuleList( [Block(d, cfg["n_heads"], cfg["ffn"]) for _ in range(cfg["n_layers"])] ) self.norm = RMSNorm(d) self.mtp = nn.Linear(d, d, bias=False) self.scale_first = nn.Parameter(torch.ones(1)) self.scale_loop = nn.Parameter(torch.ones(1) * 0.85) cos, sin = build_rope(d // cfg["n_heads"], cfg["ctx"], "cpu") self.register_buffer("rope_cos", cos, persistent=False) self.register_buffer("rope_sin", sin, persistent=False) def trunk(self, idx): c = self.cfg ctx = idx.shape[1] cos = self.rope_cos[:ctx].to(idx.device).view(1, 1, ctx, -1) sin = self.rope_sin[:ctx].to(idx.device).view(1, 1, ctx, -1) x = self.emb(idx) n = c["n_layers"] pre = 3 core_end = 9 s1 = self.scale_first for i in range(pre): x = self.blocks[i](x, cos, sin, s1) for _ in range(2): for i in range(pre, core_end): x = self.blocks[i](x, cos, sin, s1 if _ == 0 else self.scale_loop) for i in range(core_end, n): x = self.blocks[i](x, cos, sin, s1) return self.norm(x) def head(self, h): logits = F.linear(h, self.emb.weight) return self.cfg["softcap"] * torch.tanh(logits / self.cfg["softcap"]) def forward(self, idx, targets=None, targets2=None): h = self.trunk(idx) logits = self.head(h) if targets is None: return logits logits = logits[:, : targets.shape[1]] h = h[:, : targets.shape[1]] loss1 = F.cross_entropy( logits.reshape(-1, logits.size(-1)).float(), targets.reshape(-1), ignore_index=-100, ) lse = torch.logsumexp(logits.float(), dim=-1) zl = self.cfg["z_loss"] * (lse ** 2).mean() loss2 = torch.tensor(0.0, device=idx.device) if targets2 is not None and self.cfg["mtp_weight"] > 0: h2 = self.head(self.mtp(h)) h2 = h2[:, : targets2.shape[1]] loss2 = F.cross_entropy( h2.reshape(-1, h2.size(-1)).float(), targets2.reshape(-1), ignore_index=-100, ) loss = loss1 + self.cfg["mtp_weight"] * loss2 + zl return loss, loss1.detach(), loss2.detach(), zl.detach() # ============================================================ # LOADER # ============================================================ def load_model(repo_id="Monster-Code/Merlin-9M", ckpt="brasa9m_last.pt", device=None, token=None): """Load config + weights from a HF repo. Returns (model, cfg, tokenizer).""" from huggingface_hub import hf_hub_download from transformers import AutoTokenizer path = hf_hub_download( repo_id=repo_id, filename=ckpt, repo_type="model", token=token ) ck = torch.load(path, map_location="cpu", weights_only=False) cfg = dict(ck["cfg"]) model = Brasa9M(cfg) model.load_state_dict(ck["model"]) model.eval() if device is None: device = "cuda" if torch.cuda.is_available() else "cpu" model = model.to(device) # GPT-2 tokenizer. The model was trained on GPT-2's first 8192 ids. tok = AutoTokenizer.from_pretrained("gpt2", token=token) return model, cfg, tok, ck.get("step") @torch.no_grad() def generate(model, cfg, tok, prompt, max_new_tokens=40, temperature=0.8, top_k=50, device=None): if device is None: device = next(model.parameters()).device ids = torch.tensor([tok(prompt, add_special_tokens=False)["input_ids"]], dtype=torch.long, device=device) ids = ids.clamp(max=cfg["vocab"] - 1) if ids.shape[1] < 2: ids = ids.repeat(1, 2) out = [] for _ in range(max_new_tokens): inp = ids[:, -cfg["ctx"]:].clamp(max=cfg["vocab"] - 1) logits = model(inp)[:, -1, :] / max(temperature, 1e-6) if top_k: k = min(top_k, logits.size(-1)) vals, idx = torch.topk(logits, k) probs = F.softmax(vals, dim=-1) pick = torch.multinomial(probs, 1) nxt = idx[0, pick].item() else: nxt = logits.argmax(-1)[0].item() out.append(nxt) ids = torch.cat([ids, torch.tensor([[nxt]], device=device)], dim=1) text = tok.decode(out, skip_special_tokens=True) return text