MimansM1_v1 / generate.py
ankushthakurr09's picture
Add native standalone generate.py for AttnRes token generation
58a1a0f verified
Raw History Blame Contribute Delete
16.1 kB
"""Standalone inference & token generation script for Mimans M1.
This script contains the native Mimans M1 architecture with Attention Residual
Skip Gating (AttnRes), SwiGLU, and Rotary Position Embeddings (RoPE).
It does NOT require HuggingFace AutoModelForCausalLM and runs natively on CPU or CUDA.
Usage via CLI:
python generate.py --prompt "def add(a, b):\n return" --temp 0
python generate.py --prompt "def quicksort(arr):" --max-tokens 150 --temp 0.7
Usage via Python:
from generate import load_model, generate
model, tokenizer = load_model(".")
text = generate(model, tokenizer, "def fibonacci(n):", temperature=0.0)
print(text)
"""
import argparse
import math
import os
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file
from tokenizers import Tokenizer
# =====================================================================
# 1. Architecture Components
# =====================================================================
@dataclass
class ModelConfig:
vocab: int = 49152
d_model: int = 1280
n_layers: int = 27
n_heads: int = 10
n_kv_heads: int = 2
head_dim: int = 128
ffn_hidden: int = 3328
layers_per_block: int = 3
norm_eps: float = 1e-5
rope_theta: float = 500_000.0
max_seq_len: int = 8192
use_attnres: bool = True
tied: bool = True
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-5):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
orig_dtype = x.dtype
x_f32 = x.float()
var = x_f32.pow(2).mean(-1, keepdim=True)
normed = x_f32 * torch.rsqrt(var + self.eps)
return (self.weight * normed).to(orig_dtype)
class Rotary(nn.Module):
def __init__(self, head_dim: int, max_seq_len: int = 8192, theta: float = 500_000.0):
super().__init__()
self.head_dim = head_dim
self.max_seq_len = max_seq_len
self.theta = theta
freqs = 1.0 / (theta ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim))
angles = torch.outer(torch.arange(self.max_seq_len).float(), freqs)
table = torch.cat((angles, angles), dim=-1)
self.register_buffer("cos", table.cos(), persistent=False)
self.register_buffer("sin", table.sin(), persistent=False)
def rotate_half(x: torch.Tensor) -> torch.Tensor:
left, right = x.chunk(2, dim=-1)
return torch.cat((-right, left), dim=-1)
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
seq_len = x.shape[1]
c = cos[:seq_len].to(x.dtype).unsqueeze(1)
s = sin[:seq_len].to(x.dtype).unsqueeze(1)
return x * c + rotate_half(x) * s
class Attention(nn.Module):
def __init__(self, cfg: ModelConfig, rope: Rotary):
super().__init__()
self.n_heads = cfg.n_heads
self.n_kv_heads = cfg.n_kv_heads
self.head_dim = cfg.head_dim
self.d_model = cfg.d_model
self.repeats = self.n_heads // self.n_kv_heads
self.rope = rope
self.scale = 1.0 / math.sqrt(self.head_dim)
self.wq = nn.Linear(cfg.d_model, cfg.n_heads * cfg.head_dim, bias=False)
self.wk = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
self.wv = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
self.wo = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model, bias=False)
self.q_norm = RMSNorm(cfg.head_dim, cfg.norm_eps)
self.k_norm = RMSNorm(cfg.head_dim, cfg.norm_eps)
def forward(self, x: torch.Tensor) -> torch.Tensor:
batch, seq_len, _ = x.shape
q = self.q_norm(self.wq(x).view(batch, seq_len, self.n_heads, self.head_dim))
k = self.k_norm(self.wk(x).view(batch, seq_len, self.n_kv_heads, self.head_dim))
v = self.wv(x).view(batch, seq_len, self.n_kv_heads, self.head_dim)
q = apply_rope(q, self.rope.cos, self.rope.sin)
k = apply_rope(k, self.rope.cos, self.rope.sin)
qh = q.transpose(1, 2)
kh = k.transpose(1, 2)
vh = v.transpose(1, 2)
out = F.scaled_dot_product_attention(qh, kh, vh, is_causal=True, scale=self.scale, enable_gqa=True)
out = out.transpose(1, 2).contiguous().view(batch, seq_len, self.d_model)
return self.wo(out)
class SwiGLU(nn.Module):
def __init__(self, cfg: ModelConfig):
super().__init__()
self.w_gate = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden))
self.w_up = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden))
self.w_down = nn.Parameter(torch.empty(cfg.ffn_hidden, cfg.d_model))
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate = F.silu(x @ self.w_gate)
up = x @ self.w_up
return (gate * up) @ self.w_down
class AttnResGate(nn.Module):
def __init__(self, d_model: int, norm_eps: float = 1e-5):
super().__init__()
self.query = nn.Parameter(torch.zeros(d_model))
self.key_norm = RMSNorm(d_model, norm_eps)
self.recency_bias = nn.Parameter(torch.zeros(()))
def forward(self, entries: List[torch.Tensor]) -> torch.Tensor:
if len(entries) == 1:
return entries[0]
stack = torch.stack(entries, dim=0)
keys = self.key_norm(stack)
logits = torch.einsum("d,nbtd->nbt", self.query.to(keys.dtype), keys)
last = torch.zeros(stack.shape[0], 1, 1, device=logits.device, dtype=logits.dtype)
last[-1] = 1.0
logits = logits + last * self.recency_bias.to(logits.dtype)
weights = logits.softmax(dim=0)
return torch.einsum("nbt,nbtd->btd", weights, stack)
class ResidualState:
def __init__(self, running: torch.Tensor):
self.running = running
self.snapshots: List[torch.Tensor] = []
self.steps = 0
def advance(self, delta: torch.Tensor, span: int):
self.running = self.running + delta
self.steps += 1
if self.steps % span == 0:
self.snapshots.append(self.running)
def entries(self) -> List[torch.Tensor]:
return self.snapshots + [self.running]
class TransformerBlock(nn.Module):
def __init__(self, cfg: ModelConfig, rope: Rotary):
super().__init__()
self.cfg = cfg
self.attn_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
self.attn = Attention(cfg, rope)
self.ffn_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
self.ffn = SwiGLU(cfg)
self.attn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None
self.ffn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None
self.sublayers = cfg.layers_per_block * 2
def forward(self, state: ResidualState) -> ResidualState:
h = self.attn_gate(state.entries()) if self.attn_gate else state.running
delta = self.attn(self.attn_norm(h))
state.advance(delta, self.sublayers)
h = self.ffn_gate(state.entries()) if self.ffn_gate else state.running
delta = self.ffn(self.ffn_norm(h))
state.advance(delta, self.sublayers)
return state
class LMHead(nn.Module):
def __init__(self, cfg: ModelConfig, embedding: nn.Embedding):
super().__init__()
self.cfg = cfg
self.embedding = [embedding]
self.logit_scale = cfg.d_model ** -0.5
def forward(self, hidden: torch.Tensor) -> torch.Tensor:
scaled = hidden * self.logit_scale
weight = self.embedding[0].weight.t()
return scaled @ weight
class CodeModel(nn.Module):
def __init__(self, cfg: ModelConfig):
super().__init__()
self.cfg = cfg
self.embedding = nn.Embedding(cfg.vocab, cfg.d_model)
self.rope = Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
self.blocks = nn.ModuleList([TransformerBlock(cfg, self.rope) for _ in range(cfg.n_layers)])
self.final_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
self.head = LMHead(cfg, self.embedding)
def forward(self, tokens: torch.Tensor) -> torch.Tensor:
h = self.embedding(tokens)
state = ResidualState(running=h)
for block in self.blocks:
block(state)
normed = self.final_norm(state.running)
return self.head(normed)
# =====================================================================
# 2. Weights & Model Loading
# =====================================================================
def remap_hf_safetensors(hf_dict: dict) -> dict:
"""Remap HuggingFace safetensors keys to native CodeModel state_dict."""
native = {}
for k, v in hf_dict.items():
if k == "model.embed_tokens.weight":
native["embedding.weight"] = v
elif k == "model.norm.weight":
native["final_norm.weight"] = v
elif k.startswith("model.layers."):
parts = k.split(".")
layer_idx = parts[2]
sub = ".".join(parts[3:])
prefix = f"blocks.{layer_idx}"
if sub == "input_layernorm.weight":
native[f"{prefix}.attn_norm.weight"] = v
elif sub == "post_attention_layernorm.weight":
native[f"{prefix}.ffn_norm.weight"] = v
elif sub == "self_attn.q_proj.weight":
native[f"{prefix}.attn.wq.weight"] = v
elif sub == "self_attn.k_proj.weight":
native[f"{prefix}.attn.wk.weight"] = v
elif sub == "self_attn.v_proj.weight":
native[f"{prefix}.attn.wv.weight"] = v
elif sub == "self_attn.o_proj.weight":
native[f"{prefix}.attn.wo.weight"] = v
elif sub == "self_attn.q_norm.weight":
native[f"{prefix}.attn.q_norm.weight"] = v
elif sub == "self_attn.k_norm.weight":
native[f"{prefix}.attn.k_norm.weight"] = v
elif sub == "mlp.gate_proj.weight":
native[f"{prefix}.ffn.w_gate"] = v
elif sub == "mlp.up_proj.weight":
native[f"{prefix}.ffn.w_up"] = v
elif sub == "mlp.down_proj.weight":
native[f"{prefix}.ffn.w_down"] = v
elif sub in ("w_gate", "w_up", "w_down"):
native[f"{prefix}.ffn.{sub}"] = v
else:
native[f"{prefix}.{sub}"] = v
elif k != "lm_head.weight":
native[k] = v
return native
def load_model(model_dir_or_repo: str = ".", device: Optional[str] = None):
"""Load Mimans M1 CodeModel and Tokenizer from local directory or HuggingFace repo."""
if device is None:
device = "cuda" if torch.cuda.is_available() else "cpu"
model_path = Path(model_dir_or_repo)
safetensors_file = model_path / "model.safetensors"
tokenizer_file = model_path / "tokenizer.json"
# If not found locally, attempt to download from Hugging Face Hub
if not safetensors_file.exists() or not tokenizer_file.exists():
try:
from huggingface_hub import hf_hub_download
repo_id = model_dir_or_repo if "/" in model_dir_or_repo else "ankushthakurr09/MimansM1_v1"
print(f"Downloading model files from HuggingFace Hub: {repo_id}...")
safetensors_file = Path(hf_hub_download(repo_id=repo_id, filename="model.safetensors"))
tokenizer_file = Path(hf_hub_download(repo_id=repo_id, filename="tokenizer.json"))
except Exception as e:
raise FileNotFoundError(
f"Could not find model files locally in '{model_dir_or_repo}' and failed to download: {e}"
)
print(f"Loading weights from: {safetensors_file} ...")
hf_weights = load_file(str(safetensors_file))
state_dict = remap_hf_safetensors(hf_weights)
cfg = ModelConfig()
model = CodeModel(cfg)
model.load_state_dict(state_dict, strict=True)
model = model.to(device)
if device == "cuda":
model = model.to(torch.bfloat16)
model.eval()
print(f"Loading tokenizer from: {tokenizer_file} ...")
tokenizer = Tokenizer.from_file(str(tokenizer_file))
print(f"Model loaded successfully on {device} ({sum(p.numel() for p in model.parameters())/1e6:.1f}M params)!")
return model, tokenizer
# =====================================================================
# 3. Text Generation Engine
# =====================================================================
def generate(
model: CodeModel,
tokenizer: Tokenizer,
prompt: str,
max_tokens: int = 100,
temperature: float = 0.0,
top_k: int = 50,
repetition_penalty: float = 1.15,
) -> str:
"""Generate tokens from prompt using greedy or sampled decoding."""
device = next(model.parameters()).device
encoded = tokenizer.encode(prompt)
input_ids = torch.tensor([encoded.ids], dtype=torch.long, device=device)
generated_ids = list(encoded.ids)
with torch.no_grad():
for _ in range(max_tokens):
curr_input = input_ids[:, -model.cfg.max_seq_len :]
if device.type == "cuda":
with torch.autocast("cuda", dtype=torch.bfloat16):
logits = model(curr_input)
else:
logits = model(curr_input)
next_logits = logits[0, -1, :].clone()
# Greedy decoding (argmax)
if temperature <= 1e-4:
next_token = int(torch.argmax(next_logits).item())
else:
# Repetition penalty
if repetition_penalty != 1.0 and len(generated_ids) > 0:
for token_id in set(generated_ids):
if next_logits[token_id] < 0:
next_logits[token_id] *= repetition_penalty
else:
next_logits[token_id] /= repetition_penalty
next_logits = next_logits / temperature
if top_k > 0:
v, _ = torch.topk(next_logits, min(top_k, next_logits.size(-1)))
next_logits[next_logits < v[-1]] = -float("Inf")
probs = F.softmax(next_logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1).item()
generated_ids.append(next_token)
input_ids = torch.cat([input_ids, torch.tensor([[next_token]], device=device)], dim=1)
if next_token == 0: # <|endoftext|>
break
return tokenizer.decode(generated_ids, skip_special_tokens=False)
# =====================================================================
# 4. Command Line Entry Point
# =====================================================================
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Mimans M1 Native Token Generation")
parser.add_argument("--model-path", default=".", help="Local path or HuggingFace repo ID")
parser.add_argument("--prompt", default="def add(a, b):\n return", help="Input prompt")
parser.add_argument("--max-tokens", type=int, default=100, help="Max tokens to generate")
parser.add_argument("--temp", type=float, default=0.0, help="Temperature (0.0 for greedy)")
parser.add_argument("--top-k", type=int, default=50, help="Top-K sampling")
parser.add_argument("--rep-penalty", type=float, default=1.15, help="Repetition penalty")
parser.add_argument("--device", default=None, help="Device ('cpu' or 'cuda')")
args = parser.parse_args()
model, tokenizer = load_model(args.model_path, device=args.device)
print("\n" + "=" * 60)
print(f"PROMPT:\n{args.prompt}")
print("=" * 60)
output = generate(
model,
tokenizer,
args.prompt,
max_tokens=args.max_tokens,
temperature=args.temp,
top_k=args.top_k,
repetition_penalty=args.rep_penalty,
)
print("GENERATED OUTPUT:")
print(output)
print("=" * 60)