"""Standalone inference & token generation script for Mimans M1. This script contains the native Mimans M1 architecture with Attention Residual Skip Gating (AttnRes), SwiGLU, and Rotary Position Embeddings (RoPE). It does NOT require HuggingFace AutoModelForCausalLM and runs natively on CPU or CUDA. Usage via CLI: python generate.py --prompt "def add(a, b):\n return" --temp 0 python generate.py --prompt "def quicksort(arr):" --max-tokens 150 --temp 0.7 Usage via Python: from generate import load_model, generate model, tokenizer = load_model(".") text = generate(model, tokenizer, "def fibonacci(n):", temperature=0.0) print(text) """ import argparse import math import os from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Tuple import torch import torch.nn as nn import torch.nn.functional as F from safetensors.torch import load_file from tokenizers import Tokenizer # ===================================================================== # 1. Architecture Components # ===================================================================== @dataclass class ModelConfig: vocab: int = 49152 d_model: int = 1280 n_layers: int = 27 n_heads: int = 10 n_kv_heads: int = 2 head_dim: int = 128 ffn_hidden: int = 3328 layers_per_block: int = 3 norm_eps: float = 1e-5 rope_theta: float = 500_000.0 max_seq_len: int = 8192 use_attnres: bool = True tied: bool = True class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-5): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: orig_dtype = x.dtype x_f32 = x.float() var = x_f32.pow(2).mean(-1, keepdim=True) normed = x_f32 * torch.rsqrt(var + self.eps) return (self.weight * normed).to(orig_dtype) class Rotary(nn.Module): def __init__(self, head_dim: int, max_seq_len: int = 8192, theta: float = 500_000.0): super().__init__() self.head_dim = head_dim self.max_seq_len = max_seq_len self.theta = theta freqs = 1.0 / (theta ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim)) angles = torch.outer(torch.arange(self.max_seq_len).float(), freqs) table = torch.cat((angles, angles), dim=-1) self.register_buffer("cos", table.cos(), persistent=False) self.register_buffer("sin", table.sin(), persistent=False) def rotate_half(x: torch.Tensor) -> torch.Tensor: left, right = x.chunk(2, dim=-1) return torch.cat((-right, left), dim=-1) def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: seq_len = x.shape[1] c = cos[:seq_len].to(x.dtype).unsqueeze(1) s = sin[:seq_len].to(x.dtype).unsqueeze(1) return x * c + rotate_half(x) * s class Attention(nn.Module): def __init__(self, cfg: ModelConfig, rope: Rotary): super().__init__() self.n_heads = cfg.n_heads self.n_kv_heads = cfg.n_kv_heads self.head_dim = cfg.head_dim self.d_model = cfg.d_model self.repeats = self.n_heads // self.n_kv_heads self.rope = rope self.scale = 1.0 / math.sqrt(self.head_dim) self.wq = nn.Linear(cfg.d_model, cfg.n_heads * cfg.head_dim, bias=False) self.wk = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False) self.wv = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False) self.wo = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model, bias=False) self.q_norm = RMSNorm(cfg.head_dim, cfg.norm_eps) self.k_norm = RMSNorm(cfg.head_dim, cfg.norm_eps) def forward(self, x: torch.Tensor) -> torch.Tensor: batch, seq_len, _ = x.shape q = self.q_norm(self.wq(x).view(batch, seq_len, self.n_heads, self.head_dim)) k = self.k_norm(self.wk(x).view(batch, seq_len, self.n_kv_heads, self.head_dim)) v = self.wv(x).view(batch, seq_len, self.n_kv_heads, self.head_dim) q = apply_rope(q, self.rope.cos, self.rope.sin) k = apply_rope(k, self.rope.cos, self.rope.sin) qh = q.transpose(1, 2) kh = k.transpose(1, 2) vh = v.transpose(1, 2) out = F.scaled_dot_product_attention(qh, kh, vh, is_causal=True, scale=self.scale, enable_gqa=True) out = out.transpose(1, 2).contiguous().view(batch, seq_len, self.d_model) return self.wo(out) class SwiGLU(nn.Module): def __init__(self, cfg: ModelConfig): super().__init__() self.w_gate = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden)) self.w_up = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden)) self.w_down = nn.Parameter(torch.empty(cfg.ffn_hidden, cfg.d_model)) def forward(self, x: torch.Tensor) -> torch.Tensor: gate = F.silu(x @ self.w_gate) up = x @ self.w_up return (gate * up) @ self.w_down class AttnResGate(nn.Module): def __init__(self, d_model: int, norm_eps: float = 1e-5): super().__init__() self.query = nn.Parameter(torch.zeros(d_model)) self.key_norm = RMSNorm(d_model, norm_eps) self.recency_bias = nn.Parameter(torch.zeros(())) def forward(self, entries: List[torch.Tensor]) -> torch.Tensor: if len(entries) == 1: return entries[0] stack = torch.stack(entries, dim=0) keys = self.key_norm(stack) logits = torch.einsum("d,nbtd->nbt", self.query.to(keys.dtype), keys) last = torch.zeros(stack.shape[0], 1, 1, device=logits.device, dtype=logits.dtype) last[-1] = 1.0 logits = logits + last * self.recency_bias.to(logits.dtype) weights = logits.softmax(dim=0) return torch.einsum("nbt,nbtd->btd", weights, stack) class ResidualState: def __init__(self, running: torch.Tensor): self.running = running self.snapshots: List[torch.Tensor] = [] self.steps = 0 def advance(self, delta: torch.Tensor, span: int): self.running = self.running + delta self.steps += 1 if self.steps % span == 0: self.snapshots.append(self.running) def entries(self) -> List[torch.Tensor]: return self.snapshots + [self.running] class TransformerBlock(nn.Module): def __init__(self, cfg: ModelConfig, rope: Rotary): super().__init__() self.cfg = cfg self.attn_norm = RMSNorm(cfg.d_model, cfg.norm_eps) self.attn = Attention(cfg, rope) self.ffn_norm = RMSNorm(cfg.d_model, cfg.norm_eps) self.ffn = SwiGLU(cfg) self.attn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None self.ffn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None self.sublayers = cfg.layers_per_block * 2 def forward(self, state: ResidualState) -> ResidualState: h = self.attn_gate(state.entries()) if self.attn_gate else state.running delta = self.attn(self.attn_norm(h)) state.advance(delta, self.sublayers) h = self.ffn_gate(state.entries()) if self.ffn_gate else state.running delta = self.ffn(self.ffn_norm(h)) state.advance(delta, self.sublayers) return state class LMHead(nn.Module): def __init__(self, cfg: ModelConfig, embedding: nn.Embedding): super().__init__() self.cfg = cfg self.embedding = [embedding] self.logit_scale = cfg.d_model ** -0.5 def forward(self, hidden: torch.Tensor) -> torch.Tensor: scaled = hidden * self.logit_scale weight = self.embedding[0].weight.t() return scaled @ weight class CodeModel(nn.Module): def __init__(self, cfg: ModelConfig): super().__init__() self.cfg = cfg self.embedding = nn.Embedding(cfg.vocab, cfg.d_model) self.rope = Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta) self.blocks = nn.ModuleList([TransformerBlock(cfg, self.rope) for _ in range(cfg.n_layers)]) self.final_norm = RMSNorm(cfg.d_model, cfg.norm_eps) self.head = LMHead(cfg, self.embedding) def forward(self, tokens: torch.Tensor) -> torch.Tensor: h = self.embedding(tokens) state = ResidualState(running=h) for block in self.blocks: block(state) normed = self.final_norm(state.running) return self.head(normed) # ===================================================================== # 2. Weights & Model Loading # ===================================================================== def remap_hf_safetensors(hf_dict: dict) -> dict: """Remap HuggingFace safetensors keys to native CodeModel state_dict.""" native = {} for k, v in hf_dict.items(): if k == "model.embed_tokens.weight": native["embedding.weight"] = v elif k == "model.norm.weight": native["final_norm.weight"] = v elif k.startswith("model.layers."): parts = k.split(".") layer_idx = parts[2] sub = ".".join(parts[3:]) prefix = f"blocks.{layer_idx}" if sub == "input_layernorm.weight": native[f"{prefix}.attn_norm.weight"] = v elif sub == "post_attention_layernorm.weight": native[f"{prefix}.ffn_norm.weight"] = v elif sub == "self_attn.q_proj.weight": native[f"{prefix}.attn.wq.weight"] = v elif sub == "self_attn.k_proj.weight": native[f"{prefix}.attn.wk.weight"] = v elif sub == "self_attn.v_proj.weight": native[f"{prefix}.attn.wv.weight"] = v elif sub == "self_attn.o_proj.weight": native[f"{prefix}.attn.wo.weight"] = v elif sub == "self_attn.q_norm.weight": native[f"{prefix}.attn.q_norm.weight"] = v elif sub == "self_attn.k_norm.weight": native[f"{prefix}.attn.k_norm.weight"] = v elif sub == "mlp.gate_proj.weight": native[f"{prefix}.ffn.w_gate"] = v elif sub == "mlp.up_proj.weight": native[f"{prefix}.ffn.w_up"] = v elif sub == "mlp.down_proj.weight": native[f"{prefix}.ffn.w_down"] = v elif sub in ("w_gate", "w_up", "w_down"): native[f"{prefix}.ffn.{sub}"] = v else: native[f"{prefix}.{sub}"] = v elif k != "lm_head.weight": native[k] = v return native def load_model(model_dir_or_repo: str = ".", device: Optional[str] = None): """Load Mimans M1 CodeModel and Tokenizer from local directory or HuggingFace repo.""" if device is None: device = "cuda" if torch.cuda.is_available() else "cpu" model_path = Path(model_dir_or_repo) safetensors_file = model_path / "model.safetensors" tokenizer_file = model_path / "tokenizer.json" # If not found locally, attempt to download from Hugging Face Hub if not safetensors_file.exists() or not tokenizer_file.exists(): try: from huggingface_hub import hf_hub_download repo_id = model_dir_or_repo if "/" in model_dir_or_repo else "ankushthakurr09/MimansM1_v1" print(f"Downloading model files from HuggingFace Hub: {repo_id}...") safetensors_file = Path(hf_hub_download(repo_id=repo_id, filename="model.safetensors")) tokenizer_file = Path(hf_hub_download(repo_id=repo_id, filename="tokenizer.json")) except Exception as e: raise FileNotFoundError( f"Could not find model files locally in '{model_dir_or_repo}' and failed to download: {e}" ) print(f"Loading weights from: {safetensors_file} ...") hf_weights = load_file(str(safetensors_file)) state_dict = remap_hf_safetensors(hf_weights) cfg = ModelConfig() model = CodeModel(cfg) model.load_state_dict(state_dict, strict=True) model = model.to(device) if device == "cuda": model = model.to(torch.bfloat16) model.eval() print(f"Loading tokenizer from: {tokenizer_file} ...") tokenizer = Tokenizer.from_file(str(tokenizer_file)) print(f"Model loaded successfully on {device} ({sum(p.numel() for p in model.parameters())/1e6:.1f}M params)!") return model, tokenizer # ===================================================================== # 3. Text Generation Engine # ===================================================================== def generate( model: CodeModel, tokenizer: Tokenizer, prompt: str, max_tokens: int = 100, temperature: float = 0.0, top_k: int = 50, repetition_penalty: float = 1.15, ) -> str: """Generate tokens from prompt using greedy or sampled decoding.""" device = next(model.parameters()).device encoded = tokenizer.encode(prompt) input_ids = torch.tensor([encoded.ids], dtype=torch.long, device=device) generated_ids = list(encoded.ids) with torch.no_grad(): for _ in range(max_tokens): curr_input = input_ids[:, -model.cfg.max_seq_len :] if device.type == "cuda": with torch.autocast("cuda", dtype=torch.bfloat16): logits = model(curr_input) else: logits = model(curr_input) next_logits = logits[0, -1, :].clone() # Greedy decoding (argmax) if temperature <= 1e-4: next_token = int(torch.argmax(next_logits).item()) else: # Repetition penalty if repetition_penalty != 1.0 and len(generated_ids) > 0: for token_id in set(generated_ids): if next_logits[token_id] < 0: next_logits[token_id] *= repetition_penalty else: next_logits[token_id] /= repetition_penalty next_logits = next_logits / temperature if top_k > 0: v, _ = torch.topk(next_logits, min(top_k, next_logits.size(-1))) next_logits[next_logits < v[-1]] = -float("Inf") probs = F.softmax(next_logits, dim=-1) next_token = torch.multinomial(probs, num_samples=1).item() generated_ids.append(next_token) input_ids = torch.cat([input_ids, torch.tensor([[next_token]], device=device)], dim=1) if next_token == 0: # <|endoftext|> break return tokenizer.decode(generated_ids, skip_special_tokens=False) # ===================================================================== # 4. Command Line Entry Point # ===================================================================== if __name__ == "__main__": parser = argparse.ArgumentParser(description="Mimans M1 Native Token Generation") parser.add_argument("--model-path", default=".", help="Local path or HuggingFace repo ID") parser.add_argument("--prompt", default="def add(a, b):\n return", help="Input prompt") parser.add_argument("--max-tokens", type=int, default=100, help="Max tokens to generate") parser.add_argument("--temp", type=float, default=0.0, help="Temperature (0.0 for greedy)") parser.add_argument("--top-k", type=int, default=50, help="Top-K sampling") parser.add_argument("--rep-penalty", type=float, default=1.15, help="Repetition penalty") parser.add_argument("--device", default=None, help="Device ('cpu' or 'cuda')") args = parser.parse_args() model, tokenizer = load_model(args.model_path, device=args.device) print("\n" + "=" * 60) print(f"PROMPT:\n{args.prompt}") print("=" * 60) output = generate( model, tokenizer, args.prompt, max_tokens=args.max_tokens, temperature=args.temp, top_k=args.top_k, repetition_penalty=args.rep_penalty, ) print("GENERATED OUTPUT:") print(output) print("=" * 60)