Download generate.py from MiamnusAI/MimansM1_v1: direct link, hf CLI and curl.
- Browser
- Download file 16.1 kB
-
https://huggingface.co/MiamnusAI/MimansM1_v1/resolve/main/generate.py
- Command line
-
hf download hf://MiamnusAI/MimansM1_v1/generate.py
-
curl -L -o generate.py https://huggingface.co/MiamnusAI/MimansM1_v1/resolve/main/generate.py
16.1 kB
| """Standalone inference & token generation script for Mimans M1. | |
| This script contains the native Mimans M1 architecture with Attention Residual | |
| Skip Gating (AttnRes), SwiGLU, and Rotary Position Embeddings (RoPE). | |
| It does NOT require HuggingFace AutoModelForCausalLM and runs natively on CPU or CUDA. | |
| Usage via CLI: | |
| python generate.py --prompt "def add(a, b):\n return" --temp 0 | |
| python generate.py --prompt "def quicksort(arr):" --max-tokens 150 --temp 0.7 | |
| Usage via Python: | |
| from generate import load_model, generate | |
| model, tokenizer = load_model(".") | |
| text = generate(model, tokenizer, "def fibonacci(n):", temperature=0.0) | |
| print(text) | |
| """ | |
| import argparse | |
| import math | |
| import os | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import List, Optional, Tuple | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| from tokenizers import Tokenizer | |
| # ===================================================================== | |
| # 1. Architecture Components | |
| # ===================================================================== | |
| class ModelConfig: | |
| vocab: int = 49152 | |
| d_model: int = 1280 | |
| n_layers: int = 27 | |
| n_heads: int = 10 | |
| n_kv_heads: int = 2 | |
| head_dim: int = 128 | |
| ffn_hidden: int = 3328 | |
| layers_per_block: int = 3 | |
| norm_eps: float = 1e-5 | |
| rope_theta: float = 500_000.0 | |
| max_seq_len: int = 8192 | |
| use_attnres: bool = True | |
| tied: bool = True | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim: int, eps: float = 1e-5): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| orig_dtype = x.dtype | |
| x_f32 = x.float() | |
| var = x_f32.pow(2).mean(-1, keepdim=True) | |
| normed = x_f32 * torch.rsqrt(var + self.eps) | |
| return (self.weight * normed).to(orig_dtype) | |
| class Rotary(nn.Module): | |
| def __init__(self, head_dim: int, max_seq_len: int = 8192, theta: float = 500_000.0): | |
| super().__init__() | |
| self.head_dim = head_dim | |
| self.max_seq_len = max_seq_len | |
| self.theta = theta | |
| freqs = 1.0 / (theta ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim)) | |
| angles = torch.outer(torch.arange(self.max_seq_len).float(), freqs) | |
| table = torch.cat((angles, angles), dim=-1) | |
| self.register_buffer("cos", table.cos(), persistent=False) | |
| self.register_buffer("sin", table.sin(), persistent=False) | |
| def rotate_half(x: torch.Tensor) -> torch.Tensor: | |
| left, right = x.chunk(2, dim=-1) | |
| return torch.cat((-right, left), dim=-1) | |
| def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| seq_len = x.shape[1] | |
| c = cos[:seq_len].to(x.dtype).unsqueeze(1) | |
| s = sin[:seq_len].to(x.dtype).unsqueeze(1) | |
| return x * c + rotate_half(x) * s | |
| class Attention(nn.Module): | |
| def __init__(self, cfg: ModelConfig, rope: Rotary): | |
| super().__init__() | |
| self.n_heads = cfg.n_heads | |
| self.n_kv_heads = cfg.n_kv_heads | |
| self.head_dim = cfg.head_dim | |
| self.d_model = cfg.d_model | |
| self.repeats = self.n_heads // self.n_kv_heads | |
| self.rope = rope | |
| self.scale = 1.0 / math.sqrt(self.head_dim) | |
| self.wq = nn.Linear(cfg.d_model, cfg.n_heads * cfg.head_dim, bias=False) | |
| self.wk = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False) | |
| self.wv = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False) | |
| self.wo = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model, bias=False) | |
| self.q_norm = RMSNorm(cfg.head_dim, cfg.norm_eps) | |
| self.k_norm = RMSNorm(cfg.head_dim, cfg.norm_eps) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| batch, seq_len, _ = x.shape | |
| q = self.q_norm(self.wq(x).view(batch, seq_len, self.n_heads, self.head_dim)) | |
| k = self.k_norm(self.wk(x).view(batch, seq_len, self.n_kv_heads, self.head_dim)) | |
| v = self.wv(x).view(batch, seq_len, self.n_kv_heads, self.head_dim) | |
| q = apply_rope(q, self.rope.cos, self.rope.sin) | |
| k = apply_rope(k, self.rope.cos, self.rope.sin) | |
| qh = q.transpose(1, 2) | |
| kh = k.transpose(1, 2) | |
| vh = v.transpose(1, 2) | |
| out = F.scaled_dot_product_attention(qh, kh, vh, is_causal=True, scale=self.scale, enable_gqa=True) | |
| out = out.transpose(1, 2).contiguous().view(batch, seq_len, self.d_model) | |
| return self.wo(out) | |
| class SwiGLU(nn.Module): | |
| def __init__(self, cfg: ModelConfig): | |
| super().__init__() | |
| self.w_gate = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden)) | |
| self.w_up = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden)) | |
| self.w_down = nn.Parameter(torch.empty(cfg.ffn_hidden, cfg.d_model)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| gate = F.silu(x @ self.w_gate) | |
| up = x @ self.w_up | |
| return (gate * up) @ self.w_down | |
| class AttnResGate(nn.Module): | |
| def __init__(self, d_model: int, norm_eps: float = 1e-5): | |
| super().__init__() | |
| self.query = nn.Parameter(torch.zeros(d_model)) | |
| self.key_norm = RMSNorm(d_model, norm_eps) | |
| self.recency_bias = nn.Parameter(torch.zeros(())) | |
| def forward(self, entries: List[torch.Tensor]) -> torch.Tensor: | |
| if len(entries) == 1: | |
| return entries[0] | |
| stack = torch.stack(entries, dim=0) | |
| keys = self.key_norm(stack) | |
| logits = torch.einsum("d,nbtd->nbt", self.query.to(keys.dtype), keys) | |
| last = torch.zeros(stack.shape[0], 1, 1, device=logits.device, dtype=logits.dtype) | |
| last[-1] = 1.0 | |
| logits = logits + last * self.recency_bias.to(logits.dtype) | |
| weights = logits.softmax(dim=0) | |
| return torch.einsum("nbt,nbtd->btd", weights, stack) | |
| class ResidualState: | |
| def __init__(self, running: torch.Tensor): | |
| self.running = running | |
| self.snapshots: List[torch.Tensor] = [] | |
| self.steps = 0 | |
| def advance(self, delta: torch.Tensor, span: int): | |
| self.running = self.running + delta | |
| self.steps += 1 | |
| if self.steps % span == 0: | |
| self.snapshots.append(self.running) | |
| def entries(self) -> List[torch.Tensor]: | |
| return self.snapshots + [self.running] | |
| class TransformerBlock(nn.Module): | |
| def __init__(self, cfg: ModelConfig, rope: Rotary): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.attn_norm = RMSNorm(cfg.d_model, cfg.norm_eps) | |
| self.attn = Attention(cfg, rope) | |
| self.ffn_norm = RMSNorm(cfg.d_model, cfg.norm_eps) | |
| self.ffn = SwiGLU(cfg) | |
| self.attn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None | |
| self.ffn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None | |
| self.sublayers = cfg.layers_per_block * 2 | |
| def forward(self, state: ResidualState) -> ResidualState: | |
| h = self.attn_gate(state.entries()) if self.attn_gate else state.running | |
| delta = self.attn(self.attn_norm(h)) | |
| state.advance(delta, self.sublayers) | |
| h = self.ffn_gate(state.entries()) if self.ffn_gate else state.running | |
| delta = self.ffn(self.ffn_norm(h)) | |
| state.advance(delta, self.sublayers) | |
| return state | |
| class LMHead(nn.Module): | |
| def __init__(self, cfg: ModelConfig, embedding: nn.Embedding): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.embedding = [embedding] | |
| self.logit_scale = cfg.d_model ** -0.5 | |
| def forward(self, hidden: torch.Tensor) -> torch.Tensor: | |
| scaled = hidden * self.logit_scale | |
| weight = self.embedding[0].weight.t() | |
| return scaled @ weight | |
| class CodeModel(nn.Module): | |
| def __init__(self, cfg: ModelConfig): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.embedding = nn.Embedding(cfg.vocab, cfg.d_model) | |
| self.rope = Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta) | |
| self.blocks = nn.ModuleList([TransformerBlock(cfg, self.rope) for _ in range(cfg.n_layers)]) | |
| self.final_norm = RMSNorm(cfg.d_model, cfg.norm_eps) | |
| self.head = LMHead(cfg, self.embedding) | |
| def forward(self, tokens: torch.Tensor) -> torch.Tensor: | |
| h = self.embedding(tokens) | |
| state = ResidualState(running=h) | |
| for block in self.blocks: | |
| block(state) | |
| normed = self.final_norm(state.running) | |
| return self.head(normed) | |
| # ===================================================================== | |
| # 2. Weights & Model Loading | |
| # ===================================================================== | |
| def remap_hf_safetensors(hf_dict: dict) -> dict: | |
| """Remap HuggingFace safetensors keys to native CodeModel state_dict.""" | |
| native = {} | |
| for k, v in hf_dict.items(): | |
| if k == "model.embed_tokens.weight": | |
| native["embedding.weight"] = v | |
| elif k == "model.norm.weight": | |
| native["final_norm.weight"] = v | |
| elif k.startswith("model.layers."): | |
| parts = k.split(".") | |
| layer_idx = parts[2] | |
| sub = ".".join(parts[3:]) | |
| prefix = f"blocks.{layer_idx}" | |
| if sub == "input_layernorm.weight": | |
| native[f"{prefix}.attn_norm.weight"] = v | |
| elif sub == "post_attention_layernorm.weight": | |
| native[f"{prefix}.ffn_norm.weight"] = v | |
| elif sub == "self_attn.q_proj.weight": | |
| native[f"{prefix}.attn.wq.weight"] = v | |
| elif sub == "self_attn.k_proj.weight": | |
| native[f"{prefix}.attn.wk.weight"] = v | |
| elif sub == "self_attn.v_proj.weight": | |
| native[f"{prefix}.attn.wv.weight"] = v | |
| elif sub == "self_attn.o_proj.weight": | |
| native[f"{prefix}.attn.wo.weight"] = v | |
| elif sub == "self_attn.q_norm.weight": | |
| native[f"{prefix}.attn.q_norm.weight"] = v | |
| elif sub == "self_attn.k_norm.weight": | |
| native[f"{prefix}.attn.k_norm.weight"] = v | |
| elif sub == "mlp.gate_proj.weight": | |
| native[f"{prefix}.ffn.w_gate"] = v | |
| elif sub == "mlp.up_proj.weight": | |
| native[f"{prefix}.ffn.w_up"] = v | |
| elif sub == "mlp.down_proj.weight": | |
| native[f"{prefix}.ffn.w_down"] = v | |
| elif sub in ("w_gate", "w_up", "w_down"): | |
| native[f"{prefix}.ffn.{sub}"] = v | |
| else: | |
| native[f"{prefix}.{sub}"] = v | |
| elif k != "lm_head.weight": | |
| native[k] = v | |
| return native | |
| def load_model(model_dir_or_repo: str = ".", device: Optional[str] = None): | |
| """Load Mimans M1 CodeModel and Tokenizer from local directory or HuggingFace repo.""" | |
| if device is None: | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model_path = Path(model_dir_or_repo) | |
| safetensors_file = model_path / "model.safetensors" | |
| tokenizer_file = model_path / "tokenizer.json" | |
| # If not found locally, attempt to download from Hugging Face Hub | |
| if not safetensors_file.exists() or not tokenizer_file.exists(): | |
| try: | |
| from huggingface_hub import hf_hub_download | |
| repo_id = model_dir_or_repo if "/" in model_dir_or_repo else "ankushthakurr09/MimansM1_v1" | |
| print(f"Downloading model files from HuggingFace Hub: {repo_id}...") | |
| safetensors_file = Path(hf_hub_download(repo_id=repo_id, filename="model.safetensors")) | |
| tokenizer_file = Path(hf_hub_download(repo_id=repo_id, filename="tokenizer.json")) | |
| except Exception as e: | |
| raise FileNotFoundError( | |
| f"Could not find model files locally in '{model_dir_or_repo}' and failed to download: {e}" | |
| ) | |
| print(f"Loading weights from: {safetensors_file} ...") | |
| hf_weights = load_file(str(safetensors_file)) | |
| state_dict = remap_hf_safetensors(hf_weights) | |
| cfg = ModelConfig() | |
| model = CodeModel(cfg) | |
| model.load_state_dict(state_dict, strict=True) | |
| model = model.to(device) | |
| if device == "cuda": | |
| model = model.to(torch.bfloat16) | |
| model.eval() | |
| print(f"Loading tokenizer from: {tokenizer_file} ...") | |
| tokenizer = Tokenizer.from_file(str(tokenizer_file)) | |
| print(f"Model loaded successfully on {device} ({sum(p.numel() for p in model.parameters())/1e6:.1f}M params)!") | |
| return model, tokenizer | |
| # ===================================================================== | |
| # 3. Text Generation Engine | |
| # ===================================================================== | |
| def generate( | |
| model: CodeModel, | |
| tokenizer: Tokenizer, | |
| prompt: str, | |
| max_tokens: int = 100, | |
| temperature: float = 0.0, | |
| top_k: int = 50, | |
| repetition_penalty: float = 1.15, | |
| ) -> str: | |
| """Generate tokens from prompt using greedy or sampled decoding.""" | |
| device = next(model.parameters()).device | |
| encoded = tokenizer.encode(prompt) | |
| input_ids = torch.tensor([encoded.ids], dtype=torch.long, device=device) | |
| generated_ids = list(encoded.ids) | |
| with torch.no_grad(): | |
| for _ in range(max_tokens): | |
| curr_input = input_ids[:, -model.cfg.max_seq_len :] | |
| if device.type == "cuda": | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| logits = model(curr_input) | |
| else: | |
| logits = model(curr_input) | |
| next_logits = logits[0, -1, :].clone() | |
| # Greedy decoding (argmax) | |
| if temperature <= 1e-4: | |
| next_token = int(torch.argmax(next_logits).item()) | |
| else: | |
| # Repetition penalty | |
| if repetition_penalty != 1.0 and len(generated_ids) > 0: | |
| for token_id in set(generated_ids): | |
| if next_logits[token_id] < 0: | |
| next_logits[token_id] *= repetition_penalty | |
| else: | |
| next_logits[token_id] /= repetition_penalty | |
| next_logits = next_logits / temperature | |
| if top_k > 0: | |
| v, _ = torch.topk(next_logits, min(top_k, next_logits.size(-1))) | |
| next_logits[next_logits < v[-1]] = -float("Inf") | |
| probs = F.softmax(next_logits, dim=-1) | |
| next_token = torch.multinomial(probs, num_samples=1).item() | |
| generated_ids.append(next_token) | |
| input_ids = torch.cat([input_ids, torch.tensor([[next_token]], device=device)], dim=1) | |
| if next_token == 0: # <|endoftext|> | |
| break | |
| return tokenizer.decode(generated_ids, skip_special_tokens=False) | |
| # ===================================================================== | |
| # 4. Command Line Entry Point | |
| # ===================================================================== | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description="Mimans M1 Native Token Generation") | |
| parser.add_argument("--model-path", default=".", help="Local path or HuggingFace repo ID") | |
| parser.add_argument("--prompt", default="def add(a, b):\n return", help="Input prompt") | |
| parser.add_argument("--max-tokens", type=int, default=100, help="Max tokens to generate") | |
| parser.add_argument("--temp", type=float, default=0.0, help="Temperature (0.0 for greedy)") | |
| parser.add_argument("--top-k", type=int, default=50, help="Top-K sampling") | |
| parser.add_argument("--rep-penalty", type=float, default=1.15, help="Repetition penalty") | |
| parser.add_argument("--device", default=None, help="Device ('cpu' or 'cuda')") | |
| args = parser.parse_args() | |
| model, tokenizer = load_model(args.model_path, device=args.device) | |
| print("\n" + "=" * 60) | |
| print(f"PROMPT:\n{args.prompt}") | |
| print("=" * 60) | |
| output = generate( | |
| model, | |
| tokenizer, | |
| args.prompt, | |
| max_tokens=args.max_tokens, | |
| temperature=args.temp, | |
| top_k=args.top_k, | |
| repetition_penalty=args.rep_penalty, | |
| ) | |
| print("GENERATED OUTPUT:") | |
| print(output) | |
| print("=" * 60) | |