Text Generation
Transformers
Safetensors
PyTorch
byte-level
tiny-lm
micro-model
research
from-scratch
rope
swiglu
muon-optimizer
custom-code
notokenlm_microgen
Instructions to use VeroLM/NoTokenLM-MicroGen with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use VeroLM/NoTokenLM-MicroGen with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="VeroLM/NoTokenLM-MicroGen")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("VeroLM/NoTokenLM-MicroGen", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use VeroLM/NoTokenLM-MicroGen with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "VeroLM/NoTokenLM-MicroGen" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "VeroLM/NoTokenLM-MicroGen", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/VeroLM/NoTokenLM-MicroGen
- SGLang
How to use VeroLM/NoTokenLM-MicroGen with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "VeroLM/NoTokenLM-MicroGen" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "VeroLM/NoTokenLM-MicroGen", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "VeroLM/NoTokenLM-MicroGen" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "VeroLM/NoTokenLM-MicroGen", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use VeroLM/NoTokenLM-MicroGen with Docker Model Runner:
docker model run hf.co/VeroLM/NoTokenLM-MicroGen
Download modeling_notokengen.py from VeroLM/NoTokenLM-MicroGen: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/VeroLM/NoTokenLM-MicroGen/resolve/main/modeling_notokengen.py
- Command line
-
hf download hf://VeroLM/NoTokenLM-MicroGen/modeling_notokengen.py
-
curl -L -o modeling_notokengen.py https://huggingface.co/VeroLM/NoTokenLM-MicroGen/resolve/main/modeling_notokengen.py
11.4 kB
| """ | |
| modeling_notokengen.py | |
| Shared HuggingFace-compatible model class for every version in the | |
| NoTokenLM-MicroGen series (2.5 through 3.7). Wraps the original from-scratch | |
| architectures (byte-level, RoPE, SwiGLU, tied embeddings) in a | |
| transformers.PreTrainedModel so every checkpoint can be loaded the standard | |
| way: | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| model = AutoModelForCausalLM.from_pretrained( | |
| "omurberaisik/NoTokenLM-MicroGen", | |
| subfolder="3.6", | |
| trust_remote_code=True, | |
| ) | |
| There is no tokenizer - input/output is raw UTF-8 bytes (vocab_size=256). | |
| Use the `generate_bytes` helper on the model, or feed byte ids directly. | |
| """ | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from transformers import PreTrainedModel | |
| from transformers.modeling_outputs import CausalLMOutput | |
| from .configuration_notokengen import NoTokenGenConfig | |
| # ============================================================ | |
| # Building blocks | |
| # ============================================================ | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim, eps=1e-6): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| self.eps = eps | |
| def forward(self, x): | |
| norm = x.norm(dim=-1, keepdim=True) * (x.shape[-1] ** -0.5) | |
| return x / (norm + self.eps) * self.weight | |
| # ---------- "tiny" family (used only by 2.5) ---------- | |
| class TinyAttention(nn.Module): | |
| def __init__(self, d_model): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.qkv = nn.Linear(d_model, d_model * 3, bias=False) | |
| self.proj = nn.Linear(d_model, d_model, bias=False) | |
| def forward(self, x): | |
| B, T, C = x.shape | |
| q, k, v = self.qkv(x).chunk(3, dim=-1) | |
| att = (q @ k.transpose(-2, -1)) / math.sqrt(C) | |
| mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool() | |
| att = att.masked_fill(mask, float("-inf")) | |
| att = att.softmax(dim=-1) | |
| return self.proj(att @ v) | |
| class TinyMLP(nn.Module): | |
| def __init__(self, d_model, mult=1): | |
| super().__init__() | |
| hidden = d_model * mult | |
| self.fc1 = nn.Linear(d_model, hidden, bias=False) | |
| self.fc2 = nn.Linear(hidden, d_model, bias=False) | |
| def forward(self, x): | |
| return self.fc2(torch.relu(self.fc1(x))) | |
| class TinyBackbone(nn.Module): | |
| """Used only by 2.5: single layer, single head, learned positional embedding.""" | |
| def __init__(self, vocab_size, d_model, max_len): | |
| super().__init__() | |
| self.tok_emb = nn.Embedding(vocab_size, d_model) | |
| self.pos_emb = nn.Embedding(max_len, d_model) | |
| self.norm1 = RMSNorm(d_model) | |
| self.attn = TinyAttention(d_model) | |
| self.norm2 = RMSNorm(d_model) | |
| self.mlp = TinyMLP(d_model, mult=1) | |
| self.norm_out = RMSNorm(d_model) | |
| self.max_len = max_len | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| pos = torch.arange(T, device=idx.device) | |
| x = self.tok_emb(idx) + self.pos_emb(pos) | |
| x = x + self.attn(self.norm1(x)) | |
| x = x + self.mlp(self.norm2(x)) | |
| x = self.norm_out(x) | |
| return x @ self.tok_emb.weight.T | |
| # ---------- RoPE+SwiGLU family (used by 2.6 through 3.7) ---------- | |
| def precompute_rope(dim, max_len, base=10000.0): | |
| inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim)) | |
| t = torch.arange(max_len).float() | |
| freqs = torch.outer(t, inv_freq) | |
| return torch.cos(freqs), torch.sin(freqs) | |
| def apply_rope(x, cos, sin): | |
| x1, x2 = x[..., ::2], x[..., 1::2] | |
| cos = cos[None, :, None, :] | |
| sin = sin[None, :, None, :] | |
| rx1 = x1 * cos - x2 * sin | |
| rx2 = x1 * sin + x2 * cos | |
| return torch.stack([rx1, rx2], dim=-1).flatten(-2) | |
| class MultiHeadAttentionRoPE(nn.Module): | |
| """Manual (non-SDPA) attention - used by 2.6 only.""" | |
| def __init__(self, d_model, n_heads): | |
| super().__init__() | |
| assert d_model % n_heads == 0 | |
| self.n_heads = n_heads | |
| self.head_dim = d_model // n_heads | |
| self.qkv = nn.Linear(d_model, d_model * 3, bias=False) | |
| self.proj = nn.Linear(d_model, d_model, bias=False) | |
| def forward(self, x, cos, sin): | |
| B, T, C = x.shape | |
| qkv = self.qkv(x).view(B, T, 3, self.n_heads, self.head_dim) | |
| q, k, v = qkv.unbind(2) | |
| q = apply_rope(q, cos, sin) | |
| k = apply_rope(k, cos, sin) | |
| q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) | |
| att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim) | |
| mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool() | |
| att = att.masked_fill(mask, float("-inf")) | |
| att = att.softmax(dim=-1) | |
| out = (att @ v).transpose(1, 2).reshape(B, T, C) | |
| return self.proj(out) | |
| class FlashAttentionRoPE(nn.Module): | |
| """PyTorch native SDPA - used from 2.7 onward.""" | |
| def __init__(self, d_model, n_heads): | |
| super().__init__() | |
| assert d_model % n_heads == 0 | |
| self.n_heads = n_heads | |
| self.head_dim = d_model // n_heads | |
| self.qkv = nn.Linear(d_model, d_model * 3, bias=False) | |
| self.proj = nn.Linear(d_model, d_model, bias=False) | |
| def forward(self, x, cos, sin): | |
| B, T, C = x.shape | |
| qkv = self.qkv(x).view(B, T, 3, self.n_heads, self.head_dim) | |
| q, k, v = qkv.unbind(2) | |
| q = apply_rope(q, cos, sin) | |
| k = apply_rope(k, cos, sin) | |
| q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) | |
| out = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| out = out.transpose(1, 2).reshape(B, T, C) | |
| return self.proj(out) | |
| class SwiGLU(nn.Module): | |
| def __init__(self, d_model, hidden): | |
| super().__init__() | |
| self.gate = nn.Linear(d_model, hidden, bias=False) | |
| self.up = nn.Linear(d_model, hidden, bias=False) | |
| self.down = nn.Linear(hidden, d_model, bias=False) | |
| def forward(self, x): | |
| return self.down(F.silu(self.gate(x)) * self.up(x)) | |
| class Block(nn.Module): | |
| def __init__(self, d_model, n_heads, mlp_hidden, attn_cls): | |
| super().__init__() | |
| self.norm1 = RMSNorm(d_model) | |
| self.attn = attn_cls(d_model, n_heads) | |
| self.norm2 = RMSNorm(d_model) | |
| self.mlp = SwiGLU(d_model, mlp_hidden) | |
| def forward(self, x, cos, sin): | |
| x = x + self.attn(self.norm1(x), cos, sin) | |
| x = x + self.mlp(self.norm2(x)) | |
| return x | |
| ATTN_CLASSES = { | |
| "manual": MultiHeadAttentionRoPE, | |
| "flash": FlashAttentionRoPE, | |
| } | |
| class RoPEBackbone(nn.Module): | |
| """Shared skeleton for 2.6, 2.7, 3.5, 3.6, 3.7 - only dims and attn differ.""" | |
| def __init__(self, vocab_size, d_model, n_heads, n_layers, mlp_hidden, max_len, attn): | |
| super().__init__() | |
| attn_cls = ATTN_CLASSES[attn] | |
| self.tok_emb = nn.Embedding(vocab_size, d_model) | |
| nn.init.normal_(self.tok_emb.weight, mean=0.0, std=0.02) | |
| self.blocks = nn.ModuleList( | |
| [Block(d_model, n_heads, mlp_hidden, attn_cls) for _ in range(n_layers)] | |
| ) | |
| self.norm_out = RMSNorm(d_model) | |
| head_dim = d_model // n_heads | |
| cos, sin = precompute_rope(head_dim, max_len) | |
| self.register_buffer("rope_cos", cos, persistent=False) | |
| self.register_buffer("rope_sin", sin, persistent=False) | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| x = self.tok_emb(idx) | |
| cos, sin = self.rope_cos[:T], self.rope_sin[:T] | |
| # Defensive guard: on some environments the non-persistent RoPE buffers | |
| # have been observed to end up containing NaN despite being computed | |
| # cleanly at __init__ time (likely due to post_init()/from_pretrained() | |
| # touching buffers unexpectedly). If that happens, recompute them fresh | |
| # on the fly rather than silently propagating NaN through every layer. | |
| if torch.isnan(cos).any() or torch.isnan(sin).any(): | |
| head_dim = self.rope_cos.shape[-1] * 2 | |
| max_len = self.rope_cos.shape[0] | |
| fresh_cos, fresh_sin = precompute_rope(head_dim, max_len) | |
| fresh_cos, fresh_sin = fresh_cos.to(idx.device), fresh_sin.to(idx.device) | |
| self.rope_cos.copy_(fresh_cos) | |
| self.rope_sin.copy_(fresh_sin) | |
| cos, sin = self.rope_cos[:T], self.rope_sin[:T] | |
| for block in self.blocks: | |
| x = block(x, cos, sin) | |
| x = self.norm_out(x) | |
| return x @ self.tok_emb.weight.T | |
| # ============================================================ | |
| # HuggingFace-compatible wrapper | |
| # ============================================================ | |
| class NoTokenGenForCausalLM(PreTrainedModel): | |
| config_class = NoTokenGenConfig | |
| def _init_weights(self, module): | |
| # No-op on purpose: every checkpoint's real weights are loaded via | |
| # from_pretrained's own state_dict loading, which happens AFTER __init__ | |
| # runs. HF's PreTrainedModel.post_init() calls self.apply(self._init_weights) | |
| # if this method is left undefined, using its own default initialization | |
| # scheme - on some environments this has been observed to also touch | |
| # registered buffers (like our precomputed RoPE cos/sin tables), corrupting | |
| # them into NaN before the real weights are ever loaded. Defining this as | |
| # a no-op guarantees post_init() never touches any parameter or buffer. | |
| pass | |
| def __init__(self, config: NoTokenGenConfig): | |
| super().__init__(config) | |
| if config.kind == "tiny": | |
| self.backbone = TinyBackbone( | |
| vocab_size=config.vocab_size, | |
| d_model=config.d_model, | |
| max_len=config.max_len, | |
| ) | |
| else: | |
| self.backbone = RoPEBackbone( | |
| vocab_size=config.vocab_size, | |
| d_model=config.d_model, | |
| n_heads=config.n_heads, | |
| n_layers=config.n_layers, | |
| mlp_hidden=config.mlp_hidden, | |
| max_len=config.max_len, | |
| attn=config.attn, | |
| ) | |
| self.post_init() | |
| def forward(self, input_ids, labels=None, **kwargs): | |
| logits = self.backbone(input_ids) | |
| loss = None | |
| if labels is not None: | |
| loss = F.cross_entropy( | |
| logits.view(-1, logits.size(-1)), labels.view(-1) | |
| ) | |
| return CausalLMOutput(loss=loss, logits=logits) | |
| def generate_bytes(self, prompt: str, n_new_bytes: int = 200, temperature: float = 0.5) -> str: | |
| """Convenience helper: raw-byte generation without a tokenizer. | |
| This series has no tokenizer - text goes in and comes out as UTF-8 bytes. | |
| `generate_bytes` is the simplest way to use these models; `forward()` is | |
| also available directly for anyone who wants to feed byte ids themselves. | |
| """ | |
| self.eval() | |
| device = next(self.parameters()).device | |
| max_len = self.config.max_len | |
| ids = list(prompt.encode("utf-8")) | |
| for _ in range(n_new_bytes): | |
| context = ids[-max_len:] | |
| x = torch.tensor([context], dtype=torch.long, device=device) | |
| logits = self.forward(x).logits[0, -1] | |
| probs = torch.softmax(logits / temperature, dim=-1) | |
| next_id = torch.multinomial(probs, 1).item() | |
| ids.append(next_id) | |
| return bytes(ids).decode("utf-8", errors="replace") | |