firstlight-124m / model.py
adarsh-007's picture
model.py: use a local model_repo/ only when the weights are there
1f521b9 verified
Raw History Blame Contribute Delete
8.33 kB
"""
Firstlight-124M (GPT-2 architecture, pretrained from scratch on FineWeb-EDU, then SFT on smol-smoltalk), for the Space.
Weights are downloaded from the model repo (REPO_ID) on first use and cached by huggingface_hub:
base/model.safetensors the pretrained model (continues text)
chat/model.safetensors the SFT model (chat assistant)
config.json n_layer, n_head, n_embd, vocab_size, block_size + the special token ids
Same network as GPT2/sft/model.py, with two changes for serving:
- a KV cache: each new token reuses the keys/values of the earlier ones, so one token costs one position's
forward pass instead of the whole context's (needed on a CPU Space)
- no unused `bias` buffer (the causal mask comes from scaled_dot_product_attention)
Chat format (GPT2/sft/chat_format.py):
BOS <|user_start|> question <|user_end|> <|assistant_start|> answer <|assistant_end|> <|user_start|> ...
"""
import json
import os
from dataclasses import dataclass
from pathlib import Path
import tiktoken
import torch
import torch.nn as nn
from torch.nn import functional as F
from safetensors.torch import load_file
REPO_ID = "adarsh-007/firstlight-124m"
# a folder with the files, to skip the download: $GPT2_MODEL_DIR, or ../model_repo when the export has filled it
# (local preview). On the Space neither exists, so the files come from the Hub.
_EXPORT_DIR = Path(__file__).resolve().parent.parent / "model_repo"
LOCAL_DIR = os.environ.get("GPT2_MODEL_DIR") or (str(_EXPORT_DIR) if (_EXPORT_DIR / "chat" / "model.safetensors").is_file() else None)
enc = tiktoken.get_encoding("gpt2")
# GPT-2's tokenizer has ids 0..50256; the model has 50304 rows, so 50257..50303 were free for the chat tokens
BOS = 50256 # <|endoftext|>
USER_START = 50257
USER_END = 50258
ASSISTANT_START = 50259
ASSISTANT_END = 50260
REAL_VOCAB = 50257 # base model: ids >= this were never trained, never sample them
@dataclass
class GPTConfig:
n_layer: int = 12
n_head: int = 12
vocab_size: int = 50304
n_embd: int = 768
block_size: int = 1024
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
self.c_proj = nn.Linear(config.n_embd, config.n_embd)
self.n_head = config.n_head
self.n_embd = config.n_embd
def forward(self, x, cache=None):
# cache: None, or a dict that keeps this layer's keys and values ("k", "v": (B, nh, T_past, hs)) between calls
B, T, C = x.size()
q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
if cache is not None:
if "k" in cache:
k = torch.cat((cache["k"], k), dim=2)
v = torch.cat((cache["v"], v), dim=2)
cache["k"], cache["v"] = k, v
# first call: the whole prompt, causal mask. Later calls: 1 new token that may look at everything before it
y = F.scaled_dot_product_attention(q, k, v, is_causal=(k.size(2) == T))
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.c_proj(y)
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd)
self.gelu = nn.GELU(approximate='tanh')
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd)
def forward(self, x):
return self.c_proj(self.gelu(self.c_fc(x)))
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.ln_1 = nn.LayerNorm(config.n_embd)
self.attn = CausalSelfAttention(config)
self.ln_2 = nn.LayerNorm(config.n_embd)
self.mlp = MLP(config)
def forward(self, x, cache=None):
x = x + self.attn(self.ln_1(x), cache)
x = x + self.mlp(self.ln_2(x))
return x
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte=nn.Embedding(config.vocab_size, config.n_embd),
wpe=nn.Embedding(config.block_size, config.n_embd),
h=nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
ln_f=nn.LayerNorm(config.n_embd),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.transformer.wte.weight = self.lm_head.weight # weight tying, as in training
def forward(self, idx, caches=None, pos_offset=0):
# caches: None, or one dict per layer (see CausalSelfAttention); pos_offset = tokens already in the cache
B, T = idx.size()
pos = torch.arange(pos_offset, pos_offset + T, dtype=torch.long, device=idx.device)
x = self.transformer.wte(idx) + self.transformer.wpe(pos)
for i, block in enumerate(self.transformer.h):
x = block(x, None if caches is None else caches[i])
x = self.transformer.ln_f(x)
return self.lm_head(x) # (B, T, vocab_size)
def model_files(name):
names = ["config.json", f"{name}/model.safetensors"]
if LOCAL_DIR:
return [Path(LOCAL_DIR) / n for n in names]
from huggingface_hub import hf_hub_download
return [Path(hf_hub_download(REPO_ID, n)) for n in names]
def load_model(name, device="cpu"):
"""name: "base" (pretrained) or "chat" (SFT)"""
config_path, weights_path = model_files(name)
config = json.loads(config_path.read_text())
model = GPT(GPTConfig(**{k: config[k] for k in ("n_layer", "n_head", "n_embd", "vocab_size", "block_size")}))
state = load_file(weights_path)
# lm_head.weight isn't stored: it is the same tensor as transformer.wte.weight (weight tying)
missing, unexpected = model.load_state_dict(state, strict=False)
assert missing == ["lm_head.weight"] and not unexpected, (missing, unexpected)
return model.to(device).eval()
# ----------------------------------------------------------------------------- chat format + sampling
def render_chat(messages):
"""[{"role": "user"/"assistant", "content": str}, ...] ending with a user message -> ids ending with
<|assistant_start|>, ready for the model's reply"""
ids = [BOS]
for m in messages:
content = enc.encode_ordinary(m["content"])
if m["role"] == "user":
ids += [USER_START] + content + [USER_END]
else:
ids += [ASSISTANT_START] + content + [ASSISTANT_END]
return ids + [ASSISTANT_START]
@torch.no_grad()
def stream(model, ids, max_new_tokens=300, temperature=0.7, top_k=50, chat=True):
"""Yields the text generated so far after every new token.
chat=True: stops at <|assistant_end|> (or any special token). chat=False (base model): stops at <|endoftext|>."""
device = next(model.parameters()).device
block_size = model.config.block_size
ids = ids[-(block_size - max_new_tokens):] # keep the newest tokens if the prompt is too long
caches = [{} for _ in model.transformer.h]
x = torch.tensor([ids], dtype=torch.long, device=device)
logits = model(x, caches) # the whole prompt once
pos = len(ids)
new = []
for _ in range(max_new_tokens):
logits = logits[:, -1, :].float()
# chat model: only the 4 chat tokens are allowed above 50256. base model: nothing above 50256
logits[:, (ASSISTANT_END + 1) if chat else REAL_VOCAB:] = float('-inf')
if temperature == 0:
next_id = logits.argmax(dim=-1).item()
else:
topk_logits, topk_idx = torch.topk(logits / temperature, top_k, dim=-1)
next_id = topk_idx[0, torch.multinomial(F.softmax(topk_logits, dim=-1), 1)].item()
if next_id >= BOS:
break # <|assistant_end|> (chat) or <|endoftext|> (base): done
new.append(next_id)
yield enc.decode(new)
if pos >= block_size:
break # context full
logits = model(torch.tensor([[next_id]], device=device), caches, pos_offset=pos) # just the new token
pos += 1