File size: 8,333 Bytes
1f521b9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 | """
Firstlight-124M (GPT-2 architecture, pretrained from scratch on FineWeb-EDU, then SFT on smol-smoltalk), for the Space.
Weights are downloaded from the model repo (REPO_ID) on first use and cached by huggingface_hub:
base/model.safetensors the pretrained model (continues text)
chat/model.safetensors the SFT model (chat assistant)
config.json n_layer, n_head, n_embd, vocab_size, block_size + the special token ids
Same network as GPT2/sft/model.py, with two changes for serving:
- a KV cache: each new token reuses the keys/values of the earlier ones, so one token costs one position's
forward pass instead of the whole context's (needed on a CPU Space)
- no unused `bias` buffer (the causal mask comes from scaled_dot_product_attention)
Chat format (GPT2/sft/chat_format.py):
BOS <|user_start|> question <|user_end|> <|assistant_start|> answer <|assistant_end|> <|user_start|> ...
"""
import json
import os
from dataclasses import dataclass
from pathlib import Path
import tiktoken
import torch
import torch.nn as nn
from torch.nn import functional as F
from safetensors.torch import load_file
REPO_ID = "adarsh-007/firstlight-124m"
# a folder with the files, to skip the download: $GPT2_MODEL_DIR, or ../model_repo when the export has filled it
# (local preview). On the Space neither exists, so the files come from the Hub.
_EXPORT_DIR = Path(__file__).resolve().parent.parent / "model_repo"
LOCAL_DIR = os.environ.get("GPT2_MODEL_DIR") or (str(_EXPORT_DIR) if (_EXPORT_DIR / "chat" / "model.safetensors").is_file() else None)
enc = tiktoken.get_encoding("gpt2")
# GPT-2's tokenizer has ids 0..50256; the model has 50304 rows, so 50257..50303 were free for the chat tokens
BOS = 50256 # <|endoftext|>
USER_START = 50257
USER_END = 50258
ASSISTANT_START = 50259
ASSISTANT_END = 50260
REAL_VOCAB = 50257 # base model: ids >= this were never trained, never sample them
@dataclass
class GPTConfig:
n_layer: int = 12
n_head: int = 12
vocab_size: int = 50304
n_embd: int = 768
block_size: int = 1024
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
self.c_proj = nn.Linear(config.n_embd, config.n_embd)
self.n_head = config.n_head
self.n_embd = config.n_embd
def forward(self, x, cache=None):
# cache: None, or a dict that keeps this layer's keys and values ("k", "v": (B, nh, T_past, hs)) between calls
B, T, C = x.size()
q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
if cache is not None:
if "k" in cache:
k = torch.cat((cache["k"], k), dim=2)
v = torch.cat((cache["v"], v), dim=2)
cache["k"], cache["v"] = k, v
# first call: the whole prompt, causal mask. Later calls: 1 new token that may look at everything before it
y = F.scaled_dot_product_attention(q, k, v, is_causal=(k.size(2) == T))
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.c_proj(y)
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd)
self.gelu = nn.GELU(approximate='tanh')
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd)
def forward(self, x):
return self.c_proj(self.gelu(self.c_fc(x)))
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.ln_1 = nn.LayerNorm(config.n_embd)
self.attn = CausalSelfAttention(config)
self.ln_2 = nn.LayerNorm(config.n_embd)
self.mlp = MLP(config)
def forward(self, x, cache=None):
x = x + self.attn(self.ln_1(x), cache)
x = x + self.mlp(self.ln_2(x))
return x
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte=nn.Embedding(config.vocab_size, config.n_embd),
wpe=nn.Embedding(config.block_size, config.n_embd),
h=nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
ln_f=nn.LayerNorm(config.n_embd),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.transformer.wte.weight = self.lm_head.weight # weight tying, as in training
def forward(self, idx, caches=None, pos_offset=0):
# caches: None, or one dict per layer (see CausalSelfAttention); pos_offset = tokens already in the cache
B, T = idx.size()
pos = torch.arange(pos_offset, pos_offset + T, dtype=torch.long, device=idx.device)
x = self.transformer.wte(idx) + self.transformer.wpe(pos)
for i, block in enumerate(self.transformer.h):
x = block(x, None if caches is None else caches[i])
x = self.transformer.ln_f(x)
return self.lm_head(x) # (B, T, vocab_size)
def model_files(name):
names = ["config.json", f"{name}/model.safetensors"]
if LOCAL_DIR:
return [Path(LOCAL_DIR) / n for n in names]
from huggingface_hub import hf_hub_download
return [Path(hf_hub_download(REPO_ID, n)) for n in names]
def load_model(name, device="cpu"):
"""name: "base" (pretrained) or "chat" (SFT)"""
config_path, weights_path = model_files(name)
config = json.loads(config_path.read_text())
model = GPT(GPTConfig(**{k: config[k] for k in ("n_layer", "n_head", "n_embd", "vocab_size", "block_size")}))
state = load_file(weights_path)
# lm_head.weight isn't stored: it is the same tensor as transformer.wte.weight (weight tying)
missing, unexpected = model.load_state_dict(state, strict=False)
assert missing == ["lm_head.weight"] and not unexpected, (missing, unexpected)
return model.to(device).eval()
# ----------------------------------------------------------------------------- chat format + sampling
def render_chat(messages):
"""[{"role": "user"/"assistant", "content": str}, ...] ending with a user message -> ids ending with
<|assistant_start|>, ready for the model's reply"""
ids = [BOS]
for m in messages:
content = enc.encode_ordinary(m["content"])
if m["role"] == "user":
ids += [USER_START] + content + [USER_END]
else:
ids += [ASSISTANT_START] + content + [ASSISTANT_END]
return ids + [ASSISTANT_START]
@torch.no_grad()
def stream(model, ids, max_new_tokens=300, temperature=0.7, top_k=50, chat=True):
"""Yields the text generated so far after every new token.
chat=True: stops at <|assistant_end|> (or any special token). chat=False (base model): stops at <|endoftext|>."""
device = next(model.parameters()).device
block_size = model.config.block_size
ids = ids[-(block_size - max_new_tokens):] # keep the newest tokens if the prompt is too long
caches = [{} for _ in model.transformer.h]
x = torch.tensor([ids], dtype=torch.long, device=device)
logits = model(x, caches) # the whole prompt once
pos = len(ids)
new = []
for _ in range(max_new_tokens):
logits = logits[:, -1, :].float()
# chat model: only the 4 chat tokens are allowed above 50256. base model: nothing above 50256
logits[:, (ASSISTANT_END + 1) if chat else REAL_VOCAB:] = float('-inf')
if temperature == 0:
next_id = logits.argmax(dim=-1).item()
else:
topk_logits, topk_idx = torch.topk(logits / temperature, top_k, dim=-1)
next_id = topk_idx[0, torch.multinomial(F.softmax(topk_logits, dim=-1), 1)].item()
if next_id >= BOS:
break # <|assistant_end|> (chat) or <|endoftext|> (base): done
new.append(next_id)
yield enc.decode(new)
if pos >= block_size:
break # context full
logits = model(torch.tensor([[next_id]], device=device), caches, pos_offset=pos) # just the new token
pos += 1
|