Download model.py from adarsh-007/firstlight-124m: direct link, hf CLI and curl.
- Browser
- Download file 8.33 kB
-
https://huggingface.co/adarsh-007/firstlight-124m/resolve/main/model.py
- Command line
-
hf download hf://adarsh-007/firstlight-124m/model.py
-
curl -L -o model.py https://huggingface.co/adarsh-007/firstlight-124m/resolve/main/model.py
8.33 kB
| """ | |
| Firstlight-124M (GPT-2 architecture, pretrained from scratch on FineWeb-EDU, then SFT on smol-smoltalk), for the Space. | |
| Weights are downloaded from the model repo (REPO_ID) on first use and cached by huggingface_hub: | |
| base/model.safetensors the pretrained model (continues text) | |
| chat/model.safetensors the SFT model (chat assistant) | |
| config.json n_layer, n_head, n_embd, vocab_size, block_size + the special token ids | |
| Same network as GPT2/sft/model.py, with two changes for serving: | |
| - a KV cache: each new token reuses the keys/values of the earlier ones, so one token costs one position's | |
| forward pass instead of the whole context's (needed on a CPU Space) | |
| - no unused `bias` buffer (the causal mask comes from scaled_dot_product_attention) | |
| Chat format (GPT2/sft/chat_format.py): | |
| BOS <|user_start|> question <|user_end|> <|assistant_start|> answer <|assistant_end|> <|user_start|> ... | |
| """ | |
| import json | |
| import os | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import tiktoken | |
| import torch | |
| import torch.nn as nn | |
| from torch.nn import functional as F | |
| from safetensors.torch import load_file | |
| REPO_ID = "adarsh-007/firstlight-124m" | |
| # a folder with the files, to skip the download: $GPT2_MODEL_DIR, or ../model_repo when the export has filled it | |
| # (local preview). On the Space neither exists, so the files come from the Hub. | |
| _EXPORT_DIR = Path(__file__).resolve().parent.parent / "model_repo" | |
| LOCAL_DIR = os.environ.get("GPT2_MODEL_DIR") or (str(_EXPORT_DIR) if (_EXPORT_DIR / "chat" / "model.safetensors").is_file() else None) | |
| enc = tiktoken.get_encoding("gpt2") | |
| # GPT-2's tokenizer has ids 0..50256; the model has 50304 rows, so 50257..50303 were free for the chat tokens | |
| BOS = 50256 # <|endoftext|> | |
| USER_START = 50257 | |
| USER_END = 50258 | |
| ASSISTANT_START = 50259 | |
| ASSISTANT_END = 50260 | |
| REAL_VOCAB = 50257 # base model: ids >= this were never trained, never sample them | |
| class GPTConfig: | |
| n_layer: int = 12 | |
| n_head: int = 12 | |
| vocab_size: int = 50304 | |
| n_embd: int = 768 | |
| block_size: int = 1024 | |
| class CausalSelfAttention(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd) | |
| self.c_proj = nn.Linear(config.n_embd, config.n_embd) | |
| self.n_head = config.n_head | |
| self.n_embd = config.n_embd | |
| def forward(self, x, cache=None): | |
| # cache: None, or a dict that keeps this layer's keys and values ("k", "v": (B, nh, T_past, hs)) between calls | |
| B, T, C = x.size() | |
| q, k, v = self.c_attn(x).split(self.n_embd, dim=2) | |
| k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) | |
| q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) | |
| v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) | |
| if cache is not None: | |
| if "k" in cache: | |
| k = torch.cat((cache["k"], k), dim=2) | |
| v = torch.cat((cache["v"], v), dim=2) | |
| cache["k"], cache["v"] = k, v | |
| # first call: the whole prompt, causal mask. Later calls: 1 new token that may look at everything before it | |
| y = F.scaled_dot_product_attention(q, k, v, is_causal=(k.size(2) == T)) | |
| y = y.transpose(1, 2).contiguous().view(B, T, C) | |
| return self.c_proj(y) | |
| class MLP(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd) | |
| self.gelu = nn.GELU(approximate='tanh') | |
| self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd) | |
| def forward(self, x): | |
| return self.c_proj(self.gelu(self.c_fc(x))) | |
| class Block(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.ln_1 = nn.LayerNorm(config.n_embd) | |
| self.attn = CausalSelfAttention(config) | |
| self.ln_2 = nn.LayerNorm(config.n_embd) | |
| self.mlp = MLP(config) | |
| def forward(self, x, cache=None): | |
| x = x + self.attn(self.ln_1(x), cache) | |
| x = x + self.mlp(self.ln_2(x)) | |
| return x | |
| class GPT(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.config = config | |
| self.transformer = nn.ModuleDict(dict( | |
| wte=nn.Embedding(config.vocab_size, config.n_embd), | |
| wpe=nn.Embedding(config.block_size, config.n_embd), | |
| h=nn.ModuleList([Block(config) for _ in range(config.n_layer)]), | |
| ln_f=nn.LayerNorm(config.n_embd), | |
| )) | |
| self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) | |
| self.transformer.wte.weight = self.lm_head.weight # weight tying, as in training | |
| def forward(self, idx, caches=None, pos_offset=0): | |
| # caches: None, or one dict per layer (see CausalSelfAttention); pos_offset = tokens already in the cache | |
| B, T = idx.size() | |
| pos = torch.arange(pos_offset, pos_offset + T, dtype=torch.long, device=idx.device) | |
| x = self.transformer.wte(idx) + self.transformer.wpe(pos) | |
| for i, block in enumerate(self.transformer.h): | |
| x = block(x, None if caches is None else caches[i]) | |
| x = self.transformer.ln_f(x) | |
| return self.lm_head(x) # (B, T, vocab_size) | |
| def model_files(name): | |
| names = ["config.json", f"{name}/model.safetensors"] | |
| if LOCAL_DIR: | |
| return [Path(LOCAL_DIR) / n for n in names] | |
| from huggingface_hub import hf_hub_download | |
| return [Path(hf_hub_download(REPO_ID, n)) for n in names] | |
| def load_model(name, device="cpu"): | |
| """name: "base" (pretrained) or "chat" (SFT)""" | |
| config_path, weights_path = model_files(name) | |
| config = json.loads(config_path.read_text()) | |
| model = GPT(GPTConfig(**{k: config[k] for k in ("n_layer", "n_head", "n_embd", "vocab_size", "block_size")})) | |
| state = load_file(weights_path) | |
| # lm_head.weight isn't stored: it is the same tensor as transformer.wte.weight (weight tying) | |
| missing, unexpected = model.load_state_dict(state, strict=False) | |
| assert missing == ["lm_head.weight"] and not unexpected, (missing, unexpected) | |
| return model.to(device).eval() | |
| # ----------------------------------------------------------------------------- chat format + sampling | |
| def render_chat(messages): | |
| """[{"role": "user"/"assistant", "content": str}, ...] ending with a user message -> ids ending with | |
| <|assistant_start|>, ready for the model's reply""" | |
| ids = [BOS] | |
| for m in messages: | |
| content = enc.encode_ordinary(m["content"]) | |
| if m["role"] == "user": | |
| ids += [USER_START] + content + [USER_END] | |
| else: | |
| ids += [ASSISTANT_START] + content + [ASSISTANT_END] | |
| return ids + [ASSISTANT_START] | |
| def stream(model, ids, max_new_tokens=300, temperature=0.7, top_k=50, chat=True): | |
| """Yields the text generated so far after every new token. | |
| chat=True: stops at <|assistant_end|> (or any special token). chat=False (base model): stops at <|endoftext|>.""" | |
| device = next(model.parameters()).device | |
| block_size = model.config.block_size | |
| ids = ids[-(block_size - max_new_tokens):] # keep the newest tokens if the prompt is too long | |
| caches = [{} for _ in model.transformer.h] | |
| x = torch.tensor([ids], dtype=torch.long, device=device) | |
| logits = model(x, caches) # the whole prompt once | |
| pos = len(ids) | |
| new = [] | |
| for _ in range(max_new_tokens): | |
| logits = logits[:, -1, :].float() | |
| # chat model: only the 4 chat tokens are allowed above 50256. base model: nothing above 50256 | |
| logits[:, (ASSISTANT_END + 1) if chat else REAL_VOCAB:] = float('-inf') | |
| if temperature == 0: | |
| next_id = logits.argmax(dim=-1).item() | |
| else: | |
| topk_logits, topk_idx = torch.topk(logits / temperature, top_k, dim=-1) | |
| next_id = topk_idx[0, torch.multinomial(F.softmax(topk_logits, dim=-1), 1)].item() | |
| if next_id >= BOS: | |
| break # <|assistant_end|> (chat) or <|endoftext|> (base): done | |
| new.append(next_id) | |
| yield enc.decode(new) | |
| if pos >= block_size: | |
| break # context full | |
| logits = model(torch.tensor([[next_id]], device=device), caches, pos_offset=pos) # just the new token | |
| pos += 1 | |