File size: 8,333 Bytes
1f521b9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
"""
Firstlight-124M (GPT-2 architecture, pretrained from scratch on FineWeb-EDU, then SFT on smol-smoltalk), for the Space.

Weights are downloaded from the model repo (REPO_ID) on first use and cached by huggingface_hub:
    base/model.safetensors   the pretrained model (continues text)
    chat/model.safetensors   the SFT model (chat assistant)
    config.json              n_layer, n_head, n_embd, vocab_size, block_size + the special token ids

Same network as GPT2/sft/model.py, with two changes for serving:
- a KV cache: each new token reuses the keys/values of the earlier ones, so one token costs one position's
  forward pass instead of the whole context's (needed on a CPU Space)
- no unused `bias` buffer (the causal mask comes from scaled_dot_product_attention)

Chat format (GPT2/sft/chat_format.py):
    BOS <|user_start|> question <|user_end|> <|assistant_start|> answer <|assistant_end|> <|user_start|> ...
"""
import json
import os
from dataclasses import dataclass
from pathlib import Path

import tiktoken
import torch
import torch.nn as nn
from torch.nn import functional as F
from safetensors.torch import load_file

REPO_ID = "adarsh-007/firstlight-124m"
# a folder with the files, to skip the download: $GPT2_MODEL_DIR, or ../model_repo when the export has filled it
# (local preview). On the Space neither exists, so the files come from the Hub.
_EXPORT_DIR = Path(__file__).resolve().parent.parent / "model_repo"
LOCAL_DIR = os.environ.get("GPT2_MODEL_DIR") or (str(_EXPORT_DIR) if (_EXPORT_DIR / "chat" / "model.safetensors").is_file() else None)

enc = tiktoken.get_encoding("gpt2")

# GPT-2's tokenizer has ids 0..50256; the model has 50304 rows, so 50257..50303 were free for the chat tokens
BOS = 50256             # <|endoftext|>
USER_START = 50257
USER_END = 50258
ASSISTANT_START = 50259
ASSISTANT_END = 50260
REAL_VOCAB = 50257      # base model: ids >= this were never trained, never sample them


@dataclass
class GPTConfig:
    n_layer: int = 12
    n_head: int = 12
    vocab_size: int = 50304
    n_embd: int = 768
    block_size: int = 1024


class CausalSelfAttention(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
        self.n_head = config.n_head
        self.n_embd = config.n_embd

    def forward(self, x, cache=None):
        # cache: None, or a dict that keeps this layer's keys and values ("k", "v": (B, nh, T_past, hs)) between calls
        B, T, C = x.size()
        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        if cache is not None:
            if "k" in cache:
                k = torch.cat((cache["k"], k), dim=2)
                v = torch.cat((cache["v"], v), dim=2)
            cache["k"], cache["v"] = k, v
        # first call: the whole prompt, causal mask. Later calls: 1 new token that may look at everything before it
        y = F.scaled_dot_product_attention(q, k, v, is_causal=(k.size(2) == T))
        y = y.transpose(1, 2).contiguous().view(B, T, C)
        return self.c_proj(y)


class MLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd)
        self.gelu = nn.GELU(approximate='tanh')
        self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd)

    def forward(self, x):
        return self.c_proj(self.gelu(self.c_fc(x)))


class Block(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.ln_1 = nn.LayerNorm(config.n_embd)
        self.attn = CausalSelfAttention(config)
        self.ln_2 = nn.LayerNorm(config.n_embd)
        self.mlp = MLP(config)

    def forward(self, x, cache=None):
        x = x + self.attn(self.ln_1(x), cache)
        x = x + self.mlp(self.ln_2(x))
        return x


class GPT(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.transformer = nn.ModuleDict(dict(
            wte=nn.Embedding(config.vocab_size, config.n_embd),
            wpe=nn.Embedding(config.block_size, config.n_embd),
            h=nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
            ln_f=nn.LayerNorm(config.n_embd),
        ))
        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
        self.transformer.wte.weight = self.lm_head.weight  # weight tying, as in training

    def forward(self, idx, caches=None, pos_offset=0):
        # caches: None, or one dict per layer (see CausalSelfAttention); pos_offset = tokens already in the cache
        B, T = idx.size()
        pos = torch.arange(pos_offset, pos_offset + T, dtype=torch.long, device=idx.device)
        x = self.transformer.wte(idx) + self.transformer.wpe(pos)
        for i, block in enumerate(self.transformer.h):
            x = block(x, None if caches is None else caches[i])
        x = self.transformer.ln_f(x)
        return self.lm_head(x)  # (B, T, vocab_size)


def model_files(name):
    names = ["config.json", f"{name}/model.safetensors"]
    if LOCAL_DIR:
        return [Path(LOCAL_DIR) / n for n in names]
    from huggingface_hub import hf_hub_download
    return [Path(hf_hub_download(REPO_ID, n)) for n in names]


def load_model(name, device="cpu"):
    """name: "base" (pretrained) or "chat" (SFT)"""
    config_path, weights_path = model_files(name)
    config = json.loads(config_path.read_text())
    model = GPT(GPTConfig(**{k: config[k] for k in ("n_layer", "n_head", "n_embd", "vocab_size", "block_size")}))
    state = load_file(weights_path)
    # lm_head.weight isn't stored: it is the same tensor as transformer.wte.weight (weight tying)
    missing, unexpected = model.load_state_dict(state, strict=False)
    assert missing == ["lm_head.weight"] and not unexpected, (missing, unexpected)
    return model.to(device).eval()


# ----------------------------------------------------------------------------- chat format + sampling

def render_chat(messages):
    """[{"role": "user"/"assistant", "content": str}, ...] ending with a user message -> ids ending with
    <|assistant_start|>, ready for the model's reply"""
    ids = [BOS]
    for m in messages:
        content = enc.encode_ordinary(m["content"])
        if m["role"] == "user":
            ids += [USER_START] + content + [USER_END]
        else:
            ids += [ASSISTANT_START] + content + [ASSISTANT_END]
    return ids + [ASSISTANT_START]


@torch.no_grad()
def stream(model, ids, max_new_tokens=300, temperature=0.7, top_k=50, chat=True):
    """Yields the text generated so far after every new token.
    chat=True: stops at <|assistant_end|> (or any special token). chat=False (base model): stops at <|endoftext|>."""
    device = next(model.parameters()).device
    block_size = model.config.block_size
    ids = ids[-(block_size - max_new_tokens):]   # keep the newest tokens if the prompt is too long
    caches = [{} for _ in model.transformer.h]
    x = torch.tensor([ids], dtype=torch.long, device=device)
    logits = model(x, caches)                     # the whole prompt once
    pos = len(ids)
    new = []
    for _ in range(max_new_tokens):
        logits = logits[:, -1, :].float()
        # chat model: only the 4 chat tokens are allowed above 50256. base model: nothing above 50256
        logits[:, (ASSISTANT_END + 1) if chat else REAL_VOCAB:] = float('-inf')
        if temperature == 0:
            next_id = logits.argmax(dim=-1).item()
        else:
            topk_logits, topk_idx = torch.topk(logits / temperature, top_k, dim=-1)
            next_id = topk_idx[0, torch.multinomial(F.softmax(topk_logits, dim=-1), 1)].item()
        if next_id >= BOS:
            break                                 # <|assistant_end|> (chat) or <|endoftext|> (base): done
        new.append(next_id)
        yield enc.decode(new)
        if pos >= block_size:
            break                                 # context full
        logits = model(torch.tensor([[next_id]], device=device), caches, pos_offset=pos)   # just the new token
        pos += 1