File size: 1,941 Bytes
b9ee6b4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
"""Continue a text with the model: python generate.py "The problem of personal identity"

Needs torch, safetensors and tokenizers. Sampling as on gustavecortal.com/philosophy-lm/: presence penalty 2.0,

then temperature 0.7, top-k 20 and top-p 0.8. A text is continued as a new document (the end-of-text token first),

as in training."""
import argparse
import json

import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer

from decode import Decoder
from model import Transformer

p = argparse.ArgumentParser()
p.add_argument("text")
p.add_argument("--tokens", type=int, default=256)
p.add_argument("--temperature", type=float, default=0.7)
p.add_argument("--top-k", type=int, default=20)
p.add_argument("--top-p", type=float, default=0.8)
p.add_argument("--presence-penalty", type=float, default=2.0)
p.add_argument("--seed", type=int, default=0)
args = p.parse_args()
config = json.load(open("config.json"))
model = Transformer(config["model"], config["vocab_size"])
model.load_state_dict(load_file("model.safetensors"))
model.eval()
tokenizer = Tokenizer.from_file("tokenizer.json")
eos = config["eos_id"]
torch.manual_seed(args.seed)
decoder, seen, out = Decoder(model), set(), []
with torch.no_grad():
    logits = decoder.prefill([eos] + tokenizer.encode(args.text, add_special_tokens=False).ids)
    for _ in range(args.tokens):
        logits = logits.float().clone()
        logits[list(seen)] -= args.presence_penalty
        top = torch.topk(logits / args.temperature, args.top_k)
        probs = torch.softmax(top.values, -1)
        keep = (torch.cumsum(probs, -1) - probs) < args.top_p
        token = top.indices[keep][torch.multinomial(probs[keep] / probs[keep].sum(), 1)].item()
        if token == eos:
            break
        seen.add(token)
        out.append(token)
        logits = decoder.step(token)
print(args.text + tokenizer.decode(out))