File size: 2,260 Bytes
546acd4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""Chat with ChatGBA in the terminal, through transformers.

    pip install torch transformers safetensors
    python chat.py [MODEL] [--temperature 0.7] [--max-bytes 400] [--seed N]

MODEL is the repository's name on the Hub (gszauer/ChatGBA) or a local copy
of it (default: the folder this script is in). An empty line quits; /reset
starts over.

The model sees the conversation as plain bytes:
    <|u|>question<|e|><|a|>reply<|e|><|u|>...
and its reply ends when it writes <|e|>. Its context slides: past
contextLength bytes, it keeps only the latest ones.
"""
import argparse
from pathlib import Path

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

parser = argparse.ArgumentParser()
parser.add_argument("model", nargs="?", default=str(Path(__file__).resolve().parent))
parser.add_argument("--temperature", type=float, default=0.7)
parser.add_argument("--top-k", type=int, default=40)
parser.add_argument("--max-bytes", type=int, default=400)
parser.add_argument("--seed", type=int)
args = parser.parse_args()

if args.seed is not None:
    torch.manual_seed(args.seed)
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=True).eval()
end = model.config.chat["end"]
print(f"ChatGBA: {sum(p.numel() for p in model.parameters()):,} parameters, "
      f"{model.config.contextLength:,}-byte context")

messages = []
while True:
    try:
        text = input("\nyou> ")
    except EOFError:
        break
    if not text:
        break
    if text == "/reset":
        messages = []
        continue
    messages.append({"role": "user", "content": text})
    prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True)
    ids = prompt["input_ids"]
    with torch.no_grad():
        out = model.generate(ids, do_sample=args.temperature > 0, temperature=args.temperature or None, top_k=args.top_k,
                             max_new_tokens=args.max_bytes, stop_strings=[end], tokenizer=tokenizer)
    reply = tokenizer.decode(out[0, ids.shape[1] :])
    reply = reply.split(end)[0]
    print(f"model> {reply}")
    messages.append({"role": "assistant", "content": reply})