ChatGBA / chat.py
gszauer's picture
ChatGBA: chat model, Game Boy Advance cartridge and source, Transformers port, reference.js
546acd4 verified
Raw History Blame Contribute Delete
2.26 kB
"""Chat with ChatGBA in the terminal, through transformers.
pip install torch transformers safetensors
python chat.py [MODEL] [--temperature 0.7] [--max-bytes 400] [--seed N]
MODEL is the repository's name on the Hub (gszauer/ChatGBA) or a local copy
of it (default: the folder this script is in). An empty line quits; /reset
starts over.
The model sees the conversation as plain bytes:
<|u|>question<|e|><|a|>reply<|e|><|u|>...
and its reply ends when it writes <|e|>. Its context slides: past
contextLength bytes, it keeps only the latest ones.
"""
import argparse
from pathlib import Path
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
parser = argparse.ArgumentParser()
parser.add_argument("model", nargs="?", default=str(Path(__file__).resolve().parent))
parser.add_argument("--temperature", type=float, default=0.7)
parser.add_argument("--top-k", type=int, default=40)
parser.add_argument("--max-bytes", type=int, default=400)
parser.add_argument("--seed", type=int)
args = parser.parse_args()
if args.seed is not None:
torch.manual_seed(args.seed)
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=True).eval()
end = model.config.chat["end"]
print(f"ChatGBA: {sum(p.numel() for p in model.parameters()):,} parameters, "
f"{model.config.contextLength:,}-byte context")
messages = []
while True:
try:
text = input("\nyou> ")
except EOFError:
break
if not text:
break
if text == "/reset":
messages = []
continue
messages.append({"role": "user", "content": text})
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True)
ids = prompt["input_ids"]
with torch.no_grad():
out = model.generate(ids, do_sample=args.temperature > 0, temperature=args.temperature or None, top_k=args.top_k,
max_new_tokens=args.max_bytes, stop_strings=[end], tokenizer=tokenizer)
reply = tokenizer.decode(out[0, ids.shape[1] :])
reply = reply.split(end)[0]
print(f"model> {reply}")
messages.append({"role": "assistant", "content": reply})