Gab10M / chat.py
gszauer's picture
Gab 10M: chat model, Transformers port, reference.js
04036d0 verified
Raw History Blame Contribute Delete
2.26 kB
"""Chat with Gab10M in the terminal, through transformers.
pip install torch transformers safetensors
python chat.py [MODEL] [--temperature 0.7] [--max-bytes 400] [--seed N]
MODEL is the repository's name on the Hub (gszauer/Gab10M) or a local copy of
it (default: the folder this script is in). An empty line quits; /reset
starts over.
The model sees the conversation as plain bytes:
<|u|>question<|e|><|a|>reply<|e|><|u|>...
and its reply ends when it writes <|e|>. Its context slides: past
contextLength bytes, it keeps only the latest ones.
"""
import argparse
from pathlib import Path
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
parser = argparse.ArgumentParser()
parser.add_argument("model", nargs="?", default=str(Path(__file__).resolve().parent))
parser.add_argument("--temperature", type=float, default=0.7)
parser.add_argument("--top-k", type=int, default=40)
parser.add_argument("--max-bytes", type=int, default=400)
parser.add_argument("--seed", type=int)
args = parser.parse_args()
if args.seed is not None:
torch.manual_seed(args.seed)
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=True).eval()
end = model.config.chat["end"]
print(f"Gab10M: {sum(p.numel() for p in model.parameters()):,} parameters, "
f"{model.config.contextLength:,}-byte context")
messages = []
while True:
try:
text = input("\nyou> ")
except EOFError:
break
if not text:
break
if text == "/reset":
messages = []
continue
messages.append({"role": "user", "content": text})
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True)
ids = prompt["input_ids"]
with torch.no_grad():
out = model.generate(ids, do_sample=args.temperature > 0, temperature=args.temperature or None, top_k=args.top_k,
max_new_tokens=args.max_bytes, stop_strings=[end], tokenizer=tokenizer)
reply = tokenizer.decode(out[0, ids.shape[1] :])
reply = reply.split(end)[0]
print(f"model> {reply}")
messages.append({"role": "assistant", "content": reply})