"""Chat with Gab10M in the terminal, through transformers. pip install torch transformers safetensors python chat.py [MODEL] [--temperature 0.7] [--max-bytes 400] [--seed N] MODEL is the repository's name on the Hub (gszauer/Gab10M) or a local copy of it (default: the folder this script is in). An empty line quits; /reset starts over. The model sees the conversation as plain bytes: <|u|>question<|e|><|a|>reply<|e|><|u|>... and its reply ends when it writes <|e|>. Its context slides: past contextLength bytes, it keeps only the latest ones. """ import argparse from pathlib import Path import torch from transformers import AutoModelForCausalLM, AutoTokenizer parser = argparse.ArgumentParser() parser.add_argument("model", nargs="?", default=str(Path(__file__).resolve().parent)) parser.add_argument("--temperature", type=float, default=0.7) parser.add_argument("--top-k", type=int, default=40) parser.add_argument("--max-bytes", type=int, default=400) parser.add_argument("--seed", type=int) args = parser.parse_args() if args.seed is not None: torch.manual_seed(args.seed) tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=True).eval() end = model.config.chat["end"] print(f"Gab10M: {sum(p.numel() for p in model.parameters()):,} parameters, " f"{model.config.contextLength:,}-byte context") messages = [] while True: try: text = input("\nyou> ") except EOFError: break if not text: break if text == "/reset": messages = [] continue messages.append({"role": "user", "content": text}) prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) ids = prompt["input_ids"] with torch.no_grad(): out = model.generate(ids, do_sample=args.temperature > 0, temperature=args.temperature or None, top_k=args.top_k, max_new_tokens=args.max_bytes, stop_strings=[end], tokenizer=tokenizer) reply = tokenizer.decode(out[0, ids.shape[1] :]) reply = reply.split(end)[0] print(f"model> {reply}") messages.append({"role": "assistant", "content": reply})