""" Chat with a NanoI1 model (SFT checkpoint or final/ export). python chat.py --ckpt runs/nanoi1-155m-sft/final/model.pt # interactive python chat.py --ckpt ... --prompt "Summarise KYC in one line" # one-shot python chat.py --ckpt ... --system "You are a concise banking assistant." # with a system prompt Commands in interactive mode: /reset (new conversation), /system , /exit. """ import argparse import os import sys import torch from chat_format import prompt_ids, sanitize, stop_ids @torch.no_grad() def generate(model, ids, max_new_tokens=256, temperature=0.7, top_p=0.9, top_k=0, repetition_penalty=1.1, stop=(), context_len=2048, stream=None): """Sample a continuation of `ids` using the model's latent KV cache. Returns new token ids.""" device = next(model.parameters()).device ids = list(ids)[-(context_len - max_new_tokens):] if len(ids) + max_new_tokens > context_len else list(ids) model.reset_cache() logits = model(torch.tensor([ids], device=device), use_cache=True)[0, -1].float() out = [] try: for _ in range(max_new_tokens): if repetition_penalty != 1.0 and (ids or out): seen = torch.tensor(list(set(ids[-256:] + out)), device=device) vals = logits[seen] logits[seen] = torch.where(vals > 0, vals / repetition_penalty, vals * repetition_penalty) if temperature <= 0: nxt = int(logits.argmax()) else: lg = logits / temperature if top_k: kth = torch.topk(lg, top_k).values[-1] lg[lg < kth] = -float("inf") probs = torch.softmax(lg, -1) if 0 < top_p < 1: sp, si = torch.sort(probs, descending=True) cut = torch.cumsum(sp, 0) - sp > top_p sp[cut] = 0 probs = torch.zeros_like(probs).scatter_(0, si, sp) probs /= probs.sum() nxt = int(torch.multinomial(probs, 1)) if nxt in stop: break out.append(nxt) if stream: stream(nxt) logits = model(torch.tensor([[nxt]], device=device), use_cache=True)[0, -1].float() finally: model.reset_cache() return out @torch.no_grad() def generate_batch(model, prompts, max_new_tokens, stop=(), pad_id=0, until=None, tok=None): """Greedy-decode several prompts at once with the latent KV cache. Prompts are left-padded and a key-padding mask keeps padding out of attention (RoPE is relative, so the shift from left padding does not change any real token's attention). A row stops at a stop token or once any `until` string appears in its decoded text (needs `tok`). Returns new ids per row.""" device = next(model.parameters()).device B = len(prompts) L = max(len(p) for p in prompts) x = torch.full((B, L), pad_id, dtype=torch.long, device=device) mask = torch.zeros((B, L), dtype=torch.bool, device=device) for i, p in enumerate(prompts): x[i, L - len(p):] = torch.tensor(p, device=device) mask[i, L - len(p):] = True out = [[] for _ in range(B)] done = torch.zeros(B, dtype=torch.bool, device=device) stop_t = torch.tensor(list(stop) or [-1], device=device) model.reset_cache() try: logits = model(x, mask=mask, use_cache=True)[:, -1] for _ in range(max_new_tokens): nxt = logits.float().argmax(-1) done |= torch.isin(nxt, stop_t) nxt_l = nxt.tolist() done_l = done.tolist() for i in range(B): if not done_l[i]: out[i].append(nxt_l[i]) if until and tok is not None and any(u in tok.decode(out[i][-64:]) for u in until): done[i] = True if bool(done.all()): break mask = torch.cat([mask, torch.ones((B, 1), dtype=torch.bool, device=device)], 1) logits = model(nxt[:, None], mask=mask, use_cache=True)[:, -1] finally: model.reset_cache() return out def generate_reply(model, tok, messages, max_new_tokens=256, temperature=0.7, top_p=0.9, repetition_penalty=1.1, stream=False): ids = prompt_ids(tok, messages) printer = None if stream: buf = [] def printer(t): # decode incrementally, print only complete UTF-8 buf.append(t) s = tok.decode(buf) if not s.endswith("�"): sys.stdout.write(s) sys.stdout.flush() buf.clear() out = generate(model, ids, max_new_tokens, temperature, top_p, 0, repetition_penalty if temperature > 0 else 1.0, stop_ids(tok), model.config.get("context_len", 2048) if hasattr(model, "config") else 2048, printer) return tok.decode(out).strip() def load(ckpt, device): from model import NanoI1Model state = torch.load(ckpt, map_location="cpu", weights_only=False) model = NanoI1Model(**state["model_config"]) model.load_state_dict(state["model"]) model.to(device).eval() seq = (state.get("train_args") or {}).get("seq_len", 2048) model.config = {**model.config, "context_len": min(model.config["context_len"], seq)} return model def main(): ap = argparse.ArgumentParser() ap.add_argument("--ckpt", required=True) ap.add_argument("--tokenizer", default=None, help="default: tokenizer.json next to the checkpoint, else tokenizer/") ap.add_argument("--prompt", default=None) ap.add_argument("--system", default=None) ap.add_argument("--max_new_tokens", type=int, default=512) ap.add_argument("--temperature", type=float, default=0.7) ap.add_argument("--top_p", type=float, default=0.9) ap.add_argument("--repetition_penalty", type=float, default=1.1) a = ap.parse_args() from tokenizers import Tokenizer tok_path = a.tokenizer or next((p for p in (os.path.join(os.path.dirname(a.ckpt), "tokenizer.json"), "tokenizer/tokenizer.json") if os.path.exists(p)), None) if not tok_path: raise SystemExit("tokenizer.json not found: pass --tokenizer") tok = Tokenizer.from_file(tok_path) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = load(a.ckpt, device) amp = torch.autocast("cuda", dtype=torch.bfloat16) if device.type == "cuda" else torch.autocast("cpu", enabled=False) system = a.system history = [] def ask(text): msgs = ([{"role": "system", "content": sanitize(system, tok)}] if system else []) + history + \ [{"role": "user", "content": sanitize(text, tok)}] with amp: reply = generate_reply(model, tok, msgs, a.max_new_tokens, a.temperature, a.top_p, a.repetition_penalty, stream=True) print() return reply if a.prompt: ask(a.prompt) return print("NanoI1 chat - /reset, /system , /exit") while True: try: text = input("\nyou> ").strip() except (EOFError, KeyboardInterrupt): break if not text: continue if text == "/exit": break if text == "/reset": history.clear() print("(new conversation)") continue if text.startswith("/system"): system = text[len("/system"):].strip() or None history.clear() print(f"(system prompt {'set' if system else 'cleared'}; new conversation)") continue sys.stdout.write("bot> ") reply = ask(text) history += [{"role": "user", "content": text}, {"role": "assistant", "content": reply}] if __name__ == "__main__": main()