Download chat.py from KiteFishAI/NanoI1-155M-Base: direct link, hf CLI and curl.
- Browser
- Download file 7.96 kB
-
https://huggingface.co/KiteFishAI/NanoI1-155M-Base/resolve/main/chat.py
- Command line
-
hf download hf://KiteFishAI/NanoI1-155M-Base/chat.py
-
curl -L -o chat.py https://huggingface.co/KiteFishAI/NanoI1-155M-Base/resolve/main/chat.py
7.96 kB
| """ | |
| Chat with a NanoI1 model (SFT checkpoint or final/ export). | |
| python chat.py --ckpt runs/nanoi1-155m-sft/final/model.pt # interactive | |
| python chat.py --ckpt ... --prompt "Summarise KYC in one line" # one-shot | |
| python chat.py --ckpt ... --system "You are a concise banking assistant." # with a system prompt | |
| Commands in interactive mode: /reset (new conversation), /system <text>, /exit. | |
| """ | |
| import argparse | |
| import os | |
| import sys | |
| import torch | |
| from chat_format import prompt_ids, sanitize, stop_ids | |
| def generate(model, ids, max_new_tokens=256, temperature=0.7, top_p=0.9, top_k=0, | |
| repetition_penalty=1.1, stop=(), context_len=2048, stream=None): | |
| """Sample a continuation of `ids` using the model's latent KV cache. Returns new token ids.""" | |
| device = next(model.parameters()).device | |
| ids = list(ids)[-(context_len - max_new_tokens):] if len(ids) + max_new_tokens > context_len else list(ids) | |
| model.reset_cache() | |
| logits = model(torch.tensor([ids], device=device), use_cache=True)[0, -1].float() | |
| out = [] | |
| try: | |
| for _ in range(max_new_tokens): | |
| if repetition_penalty != 1.0 and (ids or out): | |
| seen = torch.tensor(list(set(ids[-256:] + out)), device=device) | |
| vals = logits[seen] | |
| logits[seen] = torch.where(vals > 0, vals / repetition_penalty, vals * repetition_penalty) | |
| if temperature <= 0: | |
| nxt = int(logits.argmax()) | |
| else: | |
| lg = logits / temperature | |
| if top_k: | |
| kth = torch.topk(lg, top_k).values[-1] | |
| lg[lg < kth] = -float("inf") | |
| probs = torch.softmax(lg, -1) | |
| if 0 < top_p < 1: | |
| sp, si = torch.sort(probs, descending=True) | |
| cut = torch.cumsum(sp, 0) - sp > top_p | |
| sp[cut] = 0 | |
| probs = torch.zeros_like(probs).scatter_(0, si, sp) | |
| probs /= probs.sum() | |
| nxt = int(torch.multinomial(probs, 1)) | |
| if nxt in stop: | |
| break | |
| out.append(nxt) | |
| if stream: | |
| stream(nxt) | |
| logits = model(torch.tensor([[nxt]], device=device), use_cache=True)[0, -1].float() | |
| finally: | |
| model.reset_cache() | |
| return out | |
| def generate_batch(model, prompts, max_new_tokens, stop=(), pad_id=0, until=None, tok=None): | |
| """Greedy-decode several prompts at once with the latent KV cache. | |
| Prompts are left-padded and a key-padding mask keeps padding out of attention (RoPE is relative, | |
| so the shift from left padding does not change any real token's attention). A row stops at a stop | |
| token or once any `until` string appears in its decoded text (needs `tok`). Returns new ids per row.""" | |
| device = next(model.parameters()).device | |
| B = len(prompts) | |
| L = max(len(p) for p in prompts) | |
| x = torch.full((B, L), pad_id, dtype=torch.long, device=device) | |
| mask = torch.zeros((B, L), dtype=torch.bool, device=device) | |
| for i, p in enumerate(prompts): | |
| x[i, L - len(p):] = torch.tensor(p, device=device) | |
| mask[i, L - len(p):] = True | |
| out = [[] for _ in range(B)] | |
| done = torch.zeros(B, dtype=torch.bool, device=device) | |
| stop_t = torch.tensor(list(stop) or [-1], device=device) | |
| model.reset_cache() | |
| try: | |
| logits = model(x, mask=mask, use_cache=True)[:, -1] | |
| for _ in range(max_new_tokens): | |
| nxt = logits.float().argmax(-1) | |
| done |= torch.isin(nxt, stop_t) | |
| nxt_l = nxt.tolist() | |
| done_l = done.tolist() | |
| for i in range(B): | |
| if not done_l[i]: | |
| out[i].append(nxt_l[i]) | |
| if until and tok is not None and any(u in tok.decode(out[i][-64:]) for u in until): | |
| done[i] = True | |
| if bool(done.all()): | |
| break | |
| mask = torch.cat([mask, torch.ones((B, 1), dtype=torch.bool, device=device)], 1) | |
| logits = model(nxt[:, None], mask=mask, use_cache=True)[:, -1] | |
| finally: | |
| model.reset_cache() | |
| return out | |
| def generate_reply(model, tok, messages, max_new_tokens=256, temperature=0.7, top_p=0.9, | |
| repetition_penalty=1.1, stream=False): | |
| ids = prompt_ids(tok, messages) | |
| printer = None | |
| if stream: | |
| buf = [] | |
| def printer(t): # decode incrementally, print only complete UTF-8 | |
| buf.append(t) | |
| s = tok.decode(buf) | |
| if not s.endswith("�"): | |
| sys.stdout.write(s) | |
| sys.stdout.flush() | |
| buf.clear() | |
| out = generate(model, ids, max_new_tokens, temperature, top_p, 0, | |
| repetition_penalty if temperature > 0 else 1.0, stop_ids(tok), | |
| model.config.get("context_len", 2048) if hasattr(model, "config") else 2048, printer) | |
| return tok.decode(out).strip() | |
| def load(ckpt, device): | |
| from model import NanoI1Model | |
| state = torch.load(ckpt, map_location="cpu", weights_only=False) | |
| model = NanoI1Model(**state["model_config"]) | |
| model.load_state_dict(state["model"]) | |
| model.to(device).eval() | |
| seq = (state.get("train_args") or {}).get("seq_len", 2048) | |
| model.config = {**model.config, "context_len": min(model.config["context_len"], seq)} | |
| return model | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--ckpt", required=True) | |
| ap.add_argument("--tokenizer", default=None, help="default: tokenizer.json next to the checkpoint, else tokenizer/") | |
| ap.add_argument("--prompt", default=None) | |
| ap.add_argument("--system", default=None) | |
| ap.add_argument("--max_new_tokens", type=int, default=512) | |
| ap.add_argument("--temperature", type=float, default=0.7) | |
| ap.add_argument("--top_p", type=float, default=0.9) | |
| ap.add_argument("--repetition_penalty", type=float, default=1.1) | |
| a = ap.parse_args() | |
| from tokenizers import Tokenizer | |
| tok_path = a.tokenizer or next((p for p in (os.path.join(os.path.dirname(a.ckpt), "tokenizer.json"), | |
| "tokenizer/tokenizer.json") if os.path.exists(p)), None) | |
| if not tok_path: | |
| raise SystemExit("tokenizer.json not found: pass --tokenizer") | |
| tok = Tokenizer.from_file(tok_path) | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model = load(a.ckpt, device) | |
| amp = torch.autocast("cuda", dtype=torch.bfloat16) if device.type == "cuda" else torch.autocast("cpu", enabled=False) | |
| system = a.system | |
| history = [] | |
| def ask(text): | |
| msgs = ([{"role": "system", "content": sanitize(system, tok)}] if system else []) + history + \ | |
| [{"role": "user", "content": sanitize(text, tok)}] | |
| with amp: | |
| reply = generate_reply(model, tok, msgs, a.max_new_tokens, a.temperature, a.top_p, | |
| a.repetition_penalty, stream=True) | |
| print() | |
| return reply | |
| if a.prompt: | |
| ask(a.prompt) | |
| return | |
| print("NanoI1 chat - /reset, /system <text>, /exit") | |
| while True: | |
| try: | |
| text = input("\nyou> ").strip() | |
| except (EOFError, KeyboardInterrupt): | |
| break | |
| if not text: | |
| continue | |
| if text == "/exit": | |
| break | |
| if text == "/reset": | |
| history.clear() | |
| print("(new conversation)") | |
| continue | |
| if text.startswith("/system"): | |
| system = text[len("/system"):].strip() or None | |
| history.clear() | |
| print(f"(system prompt {'set' if system else 'cleared'}; new conversation)") | |
| continue | |
| sys.stdout.write("bot> ") | |
| reply = ask(text) | |
| history += [{"role": "user", "content": text}, {"role": "assistant", "content": reply}] | |
| if __name__ == "__main__": | |
| main() | |