NanoI1-155M-Base / chat.py
anuj0456's picture
Upload folder using huggingface_hub
8a73bb7 verified
Raw History Blame Contribute Delete
7.96 kB
"""
Chat with a NanoI1 model (SFT checkpoint or final/ export).
python chat.py --ckpt runs/nanoi1-155m-sft/final/model.pt # interactive
python chat.py --ckpt ... --prompt "Summarise KYC in one line" # one-shot
python chat.py --ckpt ... --system "You are a concise banking assistant." # with a system prompt
Commands in interactive mode: /reset (new conversation), /system <text>, /exit.
"""
import argparse
import os
import sys
import torch
from chat_format import prompt_ids, sanitize, stop_ids
@torch.no_grad()
def generate(model, ids, max_new_tokens=256, temperature=0.7, top_p=0.9, top_k=0,
repetition_penalty=1.1, stop=(), context_len=2048, stream=None):
"""Sample a continuation of `ids` using the model's latent KV cache. Returns new token ids."""
device = next(model.parameters()).device
ids = list(ids)[-(context_len - max_new_tokens):] if len(ids) + max_new_tokens > context_len else list(ids)
model.reset_cache()
logits = model(torch.tensor([ids], device=device), use_cache=True)[0, -1].float()
out = []
try:
for _ in range(max_new_tokens):
if repetition_penalty != 1.0 and (ids or out):
seen = torch.tensor(list(set(ids[-256:] + out)), device=device)
vals = logits[seen]
logits[seen] = torch.where(vals > 0, vals / repetition_penalty, vals * repetition_penalty)
if temperature <= 0:
nxt = int(logits.argmax())
else:
lg = logits / temperature
if top_k:
kth = torch.topk(lg, top_k).values[-1]
lg[lg < kth] = -float("inf")
probs = torch.softmax(lg, -1)
if 0 < top_p < 1:
sp, si = torch.sort(probs, descending=True)
cut = torch.cumsum(sp, 0) - sp > top_p
sp[cut] = 0
probs = torch.zeros_like(probs).scatter_(0, si, sp)
probs /= probs.sum()
nxt = int(torch.multinomial(probs, 1))
if nxt in stop:
break
out.append(nxt)
if stream:
stream(nxt)
logits = model(torch.tensor([[nxt]], device=device), use_cache=True)[0, -1].float()
finally:
model.reset_cache()
return out
@torch.no_grad()
def generate_batch(model, prompts, max_new_tokens, stop=(), pad_id=0, until=None, tok=None):
"""Greedy-decode several prompts at once with the latent KV cache.
Prompts are left-padded and a key-padding mask keeps padding out of attention (RoPE is relative,
so the shift from left padding does not change any real token's attention). A row stops at a stop
token or once any `until` string appears in its decoded text (needs `tok`). Returns new ids per row."""
device = next(model.parameters()).device
B = len(prompts)
L = max(len(p) for p in prompts)
x = torch.full((B, L), pad_id, dtype=torch.long, device=device)
mask = torch.zeros((B, L), dtype=torch.bool, device=device)
for i, p in enumerate(prompts):
x[i, L - len(p):] = torch.tensor(p, device=device)
mask[i, L - len(p):] = True
out = [[] for _ in range(B)]
done = torch.zeros(B, dtype=torch.bool, device=device)
stop_t = torch.tensor(list(stop) or [-1], device=device)
model.reset_cache()
try:
logits = model(x, mask=mask, use_cache=True)[:, -1]
for _ in range(max_new_tokens):
nxt = logits.float().argmax(-1)
done |= torch.isin(nxt, stop_t)
nxt_l = nxt.tolist()
done_l = done.tolist()
for i in range(B):
if not done_l[i]:
out[i].append(nxt_l[i])
if until and tok is not None and any(u in tok.decode(out[i][-64:]) for u in until):
done[i] = True
if bool(done.all()):
break
mask = torch.cat([mask, torch.ones((B, 1), dtype=torch.bool, device=device)], 1)
logits = model(nxt[:, None], mask=mask, use_cache=True)[:, -1]
finally:
model.reset_cache()
return out
def generate_reply(model, tok, messages, max_new_tokens=256, temperature=0.7, top_p=0.9,
repetition_penalty=1.1, stream=False):
ids = prompt_ids(tok, messages)
printer = None
if stream:
buf = []
def printer(t): # decode incrementally, print only complete UTF-8
buf.append(t)
s = tok.decode(buf)
if not s.endswith("�"):
sys.stdout.write(s)
sys.stdout.flush()
buf.clear()
out = generate(model, ids, max_new_tokens, temperature, top_p, 0,
repetition_penalty if temperature > 0 else 1.0, stop_ids(tok),
model.config.get("context_len", 2048) if hasattr(model, "config") else 2048, printer)
return tok.decode(out).strip()
def load(ckpt, device):
from model import NanoI1Model
state = torch.load(ckpt, map_location="cpu", weights_only=False)
model = NanoI1Model(**state["model_config"])
model.load_state_dict(state["model"])
model.to(device).eval()
seq = (state.get("train_args") or {}).get("seq_len", 2048)
model.config = {**model.config, "context_len": min(model.config["context_len"], seq)}
return model
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--ckpt", required=True)
ap.add_argument("--tokenizer", default=None, help="default: tokenizer.json next to the checkpoint, else tokenizer/")
ap.add_argument("--prompt", default=None)
ap.add_argument("--system", default=None)
ap.add_argument("--max_new_tokens", type=int, default=512)
ap.add_argument("--temperature", type=float, default=0.7)
ap.add_argument("--top_p", type=float, default=0.9)
ap.add_argument("--repetition_penalty", type=float, default=1.1)
a = ap.parse_args()
from tokenizers import Tokenizer
tok_path = a.tokenizer or next((p for p in (os.path.join(os.path.dirname(a.ckpt), "tokenizer.json"),
"tokenizer/tokenizer.json") if os.path.exists(p)), None)
if not tok_path:
raise SystemExit("tokenizer.json not found: pass --tokenizer")
tok = Tokenizer.from_file(tok_path)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = load(a.ckpt, device)
amp = torch.autocast("cuda", dtype=torch.bfloat16) if device.type == "cuda" else torch.autocast("cpu", enabled=False)
system = a.system
history = []
def ask(text):
msgs = ([{"role": "system", "content": sanitize(system, tok)}] if system else []) + history + \
[{"role": "user", "content": sanitize(text, tok)}]
with amp:
reply = generate_reply(model, tok, msgs, a.max_new_tokens, a.temperature, a.top_p,
a.repetition_penalty, stream=True)
print()
return reply
if a.prompt:
ask(a.prompt)
return
print("NanoI1 chat - /reset, /system <text>, /exit")
while True:
try:
text = input("\nyou> ").strip()
except (EOFError, KeyboardInterrupt):
break
if not text:
continue
if text == "/exit":
break
if text == "/reset":
history.clear()
print("(new conversation)")
continue
if text.startswith("/system"):
system = text[len("/system"):].strip() or None
history.clear()
print(f"(system prompt {'set' if system else 'cleared'}; new conversation)")
continue
sys.stdout.write("bot> ")
reply = ask(text)
history += [{"role": "user", "content": text}, {"role": "assistant", "content": reply}]
if __name__ == "__main__":
main()