File size: 7,963 Bytes
be97e4b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
"""
Chat with a NanoI1 model (SFT checkpoint or final/ export).

    python chat.py --ckpt runs/nanoi1-155m-sft/final/model.pt                 # interactive
    python chat.py --ckpt ... --prompt "Summarise KYC in one line"             # one-shot
    python chat.py --ckpt ... --system "You are a concise banking assistant."  # with a system prompt

Commands in interactive mode: /reset (new conversation), /system <text>, /exit.
"""

import argparse
import os
import sys

import torch

from chat_format import prompt_ids, sanitize, stop_ids


@torch.no_grad()
def generate(model, ids, max_new_tokens=256, temperature=0.7, top_p=0.9, top_k=0,
             repetition_penalty=1.1, stop=(), context_len=2048, stream=None):
    """Sample a continuation of `ids` using the model's latent KV cache. Returns new token ids."""
    device = next(model.parameters()).device
    ids = list(ids)[-(context_len - max_new_tokens):] if len(ids) + max_new_tokens > context_len else list(ids)
    model.reset_cache()
    logits = model(torch.tensor([ids], device=device), use_cache=True)[0, -1].float()
    out = []
    try:
        for _ in range(max_new_tokens):
            if repetition_penalty != 1.0 and (ids or out):
                seen = torch.tensor(list(set(ids[-256:] + out)), device=device)
                vals = logits[seen]
                logits[seen] = torch.where(vals > 0, vals / repetition_penalty, vals * repetition_penalty)
            if temperature <= 0:
                nxt = int(logits.argmax())
            else:
                lg = logits / temperature
                if top_k:
                    kth = torch.topk(lg, top_k).values[-1]
                    lg[lg < kth] = -float("inf")
                probs = torch.softmax(lg, -1)
                if 0 < top_p < 1:
                    sp, si = torch.sort(probs, descending=True)
                    cut = torch.cumsum(sp, 0) - sp > top_p
                    sp[cut] = 0
                    probs = torch.zeros_like(probs).scatter_(0, si, sp)
                    probs /= probs.sum()
                nxt = int(torch.multinomial(probs, 1))
            if nxt in stop:
                break
            out.append(nxt)
            if stream:
                stream(nxt)
            logits = model(torch.tensor([[nxt]], device=device), use_cache=True)[0, -1].float()
    finally:
        model.reset_cache()
    return out


@torch.no_grad()
def generate_batch(model, prompts, max_new_tokens, stop=(), pad_id=0, until=None, tok=None):
    """Greedy-decode several prompts at once with the latent KV cache.

    Prompts are left-padded and a key-padding mask keeps padding out of attention (RoPE is relative,
    so the shift from left padding does not change any real token's attention). A row stops at a stop
    token or once any `until` string appears in its decoded text (needs `tok`). Returns new ids per row."""
    device = next(model.parameters()).device
    B = len(prompts)
    L = max(len(p) for p in prompts)
    x = torch.full((B, L), pad_id, dtype=torch.long, device=device)
    mask = torch.zeros((B, L), dtype=torch.bool, device=device)
    for i, p in enumerate(prompts):
        x[i, L - len(p):] = torch.tensor(p, device=device)
        mask[i, L - len(p):] = True
    out = [[] for _ in range(B)]
    done = torch.zeros(B, dtype=torch.bool, device=device)
    stop_t = torch.tensor(list(stop) or [-1], device=device)
    model.reset_cache()
    try:
        logits = model(x, mask=mask, use_cache=True)[:, -1]
        for _ in range(max_new_tokens):
            nxt = logits.float().argmax(-1)
            done |= torch.isin(nxt, stop_t)
            nxt_l = nxt.tolist()
            done_l = done.tolist()
            for i in range(B):
                if not done_l[i]:
                    out[i].append(nxt_l[i])
                    if until and tok is not None and any(u in tok.decode(out[i][-64:]) for u in until):
                        done[i] = True
            if bool(done.all()):
                break
            mask = torch.cat([mask, torch.ones((B, 1), dtype=torch.bool, device=device)], 1)
            logits = model(nxt[:, None], mask=mask, use_cache=True)[:, -1]
    finally:
        model.reset_cache()
    return out


def generate_reply(model, tok, messages, max_new_tokens=256, temperature=0.7, top_p=0.9,
                   repetition_penalty=1.1, stream=False):
    ids = prompt_ids(tok, messages)
    printer = None
    if stream:
        buf = []

        def printer(t):  # decode incrementally, print only complete UTF-8
            buf.append(t)
            s = tok.decode(buf)
            if not s.endswith("�"):
                sys.stdout.write(s)
                sys.stdout.flush()
                buf.clear()
    out = generate(model, ids, max_new_tokens, temperature, top_p, 0,
                   repetition_penalty if temperature > 0 else 1.0, stop_ids(tok),
                   model.config.get("context_len", 2048) if hasattr(model, "config") else 2048, printer)
    return tok.decode(out).strip()


def load(ckpt, device):
    from model import NanoI1Model
    state = torch.load(ckpt, map_location="cpu", weights_only=False)
    model = NanoI1Model(**state["model_config"])
    model.load_state_dict(state["model"])
    model.to(device).eval()
    seq = (state.get("train_args") or {}).get("seq_len", 2048)
    model.config = {**model.config, "context_len": min(model.config["context_len"], seq)}
    return model


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--ckpt", required=True)
    ap.add_argument("--tokenizer", default=None, help="default: tokenizer.json next to the checkpoint, else tokenizer/")
    ap.add_argument("--prompt", default=None)
    ap.add_argument("--system", default=None)
    ap.add_argument("--max_new_tokens", type=int, default=512)
    ap.add_argument("--temperature", type=float, default=0.7)
    ap.add_argument("--top_p", type=float, default=0.9)
    ap.add_argument("--repetition_penalty", type=float, default=1.1)
    a = ap.parse_args()

    from tokenizers import Tokenizer
    tok_path = a.tokenizer or next((p for p in (os.path.join(os.path.dirname(a.ckpt), "tokenizer.json"),
                                                "tokenizer/tokenizer.json") if os.path.exists(p)), None)
    if not tok_path:
        raise SystemExit("tokenizer.json not found: pass --tokenizer")
    tok = Tokenizer.from_file(tok_path)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = load(a.ckpt, device)
    amp = torch.autocast("cuda", dtype=torch.bfloat16) if device.type == "cuda" else torch.autocast("cpu", enabled=False)

    system = a.system
    history = []

    def ask(text):
        msgs = ([{"role": "system", "content": sanitize(system, tok)}] if system else []) + history + \
               [{"role": "user", "content": sanitize(text, tok)}]
        with amp:
            reply = generate_reply(model, tok, msgs, a.max_new_tokens, a.temperature, a.top_p,
                                   a.repetition_penalty, stream=True)
        print()
        return reply

    if a.prompt:
        ask(a.prompt)
        return
    print("NanoI1 chat - /reset, /system <text>, /exit")
    while True:
        try:
            text = input("\nyou> ").strip()
        except (EOFError, KeyboardInterrupt):
            break
        if not text:
            continue
        if text == "/exit":
            break
        if text == "/reset":
            history.clear()
            print("(new conversation)")
            continue
        if text.startswith("/system"):
            system = text[len("/system"):].strip() or None
            history.clear()
            print(f"(system prompt {'set' if system else 'cleared'}; new conversation)")
            continue
        sys.stdout.write("bot> ")
        reply = ask(text)
        history += [{"role": "user", "content": text}, {"role": "assistant", "content": reply}]


if __name__ == "__main__":
    main()