ecreeth commited on
Commit
491dfb2
·
verified ·
1 Parent(s): 452556a

Upload 2 files

Browse files
Files changed (2) hide show
  1. chat_cli.py +304 -0
  2. infer.py +309 -0
chat_cli.py ADDED
@@ -0,0 +1,304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Chat with the SFT'd microgpt model. Adapted from karpathy/nanochat's chat_cli.py.
2
+
3
+ Builds chat-format prompts using our 5 special tokens:
4
+ <|bos|> <|user_start|> ...user text... <|user_end|>
5
+ <|assistant_start|> ...assistant text... <|assistant_end|>
6
+
7
+ Conversation history is maintained across turns (each new user message is appended
8
+ to the running token stream). Generation stops on <|assistant_end|>.
9
+
10
+ Usage:
11
+ python3 chat_cli.py # interactive REPL
12
+ python3 chat_cli.py -p "Who are you?" # one-shot
13
+ python3 chat_cli.py --ckpt model.pt # explicit checkpoint path
14
+ python3 chat_cli.py -t 0.6 -k 50 # nanochat-CLI defaults
15
+ python3 chat_cli.py --no-history # reset per turn (no memory)
16
+
17
+ Defaults follow nanochat-CLI (T=0.6, top-k=50). Lower temperature than infer.py's
18
+ default 0.8 because chat is meant to be focused, not exploratory.
19
+
20
+ REPL commands:
21
+ quit / exit end the session
22
+ clear reset conversation history
23
+ """
24
+ import argparse
25
+ import os
26
+ import sys
27
+ import time
28
+
29
+ import torch
30
+ from tokenizers import Tokenizer
31
+
32
+ from model import GPT
33
+ from infer import _sample_token_batch # reuse our sampler (rep penalty + nucleus + top-k)
34
+
35
+
36
+ DEFAULT_CKPT_DIR = os.environ.get('CHECKPOINT_DIR', '.')
37
+ # Prefer model_sft.pt (local dev convention with both base + SFT present), fall
38
+ # back to model.pt (HF convention — only the SFT'd model is uploaded as model.pt).
39
+ DEFAULT_CKPT = os.path.join(DEFAULT_CKPT_DIR, 'model_sft.pt')
40
+ DEFAULT_TOKENIZER = 'tokenizer.json'
41
+
42
+ SPECIAL_NAMES = [
43
+ "<|bos|>",
44
+ "<|user_start|>", "<|user_end|>",
45
+ "<|assistant_start|>", "<|assistant_end|>",
46
+ ]
47
+
48
+ device = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
49
+
50
+
51
+ def _load(ckpt_path, tokenizer_path, compile_step=False):
52
+ """Resolve checkpoint (with fallback), load model + tokenizer + special token IDs.
53
+
54
+ If `compile_step=True`, torch.compile the inner-loop `model.step` (B=1, T=1 fixed
55
+ shapes — perfect for compile). We deliberately don't compile `forward_with_states`
56
+ because chat conversations grow each turn → dynamic prompt length → recompile
57
+ every turn. Prefill stays eager.
58
+
59
+ Compile warmup takes 10–30s on first generation; pays off for any conversation
60
+ long enough to generate >~50 tokens. On MPS, torch.compile is less mature than
61
+ CUDA — try it but don't be surprised if it falls back to eager.
62
+ """
63
+ if not os.path.exists(ckpt_path):
64
+ # Silent fallback for HF layout (only model.pt = the SFT'd one)
65
+ fallback = os.path.join(DEFAULT_CKPT_DIR, 'model.pt')
66
+ if ckpt_path != fallback and os.path.exists(fallback):
67
+ ckpt_path = fallback
68
+ else:
69
+ sys.exit(f"error: no checkpoint at {ckpt_path}")
70
+ if not os.path.exists(tokenizer_path):
71
+ sys.exit(f"error: no tokenizer at {tokenizer_path}")
72
+
73
+ tokenizer = Tokenizer.from_file(tokenizer_path)
74
+ vocab_size = tokenizer.get_vocab_size()
75
+
76
+ ckpt = torch.load(ckpt_path, map_location=device)
77
+ config = dict(ckpt['config'])
78
+ config['vocab_size'] = ((vocab_size + 63) // 64) * 64
79
+ model = GPT.from_config(config).to(device)
80
+ state = {k.removeprefix('_orig_mod.'): v for k, v in ckpt['model'].items()}
81
+ missing, unexpected = model.load_state_dict(state, strict=False)
82
+ if missing: print(f"warn: missing keys: {missing}", file=sys.stderr)
83
+ if unexpected: print(f"warn: unexpected keys: {unexpected}", file=sys.stderr)
84
+ model.eval()
85
+
86
+ specials = {}
87
+ for name in SPECIAL_NAMES:
88
+ tid = tokenizer.token_to_id(name)
89
+ if tid is None:
90
+ sys.exit(f"error: tokenizer is missing special token {name} — was it retrained without the SFT vocab?")
91
+ specials[name] = tid
92
+
93
+ if compile_step:
94
+ print("compiling model.step (hot inner-loop path)...", file=sys.stderr)
95
+ if device == 'mps':
96
+ print(" note: torch.compile on MPS may fall back to eager. CUDA gets the full speedup.",
97
+ file=sys.stderr)
98
+ compile_kwargs = {'dynamic': False}
99
+ # reduce-overhead enables CUDA graphs — big win on Ampere+ where kernel-launch
100
+ # overhead dominates the small per-token forward. No-op on MPS/CPU.
101
+ if device == 'cuda' and torch.cuda.get_device_capability() >= (8, 0):
102
+ compile_kwargs['mode'] = 'reduce-overhead'
103
+ model.step = torch.compile(model.step, **compile_kwargs)
104
+
105
+ return model, tokenizer, specials, ckpt, ckpt_path
106
+
107
+
108
+ @torch.no_grad()
109
+ def _generate_one_turn(model, tokenizer, conversation_tokens, specials,
110
+ max_new_tokens, temperature, top_k, top_p, repetition_penalty,
111
+ stream, cache):
112
+ """Run prefill over the NEW tokens since `cache['pos']` (using cached state),
113
+ then sample tokens until <|assistant_end|> (or <|bos|>, or max_new_tokens).
114
+ Returns the response token list. Mutates `cache` to extend coverage to the
115
+ end of conversation_tokens + the assistant_end that will be appended.
116
+
117
+ `cache` is a dict { 'pos': int, 'states': list[(B,M,D)] | None, 'seen_mask': bool tensor | None }.
118
+ On first call, `cache['states']` is None — we run prefill from scratch.
119
+ On subsequent calls, only the new tokens since `cache['pos']` are prefilled,
120
+ starting from `cache['states']`. Big speedup once the conversation grows.
121
+ """
122
+ bos = specials["<|bos|>"]
123
+ asst_end = specials["<|assistant_end|>"]
124
+ V = model.lm_head.weight.size(0) # padded vocab
125
+
126
+ # Slice off only the new tokens to prefill
127
+ new_tokens = conversation_tokens[cache['pos']:]
128
+ assert len(new_tokens) > 0, "no new tokens to prefill — cache invariant broken"
129
+ new_ctx = torch.tensor([new_tokens], device=device) # (1, T_new)
130
+
131
+ # Initialize / extend the seen_mask (used for repetition penalty)
132
+ if repetition_penalty != 1.0:
133
+ if cache.get('seen_mask') is None:
134
+ cache['seen_mask'] = torch.zeros(1, V, dtype=torch.bool, device=device)
135
+ cache['seen_mask'].scatter_(1, new_ctx, True)
136
+ seen_mask = cache.get('seen_mask')
137
+
138
+ # Prefill only the new tokens, starting from cached states (or zeros on first turn)
139
+ logits, states = model.forward_with_states(new_ctx, initial_states=cache['states'])
140
+ next_logits = logits[:, -1, :].clone()
141
+
142
+ response = []
143
+ for _ in range(max_new_tokens):
144
+ next_tok = _sample_token_batch(next_logits, temperature, top_k, top_p,
145
+ repetition_penalty, seen_mask) # (1,)
146
+ tid = next_tok.item()
147
+ if tid == asst_end or tid == bos:
148
+ break
149
+ response.append(tid)
150
+ if stream:
151
+ # Per-token decode is fine for ASCII English (the ClimbMix corpus dominates that).
152
+ piece = tokenizer.decode([tid])
153
+ sys.stdout.write(piece)
154
+ sys.stdout.flush()
155
+ if seen_mask is not None:
156
+ seen_mask.scatter_(1, next_tok.unsqueeze(1), True)
157
+ step_logits, states = model.step(next_tok.view(1, 1), states)
158
+ next_logits = step_logits[:, 0, :]
159
+
160
+ if stream:
161
+ print()
162
+
163
+ # Advance state through assistant_end so the cache reflects the closed turn.
164
+ # (Generation stopped at either an emitted assistant_end, BOS, or max_tokens;
165
+ # in the latter two cases we still need to step state through assistant_end.)
166
+ asst_end_tok = torch.tensor([[asst_end]], device=device)
167
+ _, states = model.step(asst_end_tok, states)
168
+ if seen_mask is not None:
169
+ seen_mask.scatter_(1, asst_end_tok, True)
170
+
171
+ # Update cache: state now reflects [prev cache] + new prefill tokens + generated response + assistant_end
172
+ cache['states'] = states
173
+ cache['pos'] = len(conversation_tokens) + len(response) + 1 # +1 for assistant_end
174
+
175
+ return response
176
+
177
+
178
+ def main():
179
+ p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
180
+ p.add_argument('-p', '--prompt', type=str, default='',
181
+ help='single-turn prompt — exits after one response. Empty = interactive REPL.')
182
+ p.add_argument('-t', '--temperature', type=float, default=0.6,
183
+ help='softmax temperature; 0 = greedy. nanochat-CLI default 0.6 — '
184
+ 'tighter than infer.py because chat should be focused.')
185
+ p.add_argument('-k', '--top-k', type=int, default=50,
186
+ help='top-k sampling; 0 disables. Default 50 (nanochat-CLI).')
187
+ p.add_argument('--top-p', type=float, default=1.0,
188
+ help='nucleus sampling threshold; 1.0 disables. Try 0.9 for varied responses.')
189
+ p.add_argument('-r', '--repetition-penalty', type=float, default=1.15,
190
+ help='CTRL-style repetition penalty (default 1.15) — keeps chat responses '
191
+ 'from looping. Set to 1.0 for raw sampling.')
192
+ p.add_argument('-n', '--max-tokens', type=int, default=256,
193
+ help='max tokens per assistant response')
194
+ p.add_argument('--ckpt', type=str, default=DEFAULT_CKPT,
195
+ help='checkpoint path (default $CHECKPOINT_DIR/model.pt)')
196
+ p.add_argument('--tokenizer', type=str, default=DEFAULT_TOKENIZER)
197
+ p.add_argument('--no-history', action='store_true',
198
+ help='reset conversation history before each turn (model has no memory)')
199
+ p.add_argument('--seed', type=int, default=None)
200
+ p.add_argument('--no-stream', action='store_true',
201
+ help='print full response at end instead of token-by-token')
202
+ p.add_argument('--compile', action='store_true',
203
+ help='torch.compile the inner step() path. First generation pays '
204
+ '~10–30s warmup; subsequent generations are 2–5× faster on CUDA. '
205
+ 'Best for long REPL sessions; skip for one-shots.')
206
+ args = p.parse_args()
207
+
208
+ if args.seed is not None:
209
+ torch.manual_seed(args.seed)
210
+
211
+ print(f"device: {device}", file=sys.stderr)
212
+ model, tokenizer, specials, ckpt, used_ckpt = _load(args.ckpt, args.tokenizer,
213
+ compile_step=args.compile)
214
+ step = ckpt.get('step', '?')
215
+ best = ckpt.get('best_loss')
216
+ n_params = sum(t.numel() for t in model.parameters())
217
+ best_str = f" best_loss={best:.4f}" if isinstance(best, float) else ""
218
+ print(f"loaded {used_ckpt} step={step}{best_str} params={n_params:,}", file=sys.stderr)
219
+
220
+ bos = specials["<|bos|>"]
221
+ user_s = specials["<|user_start|>"]
222
+ user_e = specials["<|user_end|>"]
223
+ asst_s = specials["<|assistant_start|>"]
224
+ asst_e = specials["<|assistant_end|>"]
225
+
226
+ print()
227
+ print("Mnemo — chat mode")
228
+ print("-" * 50)
229
+ print(f"sampling: T={args.temperature} top_k={args.top_k} top_p={args.top_p} rep_penalty={args.repetition_penalty}")
230
+ if not args.prompt:
231
+ print("commands: 'quit' / 'exit' to end, 'clear' to reset history")
232
+ print("-" * 50)
233
+
234
+ conversation_tokens = [bos]
235
+ # State cache: skips re-prefilling already-processed turns.
236
+ # 'pos' = how far into conversation_tokens the cached state covers.
237
+ # 'states' = per-layer (B,M,D) states, None on cold start.
238
+ # 'seen_mask' = (1,V) bool over the vocab for repetition penalty.
239
+ cache = {'pos': 0, 'states': None, 'seen_mask': None}
240
+
241
+ while True:
242
+ if args.prompt:
243
+ user_input = args.prompt
244
+ else:
245
+ try:
246
+ user_input = input("\nUser: ").strip()
247
+ except (EOFError, KeyboardInterrupt):
248
+ print("\nGoodbye!")
249
+ break
250
+
251
+ if user_input.lower() in ('quit', 'exit'):
252
+ print("Goodbye!")
253
+ break
254
+ if user_input.lower() == 'clear':
255
+ conversation_tokens = [bos]
256
+ cache = {'pos': 0, 'states': None, 'seen_mask': None}
257
+ print("Conversation cleared.")
258
+ continue
259
+ if not user_input:
260
+ continue
261
+
262
+ if args.no_history:
263
+ conversation_tokens = [bos]
264
+ cache = {'pos': 0, 'states': None, 'seen_mask': None}
265
+
266
+ # Append user turn
267
+ conversation_tokens.append(user_s)
268
+ conversation_tokens.extend(tokenizer.encode(user_input).ids)
269
+ conversation_tokens.append(user_e)
270
+ # Open assistant turn (the model continues from here)
271
+ conversation_tokens.append(asst_s)
272
+
273
+ if not args.no_stream:
274
+ sys.stdout.write("\nAssistant: ")
275
+ sys.stdout.flush()
276
+ t0 = time.time()
277
+ response = _generate_one_turn(
278
+ model, tokenizer, conversation_tokens, specials,
279
+ max_new_tokens=args.max_tokens,
280
+ temperature=args.temperature,
281
+ top_k=args.top_k,
282
+ top_p=args.top_p,
283
+ repetition_penalty=args.repetition_penalty,
284
+ stream=not args.no_stream,
285
+ cache=cache,
286
+ )
287
+ elapsed = time.time() - t0
288
+
289
+ if args.no_stream:
290
+ print(f"\nAssistant: {tokenizer.decode(response)}")
291
+ n_resp = len(response)
292
+ print(f" [{n_resp} tok in {elapsed:.1f}s = {n_resp/max(elapsed, 1e-9):.1f} tok/s]",
293
+ file=sys.stderr)
294
+
295
+ # Close assistant turn in the history (so the next prefill sees a complete turn)
296
+ conversation_tokens.extend(response)
297
+ conversation_tokens.append(asst_e)
298
+
299
+ if args.prompt:
300
+ break
301
+
302
+
303
+ if __name__ == '__main__':
304
+ main()
infer.py ADDED
@@ -0,0 +1,309 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sample from a trained Stream Mixer GPT checkpoint.
2
+
3
+ Default sampling: temperature=0.8, top-k=50 (empirically tuned on the v4
4
+ checkpoint — see log.md post-completion section). Override with -t / -k.
5
+ Use -t 0 for deterministic greedy decoding (factual probes / debugging).
6
+
7
+ Usage:
8
+ python3 infer.py
9
+ python3 infer.py -p "In this article, we will explore" # default stochastic (0.8, k=50)
10
+ python3 infer.py -p "Photosynthesis is the process by which" # factual recall, encyclopedic continuation
11
+ python3 infer.py -p "The chemical symbol of gold is" -t 0 # greedy — for testing factual probes
12
+ python3 infer.py -p "Once upon a time," -t 0.9 -n 200 # higher temperature for narrative diversity
13
+ python3 infer.py -s 3 # 3 samples (batched in parallel)
14
+ python3 infer.py --ckpt model.pt --seed 42
15
+ python3 infer.py --compile # torch.compile (slow first iter, faster after)
16
+ python3 infer.py --compile --repl # amortize compile across many prompts
17
+
18
+ The model architecture is imported from model.py (shared with microgpt.py).
19
+ The checkpoint carries its own config so this script needs no hardcoded shapes.
20
+ """
21
+ import argparse
22
+ import os
23
+ import sys
24
+ import time
25
+
26
+ import torch
27
+ import torch.nn.functional as F
28
+ from tokenizers import Tokenizer
29
+
30
+ from model import GPT, COMPUTE_DTYPE
31
+
32
+ # Fallback config for checkpoints saved before microgpt.py started storing it.
33
+ FALLBACK_CONFIG = dict(n_embd=128, n_layer=4, n_streams=16, stream_dim=64)
34
+
35
+ # Default points at $CHECKPOINT_DIR/model.pt if set (matches microgpt.py's training
36
+ # output location); falls back to CWD. Override with --ckpt for explicit paths.
37
+ DEFAULT_CKPT = os.path.join(os.environ.get('CHECKPOINT_DIR', '.'), 'model.pt')
38
+ DEFAULT_TOKENIZER = 'tokenizer.json'
39
+
40
+ device = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
41
+
42
+
43
+ def _sync():
44
+ if device == 'mps':
45
+ torch.mps.synchronize()
46
+ elif device == 'cuda':
47
+ torch.cuda.synchronize()
48
+
49
+
50
+ def _sample_token_batch(logits, temperature, top_k, top_p, repetition_penalty, seen_mask):
51
+ """logits: (N, V). seen_mask: (N, V) bool — True at positions of tokens
52
+ already emitted in each sample (prompt + generated so far). Returns (N,) long
53
+ tensor of sampled token IDs.
54
+
55
+ Applies, in order:
56
+ 1. Repetition penalty (CTRL-style): for tokens marked in seen_mask, divide
57
+ positive logits / multiply negative logits by `repetition_penalty`.
58
+ penalty > 1 suppresses repeats; penalty == 1.0 is a no-op. Applied BEFORE
59
+ temperature so the suppression is in raw-logit space.
60
+ 2. Temperature scaling (skipped when temperature==0 → argmax greedy).
61
+ 3+4. Fused top-k / top-p filtering. With top-k set, top-p only needs to look
62
+ at the k candidates instead of sorting the full 32k vocab — big speedup.
63
+
64
+ Honors the --temperature 0 greedy contract — but rep penalty still applies
65
+ before argmax, so greedy + penalty > 1 is a valid "deterministic anti-loop"
66
+ decoding mode.
67
+ """
68
+ # 1) Repetition penalty — via fixed-size bool mask (constant cost per step,
69
+ # vs. gather/scatter over a growing index list which is slow on MPS).
70
+ if repetition_penalty != 1.0 and seen_mask is not None:
71
+ penalized = torch.where(logits > 0,
72
+ logits / repetition_penalty,
73
+ logits * repetition_penalty)
74
+ logits = torch.where(seen_mask, penalized, logits)
75
+
76
+ if temperature == 0:
77
+ return logits.argmax(dim=-1)
78
+ logits = logits / temperature
79
+
80
+ # 2+3) Fused top-k + top-p: when both are active, only sort the k candidates
81
+ # instead of the full 32k vocab. `topk(..., sorted=True)` already returns
82
+ # descending-ordered values, so we get the sort for free as a side effect
83
+ # of the top-k pass. Big speedup vs sorting the whole logits tensor.
84
+ if top_k > 0:
85
+ k = min(top_k, logits.size(-1))
86
+ topk_vals, topk_idx = logits.topk(k, dim=-1, sorted=True) # (N, k) desc
87
+ if top_p < 1.0:
88
+ # Nucleus over just the top-k — sort already done by topk()
89
+ topk_probs = F.softmax(topk_vals, dim=-1)
90
+ cumsum = topk_probs.cumsum(dim=-1)
91
+ mask = cumsum > top_p
92
+ mask[:, 1:] = mask[:, :-1].clone()
93
+ mask[:, 0] = False
94
+ topk_vals = topk_vals.masked_fill(mask, float('-inf'))
95
+ # Rebuild logits with only the kept candidates having real values
96
+ logits = torch.full_like(logits, float('-inf')).scatter_(1, topk_idx, topk_vals)
97
+ elif top_p < 1.0:
98
+ # No top-k → must sort the full vocab (slow path; avoid by always passing top_k)
99
+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
100
+ sorted_probs = F.softmax(sorted_logits, dim=-1)
101
+ cumsum = sorted_probs.cumsum(dim=-1)
102
+ sorted_mask = cumsum > top_p
103
+ sorted_mask[:, 1:] = sorted_mask[:, :-1].clone()
104
+ sorted_mask[:, 0] = False
105
+ mask = torch.zeros_like(sorted_mask).scatter(1, sorted_indices, sorted_mask)
106
+ logits = logits.masked_fill(mask, float('-inf'))
107
+
108
+ probs = F.softmax(logits, dim=-1)
109
+ return torch.multinomial(probs, 1).squeeze(-1) # (N,)
110
+
111
+
112
+ @torch.no_grad()
113
+ def generate(model, tokenizer, BOS, prompt, n_samples=1,
114
+ max_new_tokens=256, temperature=0.8, top_k=50, top_p=1.0,
115
+ repetition_penalty=1.0, stream=True):
116
+ """Batched generation. All N samples advance one token per MPS/CUDA step.
117
+
118
+ Returns list of N decoded strings. Streams to stdout only when n_samples==1
119
+ (multi-sample streaming would interleave illegibly).
120
+
121
+ For long generations (>500 tokens) where the model loops, raise
122
+ repetition_penalty to 1.1–1.2 or lower top_p to ~0.9.
123
+ """
124
+ prompt_ids = [BOS] + tokenizer.encode(prompt).ids
125
+ ctx = torch.tensor([prompt_ids] * n_samples, device=device) # (N, T_prompt)
126
+ # Repetition-penalty memory: a (N, V) bool mask, True where the corresponding
127
+ # token id has already been emitted. Constant cost per step regardless of
128
+ # generation length — beats a growing token-list approach on MPS where
129
+ # gather/scatter over scattered indices is slow.
130
+ V = None # filled in after first forward (we don't know it yet)
131
+ seen_mask = None
132
+
133
+ t0 = time.time()
134
+ logits, states = model.forward_with_states(ctx)
135
+ next_logits = logits[:, -1, :].clone() # (N, V); clone so compile doesn't see a view
136
+ # Now we know V; build the mask and seed it with the prompt tokens.
137
+ V = next_logits.size(-1)
138
+ if repetition_penalty != 1.0:
139
+ seen_mask = torch.zeros(n_samples, V, dtype=torch.bool, device=device)
140
+ seen_mask.scatter_(1, ctx, True)
141
+ _sync()
142
+ t_prefill = time.time()
143
+
144
+ can_stream = stream and n_samples == 1
145
+ if can_stream:
146
+ print(prompt, end='', flush=True)
147
+
148
+ generated = [[] for _ in range(n_samples)]
149
+ done = [False] * n_samples
150
+
151
+ for _ in range(max_new_tokens):
152
+ next_toks = _sample_token_batch(next_logits, temperature, top_k, top_p,
153
+ repetition_penalty, seen_mask) # (N,)
154
+ next_toks_list = next_toks.tolist()
155
+ any_alive = False
156
+ for i in range(n_samples):
157
+ if done[i]:
158
+ continue
159
+ tid = next_toks_list[i]
160
+ if tid == BOS:
161
+ done[i] = True
162
+ else:
163
+ generated[i].append(tid)
164
+ any_alive = True
165
+ if not any_alive:
166
+ break
167
+ if can_stream:
168
+ # Per-token decode is O(1); avoids the O(N²) full-list re-decode each step.
169
+ # ByteLevel BPE pieces decode cleanly for ASCII English, which dominates FineWeb-Edu.
170
+ piece = tokenizer.decode([next_toks_list[0]])
171
+ sys.stdout.write(piece)
172
+ sys.stdout.flush()
173
+ # Mark the new token as seen (O(N) scatter — constant per step).
174
+ if seen_mask is not None:
175
+ seen_mask.scatter_(1, next_toks.unsqueeze(1), True)
176
+ x_in = next_toks.unsqueeze(1) # (N, 1)
177
+ step_logits, states = model.step(x_in, states)
178
+ next_logits = step_logits[:, 0, :]
179
+
180
+ _sync()
181
+ t_end = time.time()
182
+ if can_stream:
183
+ print()
184
+
185
+ total_n = sum(len(g) for g in generated)
186
+ prefill_ms = (t_prefill - t0) * 1000
187
+ decode_s = max(t_end - t_prefill, 1e-9)
188
+ print(f" [prefill {len(prompt_ids)}×{n_samples} tok in {prefill_ms:.0f}ms | "
189
+ f"decode {total_n} tok in {decode_s:.2f}s = {total_n/decode_s:.1f} tok/s aggregate]",
190
+ file=sys.stderr)
191
+ return [tokenizer.decode(g) for g in generated]
192
+
193
+
194
+ def load_model(ckpt_path, vocab_size, compile_model=False):
195
+ ckpt = torch.load(ckpt_path, map_location=device)
196
+ if 'config' in ckpt:
197
+ config = dict(ckpt['config'])
198
+ else:
199
+ print(f"note: checkpoint has no 'config' (older format); using fallback {FALLBACK_CONFIG}",
200
+ file=sys.stderr)
201
+ config = dict(FALLBACK_CONFIG)
202
+ config['vocab_size'] = vocab_size # vocab is set by the tokenizer, not the checkpoint
203
+ model = GPT.from_config(config).to(device)
204
+ # Strip torch.compile's `_orig_mod.` prefix if present in older checkpoints.
205
+ state = {k.removeprefix('_orig_mod.'): v for k, v in ckpt['model'].items()}
206
+ missing, unexpected = model.load_state_dict(state, strict=False)
207
+ if missing: print(f"warn: missing keys: {missing}", file=sys.stderr)
208
+ if unexpected: print(f"warn: unexpected keys: {unexpected}", file=sys.stderr)
209
+ model.eval()
210
+ if compile_model:
211
+ print("compiling step/prefill paths...", file=sys.stderr)
212
+ model.step = torch.compile(model.step, dynamic=False)
213
+ model.forward_with_states = torch.compile(model.forward_with_states, dynamic=False)
214
+ return model, ckpt, config
215
+
216
+
217
+ def main():
218
+ p = argparse.ArgumentParser(description="Sample from a trained microgpt Stream Mixer.")
219
+ p.add_argument('-p', '--prompt', type=str, default="In this article, we will explore")
220
+ p.add_argument('-n', '--max-tokens', type=int, default=256)
221
+ p.add_argument('-t', '--temperature', type=float, default=0.8,
222
+ help='softmax temperature; 0 = greedy argmax. Recommended 0.7–0.9 for prose. '
223
+ '(Determined empirically — see v4 log.md post-completion section.)')
224
+ p.add_argument('-k', '--top-k', type=int, default=50,
225
+ help='restrict sampling to top-k logits; 0 disables. '
226
+ 'Recommended 50 — keeps the long tail of the 32k vocab from injecting weird tokens.')
227
+ p.add_argument('--top-p', type=float, default=1.0,
228
+ help='nucleus sampling: keep the smallest token set whose cumulative '
229
+ 'probability ≥ top-p. 1.0 disables. Recommended 0.9 for long generations.')
230
+ p.add_argument('-r', '--repetition-penalty', type=float, default=1.0,
231
+ help='CTRL-style repetition penalty: tokens already in the context get '
232
+ 'their logits divided by this. 1.0 = no penalty. Recommended 1.1–1.2 '
233
+ 'for >500-token generations to break out of repetition wells.')
234
+ p.add_argument('-s', '--num-samples', type=int, default=1)
235
+ p.add_argument('--ckpt', type=str, default=DEFAULT_CKPT)
236
+ p.add_argument('--tokenizer', type=str, default=DEFAULT_TOKENIZER)
237
+ p.add_argument('--seed', type=int, default=None)
238
+ p.add_argument('--no-stream', action='store_true',
239
+ help='print full output at end instead of streaming token by token')
240
+ p.add_argument('--compile', action='store_true',
241
+ help='torch.compile the step/prefill paths (slower first call, faster after)')
242
+ p.add_argument('--repl', action='store_true',
243
+ help='after generating, read more prompts from stdin so --compile is paid once')
244
+ args = p.parse_args()
245
+
246
+ if args.seed is not None:
247
+ torch.manual_seed(args.seed)
248
+ if not os.path.exists(args.ckpt):
249
+ sys.exit(f"error: checkpoint not found: {args.ckpt}")
250
+ if not os.path.exists(args.tokenizer):
251
+ sys.exit(f"error: tokenizer not found: {args.tokenizer}")
252
+
253
+ print(f"device: {device}", file=sys.stderr)
254
+ tokenizer = Tokenizer.from_file(args.tokenizer)
255
+ BOS = tokenizer.token_to_id("<|bos|>")
256
+
257
+ model, ckpt, config = load_model(args.ckpt, tokenizer.get_vocab_size(),
258
+ compile_model=args.compile)
259
+ step = ckpt.get('step', '?')
260
+ best = ckpt.get('best_loss')
261
+ n_params = sum(p.numel() for p in model.parameters())
262
+ best_str = f" best_loss={best:.4f}" if isinstance(best, float) else ""
263
+ print(f"loaded {args.ckpt} step={step}{best_str}", file=sys.stderr)
264
+ print(f"config: {config}", file=sys.stderr)
265
+ print(f"params: {n_params:,}\n", file=sys.stderr)
266
+
267
+ stream = not args.no_stream
268
+
269
+ def run(prompt):
270
+ results = generate(model, tokenizer, BOS, prompt,
271
+ n_samples=args.num_samples,
272
+ max_new_tokens=args.max_tokens,
273
+ temperature=args.temperature,
274
+ top_k=args.top_k,
275
+ top_p=args.top_p,
276
+ repetition_penalty=args.repetition_penalty,
277
+ stream=stream)
278
+ # Single-sample streaming already printed; otherwise print collected results.
279
+ if not (stream and args.num_samples == 1):
280
+ for i, out in enumerate(results):
281
+ if args.num_samples > 1:
282
+ print(f"--- sample {i+1}/{args.num_samples} ---", file=sys.stderr)
283
+ print(prompt + out)
284
+ if args.num_samples > 1:
285
+ print(file=sys.stderr)
286
+
287
+ run(args.prompt)
288
+
289
+ # REPL: amortize torch.compile warmup over many prompts. Empty input
290
+ # reuses the previous prompt; "exit"/"quit"/Ctrl-D leaves the loop.
291
+ if args.repl:
292
+ last_prompt = args.prompt
293
+ print("\n[repl] enter prompt (empty=repeat, exit/quit/Ctrl-D to leave)", file=sys.stderr)
294
+ while True:
295
+ try:
296
+ line = input(">>> ")
297
+ except (EOFError, KeyboardInterrupt):
298
+ print(file=sys.stderr)
299
+ break
300
+ line = line.strip()
301
+ if line in ('exit', 'quit'):
302
+ break
303
+ prompt = line if line else last_prompt
304
+ last_prompt = prompt
305
+ run(prompt)
306
+
307
+
308
+ if __name__ == '__main__':
309
+ main()