Upload 2 files
Browse files- chat_cli.py +304 -0
- infer.py +309 -0
chat_cli.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Chat with the SFT'd microgpt model. Adapted from karpathy/nanochat's chat_cli.py.
|
| 2 |
+
|
| 3 |
+
Builds chat-format prompts using our 5 special tokens:
|
| 4 |
+
<|bos|> <|user_start|> ...user text... <|user_end|>
|
| 5 |
+
<|assistant_start|> ...assistant text... <|assistant_end|>
|
| 6 |
+
|
| 7 |
+
Conversation history is maintained across turns (each new user message is appended
|
| 8 |
+
to the running token stream). Generation stops on <|assistant_end|>.
|
| 9 |
+
|
| 10 |
+
Usage:
|
| 11 |
+
python3 chat_cli.py # interactive REPL
|
| 12 |
+
python3 chat_cli.py -p "Who are you?" # one-shot
|
| 13 |
+
python3 chat_cli.py --ckpt model.pt # explicit checkpoint path
|
| 14 |
+
python3 chat_cli.py -t 0.6 -k 50 # nanochat-CLI defaults
|
| 15 |
+
python3 chat_cli.py --no-history # reset per turn (no memory)
|
| 16 |
+
|
| 17 |
+
Defaults follow nanochat-CLI (T=0.6, top-k=50). Lower temperature than infer.py's
|
| 18 |
+
default 0.8 because chat is meant to be focused, not exploratory.
|
| 19 |
+
|
| 20 |
+
REPL commands:
|
| 21 |
+
quit / exit end the session
|
| 22 |
+
clear reset conversation history
|
| 23 |
+
"""
|
| 24 |
+
import argparse
|
| 25 |
+
import os
|
| 26 |
+
import sys
|
| 27 |
+
import time
|
| 28 |
+
|
| 29 |
+
import torch
|
| 30 |
+
from tokenizers import Tokenizer
|
| 31 |
+
|
| 32 |
+
from model import GPT
|
| 33 |
+
from infer import _sample_token_batch # reuse our sampler (rep penalty + nucleus + top-k)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
DEFAULT_CKPT_DIR = os.environ.get('CHECKPOINT_DIR', '.')
|
| 37 |
+
# Prefer model_sft.pt (local dev convention with both base + SFT present), fall
|
| 38 |
+
# back to model.pt (HF convention — only the SFT'd model is uploaded as model.pt).
|
| 39 |
+
DEFAULT_CKPT = os.path.join(DEFAULT_CKPT_DIR, 'model_sft.pt')
|
| 40 |
+
DEFAULT_TOKENIZER = 'tokenizer.json'
|
| 41 |
+
|
| 42 |
+
SPECIAL_NAMES = [
|
| 43 |
+
"<|bos|>",
|
| 44 |
+
"<|user_start|>", "<|user_end|>",
|
| 45 |
+
"<|assistant_start|>", "<|assistant_end|>",
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
device = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _load(ckpt_path, tokenizer_path, compile_step=False):
|
| 52 |
+
"""Resolve checkpoint (with fallback), load model + tokenizer + special token IDs.
|
| 53 |
+
|
| 54 |
+
If `compile_step=True`, torch.compile the inner-loop `model.step` (B=1, T=1 fixed
|
| 55 |
+
shapes — perfect for compile). We deliberately don't compile `forward_with_states`
|
| 56 |
+
because chat conversations grow each turn → dynamic prompt length → recompile
|
| 57 |
+
every turn. Prefill stays eager.
|
| 58 |
+
|
| 59 |
+
Compile warmup takes 10–30s on first generation; pays off for any conversation
|
| 60 |
+
long enough to generate >~50 tokens. On MPS, torch.compile is less mature than
|
| 61 |
+
CUDA — try it but don't be surprised if it falls back to eager.
|
| 62 |
+
"""
|
| 63 |
+
if not os.path.exists(ckpt_path):
|
| 64 |
+
# Silent fallback for HF layout (only model.pt = the SFT'd one)
|
| 65 |
+
fallback = os.path.join(DEFAULT_CKPT_DIR, 'model.pt')
|
| 66 |
+
if ckpt_path != fallback and os.path.exists(fallback):
|
| 67 |
+
ckpt_path = fallback
|
| 68 |
+
else:
|
| 69 |
+
sys.exit(f"error: no checkpoint at {ckpt_path}")
|
| 70 |
+
if not os.path.exists(tokenizer_path):
|
| 71 |
+
sys.exit(f"error: no tokenizer at {tokenizer_path}")
|
| 72 |
+
|
| 73 |
+
tokenizer = Tokenizer.from_file(tokenizer_path)
|
| 74 |
+
vocab_size = tokenizer.get_vocab_size()
|
| 75 |
+
|
| 76 |
+
ckpt = torch.load(ckpt_path, map_location=device)
|
| 77 |
+
config = dict(ckpt['config'])
|
| 78 |
+
config['vocab_size'] = ((vocab_size + 63) // 64) * 64
|
| 79 |
+
model = GPT.from_config(config).to(device)
|
| 80 |
+
state = {k.removeprefix('_orig_mod.'): v for k, v in ckpt['model'].items()}
|
| 81 |
+
missing, unexpected = model.load_state_dict(state, strict=False)
|
| 82 |
+
if missing: print(f"warn: missing keys: {missing}", file=sys.stderr)
|
| 83 |
+
if unexpected: print(f"warn: unexpected keys: {unexpected}", file=sys.stderr)
|
| 84 |
+
model.eval()
|
| 85 |
+
|
| 86 |
+
specials = {}
|
| 87 |
+
for name in SPECIAL_NAMES:
|
| 88 |
+
tid = tokenizer.token_to_id(name)
|
| 89 |
+
if tid is None:
|
| 90 |
+
sys.exit(f"error: tokenizer is missing special token {name} — was it retrained without the SFT vocab?")
|
| 91 |
+
specials[name] = tid
|
| 92 |
+
|
| 93 |
+
if compile_step:
|
| 94 |
+
print("compiling model.step (hot inner-loop path)...", file=sys.stderr)
|
| 95 |
+
if device == 'mps':
|
| 96 |
+
print(" note: torch.compile on MPS may fall back to eager. CUDA gets the full speedup.",
|
| 97 |
+
file=sys.stderr)
|
| 98 |
+
compile_kwargs = {'dynamic': False}
|
| 99 |
+
# reduce-overhead enables CUDA graphs — big win on Ampere+ where kernel-launch
|
| 100 |
+
# overhead dominates the small per-token forward. No-op on MPS/CPU.
|
| 101 |
+
if device == 'cuda' and torch.cuda.get_device_capability() >= (8, 0):
|
| 102 |
+
compile_kwargs['mode'] = 'reduce-overhead'
|
| 103 |
+
model.step = torch.compile(model.step, **compile_kwargs)
|
| 104 |
+
|
| 105 |
+
return model, tokenizer, specials, ckpt, ckpt_path
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@torch.no_grad()
|
| 109 |
+
def _generate_one_turn(model, tokenizer, conversation_tokens, specials,
|
| 110 |
+
max_new_tokens, temperature, top_k, top_p, repetition_penalty,
|
| 111 |
+
stream, cache):
|
| 112 |
+
"""Run prefill over the NEW tokens since `cache['pos']` (using cached state),
|
| 113 |
+
then sample tokens until <|assistant_end|> (or <|bos|>, or max_new_tokens).
|
| 114 |
+
Returns the response token list. Mutates `cache` to extend coverage to the
|
| 115 |
+
end of conversation_tokens + the assistant_end that will be appended.
|
| 116 |
+
|
| 117 |
+
`cache` is a dict { 'pos': int, 'states': list[(B,M,D)] | None, 'seen_mask': bool tensor | None }.
|
| 118 |
+
On first call, `cache['states']` is None — we run prefill from scratch.
|
| 119 |
+
On subsequent calls, only the new tokens since `cache['pos']` are prefilled,
|
| 120 |
+
starting from `cache['states']`. Big speedup once the conversation grows.
|
| 121 |
+
"""
|
| 122 |
+
bos = specials["<|bos|>"]
|
| 123 |
+
asst_end = specials["<|assistant_end|>"]
|
| 124 |
+
V = model.lm_head.weight.size(0) # padded vocab
|
| 125 |
+
|
| 126 |
+
# Slice off only the new tokens to prefill
|
| 127 |
+
new_tokens = conversation_tokens[cache['pos']:]
|
| 128 |
+
assert len(new_tokens) > 0, "no new tokens to prefill — cache invariant broken"
|
| 129 |
+
new_ctx = torch.tensor([new_tokens], device=device) # (1, T_new)
|
| 130 |
+
|
| 131 |
+
# Initialize / extend the seen_mask (used for repetition penalty)
|
| 132 |
+
if repetition_penalty != 1.0:
|
| 133 |
+
if cache.get('seen_mask') is None:
|
| 134 |
+
cache['seen_mask'] = torch.zeros(1, V, dtype=torch.bool, device=device)
|
| 135 |
+
cache['seen_mask'].scatter_(1, new_ctx, True)
|
| 136 |
+
seen_mask = cache.get('seen_mask')
|
| 137 |
+
|
| 138 |
+
# Prefill only the new tokens, starting from cached states (or zeros on first turn)
|
| 139 |
+
logits, states = model.forward_with_states(new_ctx, initial_states=cache['states'])
|
| 140 |
+
next_logits = logits[:, -1, :].clone()
|
| 141 |
+
|
| 142 |
+
response = []
|
| 143 |
+
for _ in range(max_new_tokens):
|
| 144 |
+
next_tok = _sample_token_batch(next_logits, temperature, top_k, top_p,
|
| 145 |
+
repetition_penalty, seen_mask) # (1,)
|
| 146 |
+
tid = next_tok.item()
|
| 147 |
+
if tid == asst_end or tid == bos:
|
| 148 |
+
break
|
| 149 |
+
response.append(tid)
|
| 150 |
+
if stream:
|
| 151 |
+
# Per-token decode is fine for ASCII English (the ClimbMix corpus dominates that).
|
| 152 |
+
piece = tokenizer.decode([tid])
|
| 153 |
+
sys.stdout.write(piece)
|
| 154 |
+
sys.stdout.flush()
|
| 155 |
+
if seen_mask is not None:
|
| 156 |
+
seen_mask.scatter_(1, next_tok.unsqueeze(1), True)
|
| 157 |
+
step_logits, states = model.step(next_tok.view(1, 1), states)
|
| 158 |
+
next_logits = step_logits[:, 0, :]
|
| 159 |
+
|
| 160 |
+
if stream:
|
| 161 |
+
print()
|
| 162 |
+
|
| 163 |
+
# Advance state through assistant_end so the cache reflects the closed turn.
|
| 164 |
+
# (Generation stopped at either an emitted assistant_end, BOS, or max_tokens;
|
| 165 |
+
# in the latter two cases we still need to step state through assistant_end.)
|
| 166 |
+
asst_end_tok = torch.tensor([[asst_end]], device=device)
|
| 167 |
+
_, states = model.step(asst_end_tok, states)
|
| 168 |
+
if seen_mask is not None:
|
| 169 |
+
seen_mask.scatter_(1, asst_end_tok, True)
|
| 170 |
+
|
| 171 |
+
# Update cache: state now reflects [prev cache] + new prefill tokens + generated response + assistant_end
|
| 172 |
+
cache['states'] = states
|
| 173 |
+
cache['pos'] = len(conversation_tokens) + len(response) + 1 # +1 for assistant_end
|
| 174 |
+
|
| 175 |
+
return response
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def main():
|
| 179 |
+
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 180 |
+
p.add_argument('-p', '--prompt', type=str, default='',
|
| 181 |
+
help='single-turn prompt — exits after one response. Empty = interactive REPL.')
|
| 182 |
+
p.add_argument('-t', '--temperature', type=float, default=0.6,
|
| 183 |
+
help='softmax temperature; 0 = greedy. nanochat-CLI default 0.6 — '
|
| 184 |
+
'tighter than infer.py because chat should be focused.')
|
| 185 |
+
p.add_argument('-k', '--top-k', type=int, default=50,
|
| 186 |
+
help='top-k sampling; 0 disables. Default 50 (nanochat-CLI).')
|
| 187 |
+
p.add_argument('--top-p', type=float, default=1.0,
|
| 188 |
+
help='nucleus sampling threshold; 1.0 disables. Try 0.9 for varied responses.')
|
| 189 |
+
p.add_argument('-r', '--repetition-penalty', type=float, default=1.15,
|
| 190 |
+
help='CTRL-style repetition penalty (default 1.15) — keeps chat responses '
|
| 191 |
+
'from looping. Set to 1.0 for raw sampling.')
|
| 192 |
+
p.add_argument('-n', '--max-tokens', type=int, default=256,
|
| 193 |
+
help='max tokens per assistant response')
|
| 194 |
+
p.add_argument('--ckpt', type=str, default=DEFAULT_CKPT,
|
| 195 |
+
help='checkpoint path (default $CHECKPOINT_DIR/model.pt)')
|
| 196 |
+
p.add_argument('--tokenizer', type=str, default=DEFAULT_TOKENIZER)
|
| 197 |
+
p.add_argument('--no-history', action='store_true',
|
| 198 |
+
help='reset conversation history before each turn (model has no memory)')
|
| 199 |
+
p.add_argument('--seed', type=int, default=None)
|
| 200 |
+
p.add_argument('--no-stream', action='store_true',
|
| 201 |
+
help='print full response at end instead of token-by-token')
|
| 202 |
+
p.add_argument('--compile', action='store_true',
|
| 203 |
+
help='torch.compile the inner step() path. First generation pays '
|
| 204 |
+
'~10–30s warmup; subsequent generations are 2–5× faster on CUDA. '
|
| 205 |
+
'Best for long REPL sessions; skip for one-shots.')
|
| 206 |
+
args = p.parse_args()
|
| 207 |
+
|
| 208 |
+
if args.seed is not None:
|
| 209 |
+
torch.manual_seed(args.seed)
|
| 210 |
+
|
| 211 |
+
print(f"device: {device}", file=sys.stderr)
|
| 212 |
+
model, tokenizer, specials, ckpt, used_ckpt = _load(args.ckpt, args.tokenizer,
|
| 213 |
+
compile_step=args.compile)
|
| 214 |
+
step = ckpt.get('step', '?')
|
| 215 |
+
best = ckpt.get('best_loss')
|
| 216 |
+
n_params = sum(t.numel() for t in model.parameters())
|
| 217 |
+
best_str = f" best_loss={best:.4f}" if isinstance(best, float) else ""
|
| 218 |
+
print(f"loaded {used_ckpt} step={step}{best_str} params={n_params:,}", file=sys.stderr)
|
| 219 |
+
|
| 220 |
+
bos = specials["<|bos|>"]
|
| 221 |
+
user_s = specials["<|user_start|>"]
|
| 222 |
+
user_e = specials["<|user_end|>"]
|
| 223 |
+
asst_s = specials["<|assistant_start|>"]
|
| 224 |
+
asst_e = specials["<|assistant_end|>"]
|
| 225 |
+
|
| 226 |
+
print()
|
| 227 |
+
print("Mnemo — chat mode")
|
| 228 |
+
print("-" * 50)
|
| 229 |
+
print(f"sampling: T={args.temperature} top_k={args.top_k} top_p={args.top_p} rep_penalty={args.repetition_penalty}")
|
| 230 |
+
if not args.prompt:
|
| 231 |
+
print("commands: 'quit' / 'exit' to end, 'clear' to reset history")
|
| 232 |
+
print("-" * 50)
|
| 233 |
+
|
| 234 |
+
conversation_tokens = [bos]
|
| 235 |
+
# State cache: skips re-prefilling already-processed turns.
|
| 236 |
+
# 'pos' = how far into conversation_tokens the cached state covers.
|
| 237 |
+
# 'states' = per-layer (B,M,D) states, None on cold start.
|
| 238 |
+
# 'seen_mask' = (1,V) bool over the vocab for repetition penalty.
|
| 239 |
+
cache = {'pos': 0, 'states': None, 'seen_mask': None}
|
| 240 |
+
|
| 241 |
+
while True:
|
| 242 |
+
if args.prompt:
|
| 243 |
+
user_input = args.prompt
|
| 244 |
+
else:
|
| 245 |
+
try:
|
| 246 |
+
user_input = input("\nUser: ").strip()
|
| 247 |
+
except (EOFError, KeyboardInterrupt):
|
| 248 |
+
print("\nGoodbye!")
|
| 249 |
+
break
|
| 250 |
+
|
| 251 |
+
if user_input.lower() in ('quit', 'exit'):
|
| 252 |
+
print("Goodbye!")
|
| 253 |
+
break
|
| 254 |
+
if user_input.lower() == 'clear':
|
| 255 |
+
conversation_tokens = [bos]
|
| 256 |
+
cache = {'pos': 0, 'states': None, 'seen_mask': None}
|
| 257 |
+
print("Conversation cleared.")
|
| 258 |
+
continue
|
| 259 |
+
if not user_input:
|
| 260 |
+
continue
|
| 261 |
+
|
| 262 |
+
if args.no_history:
|
| 263 |
+
conversation_tokens = [bos]
|
| 264 |
+
cache = {'pos': 0, 'states': None, 'seen_mask': None}
|
| 265 |
+
|
| 266 |
+
# Append user turn
|
| 267 |
+
conversation_tokens.append(user_s)
|
| 268 |
+
conversation_tokens.extend(tokenizer.encode(user_input).ids)
|
| 269 |
+
conversation_tokens.append(user_e)
|
| 270 |
+
# Open assistant turn (the model continues from here)
|
| 271 |
+
conversation_tokens.append(asst_s)
|
| 272 |
+
|
| 273 |
+
if not args.no_stream:
|
| 274 |
+
sys.stdout.write("\nAssistant: ")
|
| 275 |
+
sys.stdout.flush()
|
| 276 |
+
t0 = time.time()
|
| 277 |
+
response = _generate_one_turn(
|
| 278 |
+
model, tokenizer, conversation_tokens, specials,
|
| 279 |
+
max_new_tokens=args.max_tokens,
|
| 280 |
+
temperature=args.temperature,
|
| 281 |
+
top_k=args.top_k,
|
| 282 |
+
top_p=args.top_p,
|
| 283 |
+
repetition_penalty=args.repetition_penalty,
|
| 284 |
+
stream=not args.no_stream,
|
| 285 |
+
cache=cache,
|
| 286 |
+
)
|
| 287 |
+
elapsed = time.time() - t0
|
| 288 |
+
|
| 289 |
+
if args.no_stream:
|
| 290 |
+
print(f"\nAssistant: {tokenizer.decode(response)}")
|
| 291 |
+
n_resp = len(response)
|
| 292 |
+
print(f" [{n_resp} tok in {elapsed:.1f}s = {n_resp/max(elapsed, 1e-9):.1f} tok/s]",
|
| 293 |
+
file=sys.stderr)
|
| 294 |
+
|
| 295 |
+
# Close assistant turn in the history (so the next prefill sees a complete turn)
|
| 296 |
+
conversation_tokens.extend(response)
|
| 297 |
+
conversation_tokens.append(asst_e)
|
| 298 |
+
|
| 299 |
+
if args.prompt:
|
| 300 |
+
break
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
if __name__ == '__main__':
|
| 304 |
+
main()
|
infer.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sample from a trained Stream Mixer GPT checkpoint.
|
| 2 |
+
|
| 3 |
+
Default sampling: temperature=0.8, top-k=50 (empirically tuned on the v4
|
| 4 |
+
checkpoint — see log.md post-completion section). Override with -t / -k.
|
| 5 |
+
Use -t 0 for deterministic greedy decoding (factual probes / debugging).
|
| 6 |
+
|
| 7 |
+
Usage:
|
| 8 |
+
python3 infer.py
|
| 9 |
+
python3 infer.py -p "In this article, we will explore" # default stochastic (0.8, k=50)
|
| 10 |
+
python3 infer.py -p "Photosynthesis is the process by which" # factual recall, encyclopedic continuation
|
| 11 |
+
python3 infer.py -p "The chemical symbol of gold is" -t 0 # greedy — for testing factual probes
|
| 12 |
+
python3 infer.py -p "Once upon a time," -t 0.9 -n 200 # higher temperature for narrative diversity
|
| 13 |
+
python3 infer.py -s 3 # 3 samples (batched in parallel)
|
| 14 |
+
python3 infer.py --ckpt model.pt --seed 42
|
| 15 |
+
python3 infer.py --compile # torch.compile (slow first iter, faster after)
|
| 16 |
+
python3 infer.py --compile --repl # amortize compile across many prompts
|
| 17 |
+
|
| 18 |
+
The model architecture is imported from model.py (shared with microgpt.py).
|
| 19 |
+
The checkpoint carries its own config so this script needs no hardcoded shapes.
|
| 20 |
+
"""
|
| 21 |
+
import argparse
|
| 22 |
+
import os
|
| 23 |
+
import sys
|
| 24 |
+
import time
|
| 25 |
+
|
| 26 |
+
import torch
|
| 27 |
+
import torch.nn.functional as F
|
| 28 |
+
from tokenizers import Tokenizer
|
| 29 |
+
|
| 30 |
+
from model import GPT, COMPUTE_DTYPE
|
| 31 |
+
|
| 32 |
+
# Fallback config for checkpoints saved before microgpt.py started storing it.
|
| 33 |
+
FALLBACK_CONFIG = dict(n_embd=128, n_layer=4, n_streams=16, stream_dim=64)
|
| 34 |
+
|
| 35 |
+
# Default points at $CHECKPOINT_DIR/model.pt if set (matches microgpt.py's training
|
| 36 |
+
# output location); falls back to CWD. Override with --ckpt for explicit paths.
|
| 37 |
+
DEFAULT_CKPT = os.path.join(os.environ.get('CHECKPOINT_DIR', '.'), 'model.pt')
|
| 38 |
+
DEFAULT_TOKENIZER = 'tokenizer.json'
|
| 39 |
+
|
| 40 |
+
device = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _sync():
|
| 44 |
+
if device == 'mps':
|
| 45 |
+
torch.mps.synchronize()
|
| 46 |
+
elif device == 'cuda':
|
| 47 |
+
torch.cuda.synchronize()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _sample_token_batch(logits, temperature, top_k, top_p, repetition_penalty, seen_mask):
|
| 51 |
+
"""logits: (N, V). seen_mask: (N, V) bool — True at positions of tokens
|
| 52 |
+
already emitted in each sample (prompt + generated so far). Returns (N,) long
|
| 53 |
+
tensor of sampled token IDs.
|
| 54 |
+
|
| 55 |
+
Applies, in order:
|
| 56 |
+
1. Repetition penalty (CTRL-style): for tokens marked in seen_mask, divide
|
| 57 |
+
positive logits / multiply negative logits by `repetition_penalty`.
|
| 58 |
+
penalty > 1 suppresses repeats; penalty == 1.0 is a no-op. Applied BEFORE
|
| 59 |
+
temperature so the suppression is in raw-logit space.
|
| 60 |
+
2. Temperature scaling (skipped when temperature==0 → argmax greedy).
|
| 61 |
+
3+4. Fused top-k / top-p filtering. With top-k set, top-p only needs to look
|
| 62 |
+
at the k candidates instead of sorting the full 32k vocab — big speedup.
|
| 63 |
+
|
| 64 |
+
Honors the --temperature 0 greedy contract — but rep penalty still applies
|
| 65 |
+
before argmax, so greedy + penalty > 1 is a valid "deterministic anti-loop"
|
| 66 |
+
decoding mode.
|
| 67 |
+
"""
|
| 68 |
+
# 1) Repetition penalty — via fixed-size bool mask (constant cost per step,
|
| 69 |
+
# vs. gather/scatter over a growing index list which is slow on MPS).
|
| 70 |
+
if repetition_penalty != 1.0 and seen_mask is not None:
|
| 71 |
+
penalized = torch.where(logits > 0,
|
| 72 |
+
logits / repetition_penalty,
|
| 73 |
+
logits * repetition_penalty)
|
| 74 |
+
logits = torch.where(seen_mask, penalized, logits)
|
| 75 |
+
|
| 76 |
+
if temperature == 0:
|
| 77 |
+
return logits.argmax(dim=-1)
|
| 78 |
+
logits = logits / temperature
|
| 79 |
+
|
| 80 |
+
# 2+3) Fused top-k + top-p: when both are active, only sort the k candidates
|
| 81 |
+
# instead of the full 32k vocab. `topk(..., sorted=True)` already returns
|
| 82 |
+
# descending-ordered values, so we get the sort for free as a side effect
|
| 83 |
+
# of the top-k pass. Big speedup vs sorting the whole logits tensor.
|
| 84 |
+
if top_k > 0:
|
| 85 |
+
k = min(top_k, logits.size(-1))
|
| 86 |
+
topk_vals, topk_idx = logits.topk(k, dim=-1, sorted=True) # (N, k) desc
|
| 87 |
+
if top_p < 1.0:
|
| 88 |
+
# Nucleus over just the top-k — sort already done by topk()
|
| 89 |
+
topk_probs = F.softmax(topk_vals, dim=-1)
|
| 90 |
+
cumsum = topk_probs.cumsum(dim=-1)
|
| 91 |
+
mask = cumsum > top_p
|
| 92 |
+
mask[:, 1:] = mask[:, :-1].clone()
|
| 93 |
+
mask[:, 0] = False
|
| 94 |
+
topk_vals = topk_vals.masked_fill(mask, float('-inf'))
|
| 95 |
+
# Rebuild logits with only the kept candidates having real values
|
| 96 |
+
logits = torch.full_like(logits, float('-inf')).scatter_(1, topk_idx, topk_vals)
|
| 97 |
+
elif top_p < 1.0:
|
| 98 |
+
# No top-k → must sort the full vocab (slow path; avoid by always passing top_k)
|
| 99 |
+
sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
|
| 100 |
+
sorted_probs = F.softmax(sorted_logits, dim=-1)
|
| 101 |
+
cumsum = sorted_probs.cumsum(dim=-1)
|
| 102 |
+
sorted_mask = cumsum > top_p
|
| 103 |
+
sorted_mask[:, 1:] = sorted_mask[:, :-1].clone()
|
| 104 |
+
sorted_mask[:, 0] = False
|
| 105 |
+
mask = torch.zeros_like(sorted_mask).scatter(1, sorted_indices, sorted_mask)
|
| 106 |
+
logits = logits.masked_fill(mask, float('-inf'))
|
| 107 |
+
|
| 108 |
+
probs = F.softmax(logits, dim=-1)
|
| 109 |
+
return torch.multinomial(probs, 1).squeeze(-1) # (N,)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
@torch.no_grad()
|
| 113 |
+
def generate(model, tokenizer, BOS, prompt, n_samples=1,
|
| 114 |
+
max_new_tokens=256, temperature=0.8, top_k=50, top_p=1.0,
|
| 115 |
+
repetition_penalty=1.0, stream=True):
|
| 116 |
+
"""Batched generation. All N samples advance one token per MPS/CUDA step.
|
| 117 |
+
|
| 118 |
+
Returns list of N decoded strings. Streams to stdout only when n_samples==1
|
| 119 |
+
(multi-sample streaming would interleave illegibly).
|
| 120 |
+
|
| 121 |
+
For long generations (>500 tokens) where the model loops, raise
|
| 122 |
+
repetition_penalty to 1.1–1.2 or lower top_p to ~0.9.
|
| 123 |
+
"""
|
| 124 |
+
prompt_ids = [BOS] + tokenizer.encode(prompt).ids
|
| 125 |
+
ctx = torch.tensor([prompt_ids] * n_samples, device=device) # (N, T_prompt)
|
| 126 |
+
# Repetition-penalty memory: a (N, V) bool mask, True where the corresponding
|
| 127 |
+
# token id has already been emitted. Constant cost per step regardless of
|
| 128 |
+
# generation length — beats a growing token-list approach on MPS where
|
| 129 |
+
# gather/scatter over scattered indices is slow.
|
| 130 |
+
V = None # filled in after first forward (we don't know it yet)
|
| 131 |
+
seen_mask = None
|
| 132 |
+
|
| 133 |
+
t0 = time.time()
|
| 134 |
+
logits, states = model.forward_with_states(ctx)
|
| 135 |
+
next_logits = logits[:, -1, :].clone() # (N, V); clone so compile doesn't see a view
|
| 136 |
+
# Now we know V; build the mask and seed it with the prompt tokens.
|
| 137 |
+
V = next_logits.size(-1)
|
| 138 |
+
if repetition_penalty != 1.0:
|
| 139 |
+
seen_mask = torch.zeros(n_samples, V, dtype=torch.bool, device=device)
|
| 140 |
+
seen_mask.scatter_(1, ctx, True)
|
| 141 |
+
_sync()
|
| 142 |
+
t_prefill = time.time()
|
| 143 |
+
|
| 144 |
+
can_stream = stream and n_samples == 1
|
| 145 |
+
if can_stream:
|
| 146 |
+
print(prompt, end='', flush=True)
|
| 147 |
+
|
| 148 |
+
generated = [[] for _ in range(n_samples)]
|
| 149 |
+
done = [False] * n_samples
|
| 150 |
+
|
| 151 |
+
for _ in range(max_new_tokens):
|
| 152 |
+
next_toks = _sample_token_batch(next_logits, temperature, top_k, top_p,
|
| 153 |
+
repetition_penalty, seen_mask) # (N,)
|
| 154 |
+
next_toks_list = next_toks.tolist()
|
| 155 |
+
any_alive = False
|
| 156 |
+
for i in range(n_samples):
|
| 157 |
+
if done[i]:
|
| 158 |
+
continue
|
| 159 |
+
tid = next_toks_list[i]
|
| 160 |
+
if tid == BOS:
|
| 161 |
+
done[i] = True
|
| 162 |
+
else:
|
| 163 |
+
generated[i].append(tid)
|
| 164 |
+
any_alive = True
|
| 165 |
+
if not any_alive:
|
| 166 |
+
break
|
| 167 |
+
if can_stream:
|
| 168 |
+
# Per-token decode is O(1); avoids the O(N²) full-list re-decode each step.
|
| 169 |
+
# ByteLevel BPE pieces decode cleanly for ASCII English, which dominates FineWeb-Edu.
|
| 170 |
+
piece = tokenizer.decode([next_toks_list[0]])
|
| 171 |
+
sys.stdout.write(piece)
|
| 172 |
+
sys.stdout.flush()
|
| 173 |
+
# Mark the new token as seen (O(N) scatter — constant per step).
|
| 174 |
+
if seen_mask is not None:
|
| 175 |
+
seen_mask.scatter_(1, next_toks.unsqueeze(1), True)
|
| 176 |
+
x_in = next_toks.unsqueeze(1) # (N, 1)
|
| 177 |
+
step_logits, states = model.step(x_in, states)
|
| 178 |
+
next_logits = step_logits[:, 0, :]
|
| 179 |
+
|
| 180 |
+
_sync()
|
| 181 |
+
t_end = time.time()
|
| 182 |
+
if can_stream:
|
| 183 |
+
print()
|
| 184 |
+
|
| 185 |
+
total_n = sum(len(g) for g in generated)
|
| 186 |
+
prefill_ms = (t_prefill - t0) * 1000
|
| 187 |
+
decode_s = max(t_end - t_prefill, 1e-9)
|
| 188 |
+
print(f" [prefill {len(prompt_ids)}×{n_samples} tok in {prefill_ms:.0f}ms | "
|
| 189 |
+
f"decode {total_n} tok in {decode_s:.2f}s = {total_n/decode_s:.1f} tok/s aggregate]",
|
| 190 |
+
file=sys.stderr)
|
| 191 |
+
return [tokenizer.decode(g) for g in generated]
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def load_model(ckpt_path, vocab_size, compile_model=False):
|
| 195 |
+
ckpt = torch.load(ckpt_path, map_location=device)
|
| 196 |
+
if 'config' in ckpt:
|
| 197 |
+
config = dict(ckpt['config'])
|
| 198 |
+
else:
|
| 199 |
+
print(f"note: checkpoint has no 'config' (older format); using fallback {FALLBACK_CONFIG}",
|
| 200 |
+
file=sys.stderr)
|
| 201 |
+
config = dict(FALLBACK_CONFIG)
|
| 202 |
+
config['vocab_size'] = vocab_size # vocab is set by the tokenizer, not the checkpoint
|
| 203 |
+
model = GPT.from_config(config).to(device)
|
| 204 |
+
# Strip torch.compile's `_orig_mod.` prefix if present in older checkpoints.
|
| 205 |
+
state = {k.removeprefix('_orig_mod.'): v for k, v in ckpt['model'].items()}
|
| 206 |
+
missing, unexpected = model.load_state_dict(state, strict=False)
|
| 207 |
+
if missing: print(f"warn: missing keys: {missing}", file=sys.stderr)
|
| 208 |
+
if unexpected: print(f"warn: unexpected keys: {unexpected}", file=sys.stderr)
|
| 209 |
+
model.eval()
|
| 210 |
+
if compile_model:
|
| 211 |
+
print("compiling step/prefill paths...", file=sys.stderr)
|
| 212 |
+
model.step = torch.compile(model.step, dynamic=False)
|
| 213 |
+
model.forward_with_states = torch.compile(model.forward_with_states, dynamic=False)
|
| 214 |
+
return model, ckpt, config
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def main():
|
| 218 |
+
p = argparse.ArgumentParser(description="Sample from a trained microgpt Stream Mixer.")
|
| 219 |
+
p.add_argument('-p', '--prompt', type=str, default="In this article, we will explore")
|
| 220 |
+
p.add_argument('-n', '--max-tokens', type=int, default=256)
|
| 221 |
+
p.add_argument('-t', '--temperature', type=float, default=0.8,
|
| 222 |
+
help='softmax temperature; 0 = greedy argmax. Recommended 0.7–0.9 for prose. '
|
| 223 |
+
'(Determined empirically — see v4 log.md post-completion section.)')
|
| 224 |
+
p.add_argument('-k', '--top-k', type=int, default=50,
|
| 225 |
+
help='restrict sampling to top-k logits; 0 disables. '
|
| 226 |
+
'Recommended 50 — keeps the long tail of the 32k vocab from injecting weird tokens.')
|
| 227 |
+
p.add_argument('--top-p', type=float, default=1.0,
|
| 228 |
+
help='nucleus sampling: keep the smallest token set whose cumulative '
|
| 229 |
+
'probability ≥ top-p. 1.0 disables. Recommended 0.9 for long generations.')
|
| 230 |
+
p.add_argument('-r', '--repetition-penalty', type=float, default=1.0,
|
| 231 |
+
help='CTRL-style repetition penalty: tokens already in the context get '
|
| 232 |
+
'their logits divided by this. 1.0 = no penalty. Recommended 1.1–1.2 '
|
| 233 |
+
'for >500-token generations to break out of repetition wells.')
|
| 234 |
+
p.add_argument('-s', '--num-samples', type=int, default=1)
|
| 235 |
+
p.add_argument('--ckpt', type=str, default=DEFAULT_CKPT)
|
| 236 |
+
p.add_argument('--tokenizer', type=str, default=DEFAULT_TOKENIZER)
|
| 237 |
+
p.add_argument('--seed', type=int, default=None)
|
| 238 |
+
p.add_argument('--no-stream', action='store_true',
|
| 239 |
+
help='print full output at end instead of streaming token by token')
|
| 240 |
+
p.add_argument('--compile', action='store_true',
|
| 241 |
+
help='torch.compile the step/prefill paths (slower first call, faster after)')
|
| 242 |
+
p.add_argument('--repl', action='store_true',
|
| 243 |
+
help='after generating, read more prompts from stdin so --compile is paid once')
|
| 244 |
+
args = p.parse_args()
|
| 245 |
+
|
| 246 |
+
if args.seed is not None:
|
| 247 |
+
torch.manual_seed(args.seed)
|
| 248 |
+
if not os.path.exists(args.ckpt):
|
| 249 |
+
sys.exit(f"error: checkpoint not found: {args.ckpt}")
|
| 250 |
+
if not os.path.exists(args.tokenizer):
|
| 251 |
+
sys.exit(f"error: tokenizer not found: {args.tokenizer}")
|
| 252 |
+
|
| 253 |
+
print(f"device: {device}", file=sys.stderr)
|
| 254 |
+
tokenizer = Tokenizer.from_file(args.tokenizer)
|
| 255 |
+
BOS = tokenizer.token_to_id("<|bos|>")
|
| 256 |
+
|
| 257 |
+
model, ckpt, config = load_model(args.ckpt, tokenizer.get_vocab_size(),
|
| 258 |
+
compile_model=args.compile)
|
| 259 |
+
step = ckpt.get('step', '?')
|
| 260 |
+
best = ckpt.get('best_loss')
|
| 261 |
+
n_params = sum(p.numel() for p in model.parameters())
|
| 262 |
+
best_str = f" best_loss={best:.4f}" if isinstance(best, float) else ""
|
| 263 |
+
print(f"loaded {args.ckpt} step={step}{best_str}", file=sys.stderr)
|
| 264 |
+
print(f"config: {config}", file=sys.stderr)
|
| 265 |
+
print(f"params: {n_params:,}\n", file=sys.stderr)
|
| 266 |
+
|
| 267 |
+
stream = not args.no_stream
|
| 268 |
+
|
| 269 |
+
def run(prompt):
|
| 270 |
+
results = generate(model, tokenizer, BOS, prompt,
|
| 271 |
+
n_samples=args.num_samples,
|
| 272 |
+
max_new_tokens=args.max_tokens,
|
| 273 |
+
temperature=args.temperature,
|
| 274 |
+
top_k=args.top_k,
|
| 275 |
+
top_p=args.top_p,
|
| 276 |
+
repetition_penalty=args.repetition_penalty,
|
| 277 |
+
stream=stream)
|
| 278 |
+
# Single-sample streaming already printed; otherwise print collected results.
|
| 279 |
+
if not (stream and args.num_samples == 1):
|
| 280 |
+
for i, out in enumerate(results):
|
| 281 |
+
if args.num_samples > 1:
|
| 282 |
+
print(f"--- sample {i+1}/{args.num_samples} ---", file=sys.stderr)
|
| 283 |
+
print(prompt + out)
|
| 284 |
+
if args.num_samples > 1:
|
| 285 |
+
print(file=sys.stderr)
|
| 286 |
+
|
| 287 |
+
run(args.prompt)
|
| 288 |
+
|
| 289 |
+
# REPL: amortize torch.compile warmup over many prompts. Empty input
|
| 290 |
+
# reuses the previous prompt; "exit"/"quit"/Ctrl-D leaves the loop.
|
| 291 |
+
if args.repl:
|
| 292 |
+
last_prompt = args.prompt
|
| 293 |
+
print("\n[repl] enter prompt (empty=repeat, exit/quit/Ctrl-D to leave)", file=sys.stderr)
|
| 294 |
+
while True:
|
| 295 |
+
try:
|
| 296 |
+
line = input(">>> ")
|
| 297 |
+
except (EOFError, KeyboardInterrupt):
|
| 298 |
+
print(file=sys.stderr)
|
| 299 |
+
break
|
| 300 |
+
line = line.strip()
|
| 301 |
+
if line in ('exit', 'quit'):
|
| 302 |
+
break
|
| 303 |
+
prompt = line if line else last_prompt
|
| 304 |
+
last_prompt = prompt
|
| 305 |
+
run(prompt)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
if __name__ == '__main__':
|
| 309 |
+
main()
|