Download ml/train.py from paralaif/tiny-scene-three: direct link, hf CLI and curl.
- Browser
- Download file 6.1 kB
-
https://huggingface.co/paralaif/tiny-scene-three/resolve/main/ml/train.py
- Command line
-
hf download hf://paralaif/tiny-scene-three/ml/train.py
-
curl -L -o train.py https://huggingface.co/paralaif/tiny-scene-three/resolve/main/ml/train.py
6.1 kB
| """Train the model on an endless stream of generated pairs, then export it for the browser. | |
| python train.py --steps 12000 | |
| """ | |
| import argparse | |
| import json | |
| import random | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| import gen | |
| import tok | |
| from lexicon import ACT, TYPES | |
| from model import TinyGPT | |
| from qa_topics import TOPICS | |
| ROOT = Path(__file__).resolve().parent.parent | |
| OUT = ROOT / "web" / "model" | |
| def batch(r, n, dev): | |
| xs, ys = [], [] | |
| for _ in range(n): | |
| text, tokens = gen.sample(r) | |
| p, t = tok.encode_pair(text, tokens) | |
| seq = p + t | |
| xs.append(seq[:-1]) | |
| ys.append([-100] * (len(p) - 1) + t) | |
| T = max(map(len, xs)) | |
| x = torch.zeros(n, T, dtype=torch.long) | |
| y = torch.full((n, T), -100, dtype=torch.long) | |
| for i, (a, b) in enumerate(zip(xs, ys)): | |
| x[i, :len(a)] = torch.tensor(a) | |
| y[i, :len(b)] = torch.tensor(b) | |
| return x.to(dev), y.to(dev) | |
| def exact_match(model, pairs): | |
| """Greedy-decode every pair in one padded batch; a pair counts only if every token matches.""" | |
| model.eval() | |
| dev = model.tok.weight.device | |
| enc = [tok.encode_pair(t, k) for t, k in pairs] | |
| order = sorted(range(len(enc)), key=lambda i: len(enc[i][0])) | |
| ok = 0 | |
| for s in range(0, len(order), 256): | |
| ids = order[s:s + 256] | |
| seqs = [list(enc[i][0]) for i in ids] | |
| done = [False] * len(ids) | |
| outs = [[] for _ in ids] | |
| for _ in range(40): | |
| T = max(map(len, seqs)) | |
| x = torch.zeros(len(ids), T, dtype=torch.long, device=dev) | |
| for j, q in enumerate(seqs): | |
| x[j, :len(q)] = torch.tensor(q) | |
| logits = model(x) | |
| for j, q in enumerate(seqs): | |
| if done[j]: | |
| continue | |
| n = int(logits[j, len(q) - 1].argmax()) | |
| if n == tok.EOS or len(q) >= tok.CTX - 1: | |
| done[j] = True | |
| else: | |
| q.append(n) | |
| outs[j].append(n) | |
| if all(done): | |
| break | |
| ok += sum(outs[j] + [tok.EOS] == enc[i][1] for j, i in enumerate(ids)) | |
| model.train() | |
| return ok / len(pairs) | |
| def export(model, meta): | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| tensors, blob, off = [], [], 0 | |
| for name, p in model.state_dict().items(): | |
| a = p.detach().float().cpu().numpy().astype(np.float16) | |
| tensors.append({"name": name, "shape": list(a.shape), "offset": off}) | |
| blob.append(a.tobytes()) | |
| off += a.size | |
| (OUT / "weights.bin").write_bytes(b"".join(blob)) | |
| (OUT / "model.json").write_text(json.dumps({ | |
| "config": model.cfg, "dtype": "float16", "vocab": tok.VOCAB, "max_prompt": tok.MAX_PROMPT, | |
| "special": {"pad": tok.PAD, "bos": tok.BOS, "sep": tok.SEP, "eos": tok.EOS}, | |
| "tensors": tensors, "meta": meta}, ensure_ascii=False)) | |
| names = lambda table, lang: ", ".join((v.en[0][0] if lang == "en" else v.es[0][0]) for v in table.values()) | |
| acts = lambda lang: ", ".join((v.ing[0] if lang == "en" else v.ger[0].replace("{o}", "o")) for v in ACT.values()) | |
| fill = lambda s, lang: {"@objects": ("I know: " if lang == "en" else "Conozco: ") + names(TYPES, lang) + ".", | |
| "@actions": ("Actions: " if lang == "en" else "Acciones: ") + acts(lang) + "."}.get(s, s) | |
| (OUT / "qa.json").write_text(json.dumps( | |
| {k: {"en": fill(v[4], "en"), "es": fill(v[5], "es")} for k, v in TOPICS.items()}, ensure_ascii=False, indent=1)) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--steps", type=int, default=12000) | |
| ap.add_argument("--bs", type=int, default=256) | |
| ap.add_argument("--lr", type=float, default=1e-3) | |
| ap.add_argument("--d", type=int, default=256) | |
| ap.add_argument("--layers", type=int, default=6) | |
| ap.add_argument("--heads", type=int, default=8) | |
| a = ap.parse_args() | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| torch.manual_seed(0) | |
| model = TinyGPT(len(tok.VOCAB), tok.CTX, a.d, a.layers, a.heads).to(dev) | |
| n_params = sum(p.numel() for p in model.parameters()) | |
| print(f"device {dev} | vocab {len(tok.VOCAB)} | params {n_params:,}", flush=True) | |
| opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01, betas=(0.9, 0.98)) | |
| sched = torch.optim.lr_scheduler.OneCycleLR(opt, a.lr, total_steps=a.steps, pct_start=0.05) | |
| er = random.Random(10_000_019) | |
| held_noisy = [gen.sample(er) for _ in range(3000)] | |
| held_clean = [gen.sample(er, clean=True) for _ in range(3000)] | |
| (ROOT / "ml" / "heldout.jsonl").write_text( | |
| "\n".join(json.dumps({"prompt": t, "target": " ".join(k)}, ensure_ascii=False) for t, k in held_clean)) | |
| r = random.Random(0) | |
| t0 = time.time() | |
| for step in range(1, a.steps + 1): | |
| x, y = batch(r, a.bs, dev) | |
| with torch.autocast(dev, dtype=torch.bfloat16, enabled=dev == "cuda"): | |
| loss = F.cross_entropy(model(x).float().view(-1, len(tok.VOCAB)), y.view(-1), ignore_index=-100) | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| opt.step() | |
| sched.step() | |
| if step % 250 == 0: | |
| print(f"step {step:6d} loss {loss.item():.4f} {time.time() - t0:.0f}s", flush=True) | |
| if step % 3000 == 0 or step == a.steps: | |
| c, n = exact_match(model, held_clean[:1000]), exact_match(model, held_noisy[:1000]) | |
| print(f" exact match: clean {c:.3f} noisy {n:.3f}", flush=True) | |
| clean, noisy = exact_match(model, held_clean), exact_match(model, held_noisy) | |
| meta = {"params": n_params, "steps": a.steps, "batch": a.bs, "pairs_seen": a.steps * a.bs, | |
| "heldout_exact_clean": round(clean, 4), "heldout_exact_noisy": round(noisy, 4), | |
| "train_seconds": round(time.time() - t0)} | |
| print("final", meta, flush=True) | |
| torch.save(model.state_dict(), ROOT / "ml" / "model.pt") | |
| export(model, meta) | |
| if __name__ == "__main__": | |
| main() | |