tiny-scene-three / ml /train.py
paralaif's picture
Tiny Scene: text to animated three.js scenes with a 4.85M Transformer
dd53693 verified
Raw History Blame Contribute Delete
6.1 kB
"""Train the model on an endless stream of generated pairs, then export it for the browser.
python train.py --steps 12000
"""
import argparse
import json
import random
import time
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
import gen
import tok
from lexicon import ACT, TYPES
from model import TinyGPT
from qa_topics import TOPICS
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "web" / "model"
def batch(r, n, dev):
xs, ys = [], []
for _ in range(n):
text, tokens = gen.sample(r)
p, t = tok.encode_pair(text, tokens)
seq = p + t
xs.append(seq[:-1])
ys.append([-100] * (len(p) - 1) + t)
T = max(map(len, xs))
x = torch.zeros(n, T, dtype=torch.long)
y = torch.full((n, T), -100, dtype=torch.long)
for i, (a, b) in enumerate(zip(xs, ys)):
x[i, :len(a)] = torch.tensor(a)
y[i, :len(b)] = torch.tensor(b)
return x.to(dev), y.to(dev)
@torch.no_grad()
def exact_match(model, pairs):
"""Greedy-decode every pair in one padded batch; a pair counts only if every token matches."""
model.eval()
dev = model.tok.weight.device
enc = [tok.encode_pair(t, k) for t, k in pairs]
order = sorted(range(len(enc)), key=lambda i: len(enc[i][0]))
ok = 0
for s in range(0, len(order), 256):
ids = order[s:s + 256]
seqs = [list(enc[i][0]) for i in ids]
done = [False] * len(ids)
outs = [[] for _ in ids]
for _ in range(40):
T = max(map(len, seqs))
x = torch.zeros(len(ids), T, dtype=torch.long, device=dev)
for j, q in enumerate(seqs):
x[j, :len(q)] = torch.tensor(q)
logits = model(x)
for j, q in enumerate(seqs):
if done[j]:
continue
n = int(logits[j, len(q) - 1].argmax())
if n == tok.EOS or len(q) >= tok.CTX - 1:
done[j] = True
else:
q.append(n)
outs[j].append(n)
if all(done):
break
ok += sum(outs[j] + [tok.EOS] == enc[i][1] for j, i in enumerate(ids))
model.train()
return ok / len(pairs)
def export(model, meta):
OUT.mkdir(parents=True, exist_ok=True)
tensors, blob, off = [], [], 0
for name, p in model.state_dict().items():
a = p.detach().float().cpu().numpy().astype(np.float16)
tensors.append({"name": name, "shape": list(a.shape), "offset": off})
blob.append(a.tobytes())
off += a.size
(OUT / "weights.bin").write_bytes(b"".join(blob))
(OUT / "model.json").write_text(json.dumps({
"config": model.cfg, "dtype": "float16", "vocab": tok.VOCAB, "max_prompt": tok.MAX_PROMPT,
"special": {"pad": tok.PAD, "bos": tok.BOS, "sep": tok.SEP, "eos": tok.EOS},
"tensors": tensors, "meta": meta}, ensure_ascii=False))
names = lambda table, lang: ", ".join((v.en[0][0] if lang == "en" else v.es[0][0]) for v in table.values())
acts = lambda lang: ", ".join((v.ing[0] if lang == "en" else v.ger[0].replace("{o}", "o")) for v in ACT.values())
fill = lambda s, lang: {"@objects": ("I know: " if lang == "en" else "Conozco: ") + names(TYPES, lang) + ".",
"@actions": ("Actions: " if lang == "en" else "Acciones: ") + acts(lang) + "."}.get(s, s)
(OUT / "qa.json").write_text(json.dumps(
{k: {"en": fill(v[4], "en"), "es": fill(v[5], "es")} for k, v in TOPICS.items()}, ensure_ascii=False, indent=1))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--steps", type=int, default=12000)
ap.add_argument("--bs", type=int, default=256)
ap.add_argument("--lr", type=float, default=1e-3)
ap.add_argument("--d", type=int, default=256)
ap.add_argument("--layers", type=int, default=6)
ap.add_argument("--heads", type=int, default=8)
a = ap.parse_args()
dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)
model = TinyGPT(len(tok.VOCAB), tok.CTX, a.d, a.layers, a.heads).to(dev)
n_params = sum(p.numel() for p in model.parameters())
print(f"device {dev} | vocab {len(tok.VOCAB)} | params {n_params:,}", flush=True)
opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01, betas=(0.9, 0.98))
sched = torch.optim.lr_scheduler.OneCycleLR(opt, a.lr, total_steps=a.steps, pct_start=0.05)
er = random.Random(10_000_019)
held_noisy = [gen.sample(er) for _ in range(3000)]
held_clean = [gen.sample(er, clean=True) for _ in range(3000)]
(ROOT / "ml" / "heldout.jsonl").write_text(
"\n".join(json.dumps({"prompt": t, "target": " ".join(k)}, ensure_ascii=False) for t, k in held_clean))
r = random.Random(0)
t0 = time.time()
for step in range(1, a.steps + 1):
x, y = batch(r, a.bs, dev)
with torch.autocast(dev, dtype=torch.bfloat16, enabled=dev == "cuda"):
loss = F.cross_entropy(model(x).float().view(-1, len(tok.VOCAB)), y.view(-1), ignore_index=-100)
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
sched.step()
if step % 250 == 0:
print(f"step {step:6d} loss {loss.item():.4f} {time.time() - t0:.0f}s", flush=True)
if step % 3000 == 0 or step == a.steps:
c, n = exact_match(model, held_clean[:1000]), exact_match(model, held_noisy[:1000])
print(f" exact match: clean {c:.3f} noisy {n:.3f}", flush=True)
clean, noisy = exact_match(model, held_clean), exact_match(model, held_noisy)
meta = {"params": n_params, "steps": a.steps, "batch": a.bs, "pairs_seen": a.steps * a.bs,
"heldout_exact_clean": round(clean, 4), "heldout_exact_noisy": round(noisy, 4),
"train_seconds": round(time.time() - t0)}
print("final", meta, flush=True)
torch.save(model.state_dict(), ROOT / "ml" / "model.pt")
export(model, meta)
if __name__ == "__main__":
main()