File size: 6,096 Bytes
dd53693
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
"""Train the model on an endless stream of generated pairs, then export it for the browser.

    python train.py --steps 12000
"""
import argparse
import json
import random
import time
from pathlib import Path

import numpy as np
import torch
import torch.nn.functional as F

import gen
import tok
from lexicon import ACT, TYPES
from model import TinyGPT
from qa_topics import TOPICS

ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "web" / "model"


def batch(r, n, dev):
    xs, ys = [], []
    for _ in range(n):
        text, tokens = gen.sample(r)
        p, t = tok.encode_pair(text, tokens)
        seq = p + t
        xs.append(seq[:-1])
        ys.append([-100] * (len(p) - 1) + t)
    T = max(map(len, xs))
    x = torch.zeros(n, T, dtype=torch.long)
    y = torch.full((n, T), -100, dtype=torch.long)
    for i, (a, b) in enumerate(zip(xs, ys)):
        x[i, :len(a)] = torch.tensor(a)
        y[i, :len(b)] = torch.tensor(b)
    return x.to(dev), y.to(dev)


@torch.no_grad()
def exact_match(model, pairs):
    """Greedy-decode every pair in one padded batch; a pair counts only if every token matches."""
    model.eval()
    dev = model.tok.weight.device
    enc = [tok.encode_pair(t, k) for t, k in pairs]
    order = sorted(range(len(enc)), key=lambda i: len(enc[i][0]))
    ok = 0
    for s in range(0, len(order), 256):
        ids = order[s:s + 256]
        seqs = [list(enc[i][0]) for i in ids]
        done = [False] * len(ids)
        outs = [[] for _ in ids]
        for _ in range(40):
            T = max(map(len, seqs))
            x = torch.zeros(len(ids), T, dtype=torch.long, device=dev)
            for j, q in enumerate(seqs):
                x[j, :len(q)] = torch.tensor(q)
            logits = model(x)
            for j, q in enumerate(seqs):
                if done[j]:
                    continue
                n = int(logits[j, len(q) - 1].argmax())
                if n == tok.EOS or len(q) >= tok.CTX - 1:
                    done[j] = True
                else:
                    q.append(n)
                    outs[j].append(n)
            if all(done):
                break
        ok += sum(outs[j] + [tok.EOS] == enc[i][1] for j, i in enumerate(ids))
    model.train()
    return ok / len(pairs)


def export(model, meta):
    OUT.mkdir(parents=True, exist_ok=True)
    tensors, blob, off = [], [], 0
    for name, p in model.state_dict().items():
        a = p.detach().float().cpu().numpy().astype(np.float16)
        tensors.append({"name": name, "shape": list(a.shape), "offset": off})
        blob.append(a.tobytes())
        off += a.size
    (OUT / "weights.bin").write_bytes(b"".join(blob))
    (OUT / "model.json").write_text(json.dumps({
        "config": model.cfg, "dtype": "float16", "vocab": tok.VOCAB, "max_prompt": tok.MAX_PROMPT,
        "special": {"pad": tok.PAD, "bos": tok.BOS, "sep": tok.SEP, "eos": tok.EOS},
        "tensors": tensors, "meta": meta}, ensure_ascii=False))
    names = lambda table, lang: ", ".join((v.en[0][0] if lang == "en" else v.es[0][0]) for v in table.values())
    acts = lambda lang: ", ".join((v.ing[0] if lang == "en" else v.ger[0].replace("{o}", "o")) for v in ACT.values())
    fill = lambda s, lang: {"@objects": ("I know: " if lang == "en" else "Conozco: ") + names(TYPES, lang) + ".",
                            "@actions": ("Actions: " if lang == "en" else "Acciones: ") + acts(lang) + "."}.get(s, s)
    (OUT / "qa.json").write_text(json.dumps(
        {k: {"en": fill(v[4], "en"), "es": fill(v[5], "es")} for k, v in TOPICS.items()}, ensure_ascii=False, indent=1))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--steps", type=int, default=12000)
    ap.add_argument("--bs", type=int, default=256)
    ap.add_argument("--lr", type=float, default=1e-3)
    ap.add_argument("--d", type=int, default=256)
    ap.add_argument("--layers", type=int, default=6)
    ap.add_argument("--heads", type=int, default=8)
    a = ap.parse_args()

    dev = "cuda" if torch.cuda.is_available() else "cpu"
    torch.manual_seed(0)
    model = TinyGPT(len(tok.VOCAB), tok.CTX, a.d, a.layers, a.heads).to(dev)
    n_params = sum(p.numel() for p in model.parameters())
    print(f"device {dev} | vocab {len(tok.VOCAB)} | params {n_params:,}", flush=True)
    opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01, betas=(0.9, 0.98))
    sched = torch.optim.lr_scheduler.OneCycleLR(opt, a.lr, total_steps=a.steps, pct_start=0.05)

    er = random.Random(10_000_019)
    held_noisy = [gen.sample(er) for _ in range(3000)]
    held_clean = [gen.sample(er, clean=True) for _ in range(3000)]
    (ROOT / "ml" / "heldout.jsonl").write_text(
        "\n".join(json.dumps({"prompt": t, "target": " ".join(k)}, ensure_ascii=False) for t, k in held_clean))

    r = random.Random(0)
    t0 = time.time()
    for step in range(1, a.steps + 1):
        x, y = batch(r, a.bs, dev)
        with torch.autocast(dev, dtype=torch.bfloat16, enabled=dev == "cuda"):
            loss = F.cross_entropy(model(x).float().view(-1, len(tok.VOCAB)), y.view(-1), ignore_index=-100)
        opt.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        sched.step()
        if step % 250 == 0:
            print(f"step {step:6d} loss {loss.item():.4f} {time.time() - t0:.0f}s", flush=True)
        if step % 3000 == 0 or step == a.steps:
            c, n = exact_match(model, held_clean[:1000]), exact_match(model, held_noisy[:1000])
            print(f"  exact match: clean {c:.3f} noisy {n:.3f}", flush=True)

    clean, noisy = exact_match(model, held_clean), exact_match(model, held_noisy)
    meta = {"params": n_params, "steps": a.steps, "batch": a.bs, "pairs_seen": a.steps * a.bs,
            "heldout_exact_clean": round(clean, 4), "heldout_exact_noisy": round(noisy, 4),
            "train_seconds": round(time.time() - t0)}
    print("final", meta, flush=True)
    torch.save(model.state_dict(), ROOT / "ml" / "model.pt")
    export(model, meta)


if __name__ == "__main__":
    main()