Download code/python/motion2.py from StarpowerTechnology/WVY-3D-1M: direct link, hf CLI and curl.
- Browser
- Download file 17.8 kB
-
https://huggingface.co/StarpowerTechnology/WVY-3D-1M/resolve/main/code/python/motion2.py
- Command line
-
hf download hf://StarpowerTechnology/WVY-3D-1M/code/python/motion2.py
-
curl -L -o motion2.py https://huggingface.co/StarpowerTechnology/WVY-3D-1M/resolve/main/code/python/motion2.py
17.8 kB
| """MotionGPT v2: one language model that writes motion for six body plans. | |
| Vocabulary = 6 x 256 pose words (one dictionary per body plan) + 25 prompt words like 'quadruped:run'.""" | |
| import numpy as np, time, sys, json | |
| from tinygrad import T, Param, Linear, Adam, cross_entropy, embed | |
| FPS, CLIP, K = 15, 64, 256 | |
| FAM = { # joint counts and actions; order matters (token ranges) | |
| "biped": (17, ["idle", "walk", "run", "wave", "jump", "dance"]), | |
| "quadruped": (19, ["idle", "walk", "run", "jump", "sit", "eat"]), | |
| "bird": (17, ["idle", "hop", "peck", "fly", "glide"]), | |
| "fish": (8, ["idle", "swim", "dart"]), | |
| "snake": (10, ["idle", "slither"]), | |
| "insect": (21, ["idle", "crawl", "run"]), | |
| } | |
| FAMS = list(FAM) | |
| PROMPTS = [f"{f}:{a}" for f in FAMS for a in FAM[f][1]] | |
| V = K * len(FAMS) + len(PROMPTS) | |
| dim = lambda f: FAM[f][0] * 3 + 4 | |
| SX, DY, VZ, YAW = 0, 1, 2, 3 # root channel offsets (after joint eulers) | |
| class Clip: | |
| def __init__(self, fam): | |
| self.f = fam; self.J = FAM[fam][0] | |
| self.X = np.zeros((CLIP, dim(fam))) | |
| self.t = np.arange(CLIP) / FPS | |
| def set(self, i, j, x=None, y=None, z=None): | |
| if x is not None: self.X[i, j * 3] = x | |
| if y is not None: self.X[i, j * 3 + 1] = y | |
| if z is not None: self.X[i, j * 3 + 2] = z | |
| def root(self, i, c, v): | |
| self.X[i, self.J * 3 + c] = v | |
| def ease(q, kf): | |
| kf = np.array(kf, float) | |
| return np.interp(q, kf[:, 0], kf[:, 1]) | |
| # ------------------------------------------------------------------ biped | |
| def biped(a, r): | |
| c = Clip("biped"); u = r.uniform; ph0 = u(0, 6.28) | |
| for i, tt in enumerate(c.t): | |
| if a in ("walk", "run"): | |
| run = a == "run" | |
| if i == 0: | |
| f = u(1.35, 1.6) if run else u(0.85, 1.15); A = u(0.6, 0.8) if run else u(0.3, 0.5) | |
| Kn = u(1.3, 1.7) if run else u(0.6, 0.9); S = u(0.6, 0.9) if run else u(0.25, 0.45) | |
| E = u(1.3, 1.6) if run else u(0.2, 0.4); lean = u(0.15, 0.3) if run else u(0, 0.06) | |
| v = (u(2.2, 2.8) if run else u(0.8, 1.1)) / FPS | |
| ph = 6.283 * f * tt + ph0 | |
| c.set(i, 11, x=-A * np.sin(ph)); c.set(i, 14, x=A * np.sin(ph)) | |
| c.set(i, 12, x=Kn * max(0, np.cos(ph)) + 0.05); c.set(i, 15, x=Kn * max(0, -np.cos(ph)) + 0.05) | |
| c.set(i, 13, x=-0.2 * np.sin(ph)); c.set(i, 16, x=0.2 * np.sin(ph)) | |
| c.set(i, 5, x=S * np.sin(ph), z=-1.3); c.set(i, 8, x=-S * np.sin(ph), z=1.3) | |
| c.set(i, 6, y=-E); c.set(i, 9, y=E) | |
| c.set(i, 1, x=lean, y=0.12 * np.sin(ph)); c.set(i, 0, y=-0.1 * np.sin(ph)); c.set(i, 3, x=-lean * 0.5) | |
| c.root(i, SX, 0.015 * np.sin(ph)); c.root(i, DY, (0.06 if run else 0.025) * np.cos(2 * ph) - (0.06 if run else 0)); c.root(i, VZ, v) | |
| elif a == "wave": | |
| if i == 0: w = u(2, 3); amp = u(0.35, 0.6); rz = u(0.95, 1.3) | |
| s = np.sin(6.283 * w * tt + ph0) | |
| c.set(i, 8, z=-rz, x=-0.2); c.set(i, 9, z=amp * s - 0.2, y=0.3) | |
| c.set(i, 5, z=-1.35, x=0.05); c.set(i, 6, y=-0.25) | |
| c.set(i, 1, z=0.06 * s); c.set(i, 4, z=-0.12 + 0.05 * s, x=0.05) | |
| c.set(i, 12, x=0.05); c.set(i, 15, x=0.05) | |
| elif a == "jump": | |
| if i == 0: P = u(1.3, 1.7); H = u(0.25, 0.45) | |
| q = (tt / P + ph0 / 6.283) % 1 | |
| cr = ease(q, [[0, 0], [0.3, 1], [0.42, 0], [0.58, 0.25], [0.74, 0.1], [0.84, 0.8], [1, 0]]) | |
| hg = np.sin(np.clip((q - 0.42) / 0.32, 0, 1) * np.pi) * H | |
| arm = ease(q, [[0, 0], [0.3, 0.9], [0.42, -1.2], [0.58, -1.1], [0.74, -0.5], [0.84, 0.3], [1, 0]]) | |
| for hp, kn, an in ((11, 12, 13), (14, 15, 16)): | |
| c.set(i, hp, x=-0.9 * cr); c.set(i, kn, x=1.7 * cr + 0.5 * hg / H); c.set(i, an, x=-0.75 * cr) | |
| c.set(i, 1, x=0.45 * cr); c.set(i, 3, x=-0.3 * cr) | |
| c.set(i, 5, x=arm, z=-1.3 + 0.2 * cr); c.set(i, 8, x=arm, z=1.3 - 0.2 * cr) | |
| c.root(i, DY, -0.32 * cr + hg) | |
| elif a == "dance": | |
| if i == 0: f = u(0.9, 1.2); spin = u(-0.6, 0.6) / FPS | |
| ph = 6.283 * f * tt + ph0; b = 0.5 * (1 + np.cos(2 * ph)) | |
| c.set(i, 0, z=0.18 * np.sin(ph)); c.set(i, 1, z=-0.22 * np.sin(ph)); c.set(i, 4, z=0.15 * np.sin(ph + 0.5), x=0.1 * b) | |
| ul, ur = max(0, np.sin(ph)), max(0, -np.sin(ph)) | |
| c.set(i, 5, z=-1.2 + 2.4 * ul, x=-0.3); c.set(i, 8, z=1.2 - 2.4 * ur, x=-0.3) | |
| c.set(i, 6, y=-1.2 + 0.8 * ul); c.set(i, 9, y=1.2 - 0.8 * ur) | |
| for hp, kn, an in ((11, 12, 13), (14, 15, 16)): | |
| c.set(i, hp, x=-0.35 * b); c.set(i, kn, x=0.7 * b); c.set(i, an, x=-0.35 * b) | |
| c.root(i, SX, 0.05 * np.sin(ph)); c.root(i, DY, -0.09 * b); c.root(i, YAW, spin) | |
| else: # idle | |
| if i == 0: lk = u(0.15, 0.3); fr = u(0.15, 0.3) | |
| br = np.sin(6.283 * 0.3 * tt + ph0) | |
| c.set(i, 5, z=-1.3 + 0.03 * br); c.set(i, 8, z=1.3 - 0.03 * br); c.set(i, 6, y=-0.15); c.set(i, 9, y=0.15) | |
| c.set(i, 2, x=0.02 * br); c.set(i, 3, y=lk * np.sin(6.283 * fr * tt + ph0)) | |
| c.set(i, 0, z=0.03 * np.sin(6.283 * 0.2 * tt)) | |
| return c.X | |
| # ------------------------------------------------------------------ quadruped | |
| def quadruped(a, r): | |
| c = Clip("quadruped"); u = r.uniform; ph0 = u(0, 6.28) | |
| legs = [(11, 12, 13), (5, 6, 7), (14, 15, 16), (8, 9, 10)] # BL, FL, BR, FR | |
| for i, tt in enumerate(c.t): | |
| if a in ("walk", "run"): | |
| run = a == "run" | |
| if i == 0: | |
| f = u(1.8, 2.4) if run else u(0.8, 1.2); A = u(0.55, 0.75) if run else u(0.3, 0.45) | |
| Kn = u(1.0, 1.3) if run else u(0.5, 0.8); v = (u(4, 5) if run else u(1.2, 1.6)) / FPS | |
| offs = [0, 0.5, 0.1, 0.6] if run else [0, 0.25, 0.5, 0.75] | |
| ph = 6.283 * f * tt + ph0 | |
| for (h, k, ft), o in zip(legs, offs): | |
| p = ph + 6.283 * o | |
| c.set(i, h, x=-A * np.sin(p)); c.set(i, k, x=Kn * max(0, np.cos(p))) | |
| c.set(i, 1, x=(0.15 if run else 0.03) * np.sin(ph), y=0.06 * np.sin(ph)) | |
| c.set(i, 3, x=0.06 * np.sin(2 * ph)); c.set(i, 17, y=0.35 * np.sin(ph), x=-0.2 if run else 0) | |
| c.root(i, DY, (0.07 if run else 0.02) * np.cos(2 * ph)); c.root(i, VZ, v) | |
| elif a == "jump": | |
| if i == 0: P = u(1.4, 1.8); H = u(0.4, 0.7) | |
| q = (tt / P + ph0 / 6.283) % 1 | |
| cr = ease(q, [[0, 0], [0.3, 1], [0.4, 0], [0.75, 0], [0.85, 0.7], [1, 0]]) | |
| air = np.sin(np.clip((q - 0.38) / 0.37, 0, 1) * np.pi) | |
| for h, k, ft in legs: | |
| front = h in (5, 8) | |
| c.set(i, h, x=-0.5 * cr + (-0.9 if front else 0.7) * air); c.set(i, k, x=1.0 * cr + 0.3 * air) | |
| c.set(i, 0, x=-0.25 * air * np.cos(np.clip((q - 0.38) / 0.37, 0, 1) * np.pi)) | |
| c.set(i, 3, x=-0.3 * air); c.set(i, 17, x=-0.5 * air) | |
| c.root(i, DY, -0.25 * cr + H * air); c.root(i, VZ, (2.5 / FPS) * (air > 0.02)) | |
| elif a == "sit": | |
| if i == 0: pt = u(0.6, 0.8) | |
| br = np.sin(6.283 * 0.3 * tt + ph0) | |
| c.set(i, 0, x=-pt) | |
| for h, k, ft in (legs[0], legs[2]): c.set(i, h, x=pt - 1.3); c.set(i, k, x=2.2) | |
| for h, k, ft in (legs[1], legs[3]): c.set(i, h, x=pt) | |
| c.set(i, 3, x=pt * 0.7 + 0.03 * br, y=0.3 * np.sin(6.283 * 0.15 * tt + ph0)) | |
| c.set(i, 17, x=pt + 0.4, y=0.4 * np.sin(6.283 * 1.2 * tt)) | |
| elif a == "eat": | |
| ch = np.sin(6.283 * 2.5 * tt + ph0) | |
| c.set(i, 3, x=0.95 + 0.05 * ch); c.set(i, 4, x=0.2 + 0.05 * ch) | |
| c.set(i, 5, x=-0.15); c.set(i, 8, x=-0.15); c.set(i, 1, x=0.05) | |
| c.set(i, 17, y=0.3 * np.sin(6.283 * 0.8 * tt)) | |
| else: | |
| br = np.sin(6.283 * 0.3 * tt + ph0) | |
| c.set(i, 2, x=0.02 * br); c.set(i, 3, x=-0.05 + 0.03 * br, y=0.35 * np.sin(6.283 * 0.12 * tt + ph0)) | |
| c.set(i, 4, z=0.1 * np.sin(6.283 * 0.2 * tt)) | |
| c.set(i, 17, y=0.6 * np.sin(6.283 * 1.6 * tt + ph0)); c.set(i, 18, y=0.4 * np.sin(6.283 * 1.6 * tt + ph0 - 0.8)) | |
| return c.X | |
| # ------------------------------------------------------------------ bird | |
| def bird(a, r): | |
| c = Clip("bird"); u = r.uniform; ph0 = u(0, 6.28) | |
| def fold(i, open_=0.0): | |
| c.set(i, 4, y=1.25 * (1 - open_), z=-0.3 * (1 - open_)); c.set(i, 5, y=0.5 * (1 - open_)) | |
| c.set(i, 7, y=-1.25 * (1 - open_), z=0.3 * (1 - open_)); c.set(i, 8, y=-0.5 * (1 - open_)) | |
| for i, tt in enumerate(c.t): | |
| if a == "fly": | |
| if i == 0: f = u(2, 3.5); A = u(0.45, 0.65); alt = u(3, 6); v = u(3, 5) / FPS | |
| ph = 6.283 * f * tt + ph0 | |
| c.set(i, 4, z=A * np.sin(ph)); c.set(i, 5, z=0.3 * A * np.sin(ph - 0.6)); c.set(i, 6, z=0.15 * A * np.sin(ph - 1.2)) | |
| c.set(i, 7, z=-A * np.sin(ph)); c.set(i, 8, z=-0.3 * A * np.sin(ph - 0.6)); c.set(i, 9, z=-0.15 * A * np.sin(ph - 1.2)) | |
| for th, sh in ((10, 11), (13, 14)): c.set(i, th, x=1.3); c.set(i, sh, x=1.0) | |
| c.set(i, 16, x=-0.1 * np.sin(ph)); c.set(i, 2, x=0.2) | |
| c.root(i, DY, alt + 0.15 * np.sin(ph)); c.root(i, VZ, v) | |
| elif a == "glide": | |
| if i == 0: alt = u(4, 7); v = u(3, 4.5) / FPS; turn = u(-0.4, 0.4) / FPS | |
| s = np.sin(6.283 * 0.3 * tt + ph0) | |
| c.set(i, 4, z=0.08 + 0.05 * s); c.set(i, 7, z=-0.08 - 0.05 * s); c.set(i, 0, z=turn * FPS * 0.6) | |
| for th, sh in ((10, 11), (13, 14)): c.set(i, th, x=1.3); c.set(i, sh, x=1.0) | |
| c.root(i, DY, alt + 0.2 * s); c.root(i, VZ, v); c.root(i, YAW, turn) | |
| elif a == "hop": | |
| if i == 0: P = u(0.5, 0.8); H = u(0.3, 0.6) | |
| q = (tt / P + ph0 / 6.283) % 1 | |
| cr = ease(q, [[0, 0.2], [0.25, 1], [0.35, 0], [0.75, 0], [0.85, 0.8], [1, 0.2]]) | |
| air = np.sin(np.clip((q - 0.33) / 0.42, 0, 1) * np.pi) | |
| fold(i, 0.25 * air) | |
| for th, sh, ft in ((10, 11, 12), (13, 14, 15)): c.set(i, th, x=-0.6 * cr); c.set(i, sh, x=1.0 * cr); c.set(i, ft, x=-0.4 * cr) | |
| c.set(i, 2, x=-0.2 * air); c.root(i, DY, -0.15 * cr + H * air); c.root(i, VZ, (1.2 / FPS) * (air > 0.02)) | |
| elif a == "peck": | |
| if i == 0: P = u(0.6, 1.0) | |
| q = (tt / P + ph0 / 6.283) % 1 | |
| pk = ease(q, [[0, 0], [0.4, 0], [0.55, 1], [0.7, 0.2], [0.8, 1], [1, 0]]) | |
| fold(i) | |
| c.set(i, 0, x=0.35 * pk); c.set(i, 2, x=0.9 * pk) | |
| for th in (10, 13): c.set(i, th, x=-0.35 * pk) | |
| c.set(i, 16, x=-0.3 * pk) | |
| else: | |
| fold(i) | |
| c.set(i, 2, y=0.5 * np.sin(6.283 * 0.25 * tt + ph0) * (np.sin(6.283 * 0.1 * tt) > 0)) | |
| c.set(i, 3, z=0.2 * np.sin(6.283 * 0.4 * tt)); c.set(i, 16, x=0.15 * max(0, np.sin(6.283 * 1.0 * tt))) | |
| c.set(i, 1, x=0.02 * np.sin(6.283 * 0.5 * tt)) | |
| return c.X | |
| # ------------------------------------------------------------------ fish | |
| def fish(a, r): | |
| c = Clip("fish"); u = r.uniform; ph0 = u(0, 6.28) | |
| if a == "swim": f, A, v = u(1, 2), u(0.8, 1.1), u(1.0, 1.6) | |
| elif a == "dart": f, A, v = u(3, 4), u(1.1, 1.4), u(4, 6) | |
| else: f, A, v = u(0.3, 0.5), u(0.3, 0.5), u(0, 0.08) | |
| for i, tt in enumerate(c.t): | |
| ph = 6.283 * f * tt + ph0 | |
| c.set(i, 1, y=-0.06 * A * np.sin(ph + 0.6)); c.set(i, 2, y=-0.05 * A * np.sin(ph + 0.6)) | |
| c.set(i, 3, y=0.15 * A * np.sin(ph)); c.set(i, 4, y=0.3 * A * np.sin(ph - 0.8)); c.set(i, 5, y=0.45 * A * np.sin(ph - 1.6)) | |
| c.set(i, 6, z=0.3 * np.sin(ph * 0.7), y=0.2); c.set(i, 7, z=-0.3 * np.sin(ph * 0.7), y=-0.2) | |
| c.root(i, DY, 0.05 * np.sin(6.283 * 0.3 * tt + ph0)); c.root(i, VZ, v / FPS) | |
| return c.X | |
| # ------------------------------------------------------------------ snake | |
| def snake(a, r): | |
| c = Clip("snake"); u = r.uniform; ph0 = u(0, 6.28) | |
| f, A, v = (u(0.7, 1.1), u(0.35, 0.5), u(3, 5)) if a == "slither" else (u(0.2, 0.3), u(0.08, 0.15), 0) | |
| for i, tt in enumerate(c.t): | |
| ph = 6.283 * f * tt + ph0 | |
| c.set(i, 0, y=-0.4 * A * np.sin(ph + 0.9)) | |
| for j in range(1, 10): c.set(i, j, y=A * np.sin(ph - j * 0.9)) | |
| c.root(i, VZ, v / FPS) | |
| return c.X | |
| # ------------------------------------------------------------------ insect | |
| def insect(a, r): | |
| c = Clip("insect"); u = r.uniform; ph0 = u(0, 6.28) | |
| legs = [(3, 1, 0), (6, -1, 1), (9, 1, 1), (12, -1, 0), (15, 1, 0), (18, -1, 1)] # (hip, side, tripod group) | |
| if a == "crawl": f, A, L, v = u(2, 3), u(0.35, 0.5), u(0.3, 0.45), u(2, 3) | |
| elif a == "run": f, A, L, v = u(5, 6.5), u(0.4, 0.55), u(0.3, 0.4), u(6, 8) | |
| else: f, A, L, v = 0, 0, 0, 0 | |
| for i, tt in enumerate(c.t): | |
| ph = 6.283 * f * tt + ph0 | |
| for h, s, g in legs: | |
| p = ph + np.pi * g | |
| c.set(i, h, y=-s * A * np.sin(p), z=s * L * max(0, np.cos(p))) | |
| c.set(i, h + 1, z=-s * 0.2 * L * max(0, np.cos(p))) | |
| if a == "idle": | |
| c.set(i, 1, y=0.25 * np.sin(6.283 * 0.3 * tt + ph0)); c.set(i, 2, x=0.06 * np.sin(6.283 * 0.8 * tt)) | |
| for h, s, g in legs: c.set(i, h, z=s * 0.05 * np.sin(6.283 * 0.5 * tt + h)) | |
| c.root(i, DY, 0.01 * np.sin(2 * ph)); c.root(i, VZ, v / FPS) | |
| return c.X | |
| GEN = {"biped": biped, "quadruped": quadruped, "bird": bird, "fish": fish, "snake": snake, "insect": insect} | |
| def clip(prompt, r): | |
| f, a = prompt.split(":") | |
| X = GEN[f](a, r) | |
| X[:, :FAM[f][0] * 3] += r.normal(0, 0.01, (CLIP, FAM[f][0] * 3)) | |
| return X | |
| # ------------------------------------------------------------------ tokenizer (per body plan) | |
| class Tok: | |
| def fit(self, X, k=K, iters=25, seed=0): | |
| rng = np.random.default_rng(seed) | |
| self.mu, self.sd = X.mean(0), np.maximum(X.std(0), 0.15) | |
| Z = (X - self.mu) / self.sd | |
| C = Z[rng.choice(len(Z), k, replace=False)].copy() | |
| for _ in range(iters): | |
| a = self.assign(Z, C) | |
| for j in range(k): | |
| m = a == j | |
| C[j] = Z[m].mean(0) if m.any() else Z[rng.integers(len(Z))] | |
| self.C = C; return self | |
| def assign(Z, C): | |
| return ((Z ** 2).sum(1)[:, None] - 2 * Z @ C.T + (C ** 2).sum(1)[None]).argmin(1) | |
| def encode(self, X): return self.assign((X - self.mu) / self.sd, self.C) | |
| def decode(self, t): return self.C[t] * self.sd + self.mu | |
| # ------------------------------------------------------------------ GPT (same design as v1) | |
| class Block: | |
| def __init__(s, d, h): | |
| s.h = h | |
| s.l1w, s.l1b, s.l2w, s.l2b = Param(d, init=1.0), Param(d, init=0.0), Param(d, init=1.0), Param(d, init=0.0) | |
| s.q, s.k, s.v, s.o, s.f1, s.f2 = Linear(d, d), Linear(d, d), Linear(d, d), Linear(d, d), Linear(d, 4 * d), Linear(4 * d, d) | |
| for p in s.o.params + s.f2.params: p.data *= 0.3 | |
| s.params = [s.l1w, s.l1b, s.l2w, s.l2b] + [p for m in (s.q, s.k, s.v, s.o, s.f1, s.f2) for p in m.params] | |
| def __call__(s, x, mask): | |
| B, L, d = x.shape; h, dh = s.h, d // s.h | |
| a = x.layernorm(s.l1w, s.l1b) | |
| sp = lambda t: t.reshape(B, L, h, dh).transpose(0, 2, 1, 3) | |
| q, k, v = sp(s.q(a)), sp(s.k(a)), sp(s.v(a)) | |
| att = ((q @ k.transpose(0, 1, 3, 2)) * (1 / np.sqrt(dh)) + mask).softmax(-1) | |
| x = x + s.o((att @ v).transpose(0, 2, 1, 3).reshape(B, L, d)) | |
| return x + s.f2(s.f1(x.layernorm(s.l2w, s.l2b)).gelu()) | |
| class GPT: | |
| def __init__(s, d=96, h=4, layers=4, seed=0): | |
| np.random.seed(seed) | |
| s.tok, s.pos = Param(V, d, scale=0.05), Param(CLIP, d, scale=0.02) | |
| s.blocks = [Block(d, h) for _ in range(layers)] | |
| s.lnw, s.lnb = Param(d, init=1.0), Param(d, init=0.0) | |
| s.head = Linear(d, V) | |
| s.params = [s.tok, s.pos, s.lnw, s.lnb] + [p for b in s.blocks for p in b.params] + s.head.params | |
| s.cfg = dict(d=d, h=h, layers=layers) | |
| def __call__(s, idx): | |
| B, L = idx.shape | |
| x = embed(s.tok, idx) + embed(s.pos, np.arange(L)) | |
| mask = T(np.triu(np.full((L, L), -1e9), 1)) | |
| for b in s.blocks: x = b(x, mask) | |
| return s.head(x.layernorm(s.lnw, s.lnb)) | |
| def make_batch(rng, toks, B): | |
| ps = rng.integers(0, len(PROMPTS), B) | |
| seqs = [] | |
| for p in ps: | |
| f = PROMPTS[p].split(":")[0]; fi = FAMS.index(f) | |
| t = toks[f].encode(clip(PROMPTS[p], rng)) + fi * K | |
| seqs.append(np.concatenate([[K * len(FAMS) + p], t])) | |
| return np.array(seqs) | |
| def train(steps=3000, B=24): | |
| rng = np.random.default_rng(0) | |
| toks = {} | |
| for f in FAMS: | |
| acts = FAM[f][1] | |
| X = np.concatenate([clip(f"{f}:{a}", rng) for a in acts for _ in range(250)]) | |
| toks[f] = Tok().fit(X) | |
| rec = toks[f].decode(toks[f].encode(X)) | |
| print(f, "tokenizer err", np.abs(rec - X)[:, :FAM[f][0] * 3].mean(), flush=True) | |
| vr = np.random.default_rng(123) | |
| val = make_batch(vr, toks, 96) | |
| m = GPT(); opt = Adam(m.params, lr=1e-3, wd=1e-4) | |
| log = []; t0 = time.time() | |
| for it in range(steps): | |
| s = make_batch(rng, toks, B) | |
| loss = cross_entropy(m(s[:, :-1]), s[:, 1:]); loss.backward() | |
| opt.step(1e-3 * min(1, (it + 1) / 200) * 0.5 * (1 + np.cos(np.pi * it / steps)) + 1e-5) | |
| if it % 100 == 0 or it == steps - 1: | |
| vl = float(cross_entropy(m(val[:, :-1]), val[:, 1:]).data) | |
| log.append((it, float(loss.data), vl)) | |
| print(f"{it} train {loss.data:.3f} val {vl:.3f} {time.time()-t0:.0f}s", flush=True) | |
| if it % 500 == 0: save(m, toks, log) | |
| save(m, toks, log) | |
| def save(m, toks, log): | |
| np.savez("gpt2.npz", *[p.data for p in m.params]) | |
| np.savez("tok2.npz", **{f"{f}_{k}": getattr(toks[f], k) for f in FAMS for k in ("mu", "sd", "C")}) | |
| np.save("gpt2_log.npy", np.array(log)) | |
| if __name__ == "__main__": | |
| train(int(sys.argv[1]) if len(sys.argv) > 1 else 3000) | |