WVY-3D-1M / code /python /motion2.py
StarpowerTechnology's picture
Release WVY-3D-1M v1
7207c65 verified
Raw History Blame Contribute Delete
17.8 kB
"""MotionGPT v2: one language model that writes motion for six body plans.
Vocabulary = 6 x 256 pose words (one dictionary per body plan) + 25 prompt words like 'quadruped:run'."""
import numpy as np, time, sys, json
from tinygrad import T, Param, Linear, Adam, cross_entropy, embed
FPS, CLIP, K = 15, 64, 256
FAM = { # joint counts and actions; order matters (token ranges)
"biped": (17, ["idle", "walk", "run", "wave", "jump", "dance"]),
"quadruped": (19, ["idle", "walk", "run", "jump", "sit", "eat"]),
"bird": (17, ["idle", "hop", "peck", "fly", "glide"]),
"fish": (8, ["idle", "swim", "dart"]),
"snake": (10, ["idle", "slither"]),
"insect": (21, ["idle", "crawl", "run"]),
}
FAMS = list(FAM)
PROMPTS = [f"{f}:{a}" for f in FAMS for a in FAM[f][1]]
V = K * len(FAMS) + len(PROMPTS)
dim = lambda f: FAM[f][0] * 3 + 4
SX, DY, VZ, YAW = 0, 1, 2, 3 # root channel offsets (after joint eulers)
class Clip:
def __init__(self, fam):
self.f = fam; self.J = FAM[fam][0]
self.X = np.zeros((CLIP, dim(fam)))
self.t = np.arange(CLIP) / FPS
def set(self, i, j, x=None, y=None, z=None):
if x is not None: self.X[i, j * 3] = x
if y is not None: self.X[i, j * 3 + 1] = y
if z is not None: self.X[i, j * 3 + 2] = z
def root(self, i, c, v):
self.X[i, self.J * 3 + c] = v
def ease(q, kf):
kf = np.array(kf, float)
return np.interp(q, kf[:, 0], kf[:, 1])
# ------------------------------------------------------------------ biped
def biped(a, r):
c = Clip("biped"); u = r.uniform; ph0 = u(0, 6.28)
for i, tt in enumerate(c.t):
if a in ("walk", "run"):
run = a == "run"
if i == 0:
f = u(1.35, 1.6) if run else u(0.85, 1.15); A = u(0.6, 0.8) if run else u(0.3, 0.5)
Kn = u(1.3, 1.7) if run else u(0.6, 0.9); S = u(0.6, 0.9) if run else u(0.25, 0.45)
E = u(1.3, 1.6) if run else u(0.2, 0.4); lean = u(0.15, 0.3) if run else u(0, 0.06)
v = (u(2.2, 2.8) if run else u(0.8, 1.1)) / FPS
ph = 6.283 * f * tt + ph0
c.set(i, 11, x=-A * np.sin(ph)); c.set(i, 14, x=A * np.sin(ph))
c.set(i, 12, x=Kn * max(0, np.cos(ph)) + 0.05); c.set(i, 15, x=Kn * max(0, -np.cos(ph)) + 0.05)
c.set(i, 13, x=-0.2 * np.sin(ph)); c.set(i, 16, x=0.2 * np.sin(ph))
c.set(i, 5, x=S * np.sin(ph), z=-1.3); c.set(i, 8, x=-S * np.sin(ph), z=1.3)
c.set(i, 6, y=-E); c.set(i, 9, y=E)
c.set(i, 1, x=lean, y=0.12 * np.sin(ph)); c.set(i, 0, y=-0.1 * np.sin(ph)); c.set(i, 3, x=-lean * 0.5)
c.root(i, SX, 0.015 * np.sin(ph)); c.root(i, DY, (0.06 if run else 0.025) * np.cos(2 * ph) - (0.06 if run else 0)); c.root(i, VZ, v)
elif a == "wave":
if i == 0: w = u(2, 3); amp = u(0.35, 0.6); rz = u(0.95, 1.3)
s = np.sin(6.283 * w * tt + ph0)
c.set(i, 8, z=-rz, x=-0.2); c.set(i, 9, z=amp * s - 0.2, y=0.3)
c.set(i, 5, z=-1.35, x=0.05); c.set(i, 6, y=-0.25)
c.set(i, 1, z=0.06 * s); c.set(i, 4, z=-0.12 + 0.05 * s, x=0.05)
c.set(i, 12, x=0.05); c.set(i, 15, x=0.05)
elif a == "jump":
if i == 0: P = u(1.3, 1.7); H = u(0.25, 0.45)
q = (tt / P + ph0 / 6.283) % 1
cr = ease(q, [[0, 0], [0.3, 1], [0.42, 0], [0.58, 0.25], [0.74, 0.1], [0.84, 0.8], [1, 0]])
hg = np.sin(np.clip((q - 0.42) / 0.32, 0, 1) * np.pi) * H
arm = ease(q, [[0, 0], [0.3, 0.9], [0.42, -1.2], [0.58, -1.1], [0.74, -0.5], [0.84, 0.3], [1, 0]])
for hp, kn, an in ((11, 12, 13), (14, 15, 16)):
c.set(i, hp, x=-0.9 * cr); c.set(i, kn, x=1.7 * cr + 0.5 * hg / H); c.set(i, an, x=-0.75 * cr)
c.set(i, 1, x=0.45 * cr); c.set(i, 3, x=-0.3 * cr)
c.set(i, 5, x=arm, z=-1.3 + 0.2 * cr); c.set(i, 8, x=arm, z=1.3 - 0.2 * cr)
c.root(i, DY, -0.32 * cr + hg)
elif a == "dance":
if i == 0: f = u(0.9, 1.2); spin = u(-0.6, 0.6) / FPS
ph = 6.283 * f * tt + ph0; b = 0.5 * (1 + np.cos(2 * ph))
c.set(i, 0, z=0.18 * np.sin(ph)); c.set(i, 1, z=-0.22 * np.sin(ph)); c.set(i, 4, z=0.15 * np.sin(ph + 0.5), x=0.1 * b)
ul, ur = max(0, np.sin(ph)), max(0, -np.sin(ph))
c.set(i, 5, z=-1.2 + 2.4 * ul, x=-0.3); c.set(i, 8, z=1.2 - 2.4 * ur, x=-0.3)
c.set(i, 6, y=-1.2 + 0.8 * ul); c.set(i, 9, y=1.2 - 0.8 * ur)
for hp, kn, an in ((11, 12, 13), (14, 15, 16)):
c.set(i, hp, x=-0.35 * b); c.set(i, kn, x=0.7 * b); c.set(i, an, x=-0.35 * b)
c.root(i, SX, 0.05 * np.sin(ph)); c.root(i, DY, -0.09 * b); c.root(i, YAW, spin)
else: # idle
if i == 0: lk = u(0.15, 0.3); fr = u(0.15, 0.3)
br = np.sin(6.283 * 0.3 * tt + ph0)
c.set(i, 5, z=-1.3 + 0.03 * br); c.set(i, 8, z=1.3 - 0.03 * br); c.set(i, 6, y=-0.15); c.set(i, 9, y=0.15)
c.set(i, 2, x=0.02 * br); c.set(i, 3, y=lk * np.sin(6.283 * fr * tt + ph0))
c.set(i, 0, z=0.03 * np.sin(6.283 * 0.2 * tt))
return c.X
# ------------------------------------------------------------------ quadruped
def quadruped(a, r):
c = Clip("quadruped"); u = r.uniform; ph0 = u(0, 6.28)
legs = [(11, 12, 13), (5, 6, 7), (14, 15, 16), (8, 9, 10)] # BL, FL, BR, FR
for i, tt in enumerate(c.t):
if a in ("walk", "run"):
run = a == "run"
if i == 0:
f = u(1.8, 2.4) if run else u(0.8, 1.2); A = u(0.55, 0.75) if run else u(0.3, 0.45)
Kn = u(1.0, 1.3) if run else u(0.5, 0.8); v = (u(4, 5) if run else u(1.2, 1.6)) / FPS
offs = [0, 0.5, 0.1, 0.6] if run else [0, 0.25, 0.5, 0.75]
ph = 6.283 * f * tt + ph0
for (h, k, ft), o in zip(legs, offs):
p = ph + 6.283 * o
c.set(i, h, x=-A * np.sin(p)); c.set(i, k, x=Kn * max(0, np.cos(p)))
c.set(i, 1, x=(0.15 if run else 0.03) * np.sin(ph), y=0.06 * np.sin(ph))
c.set(i, 3, x=0.06 * np.sin(2 * ph)); c.set(i, 17, y=0.35 * np.sin(ph), x=-0.2 if run else 0)
c.root(i, DY, (0.07 if run else 0.02) * np.cos(2 * ph)); c.root(i, VZ, v)
elif a == "jump":
if i == 0: P = u(1.4, 1.8); H = u(0.4, 0.7)
q = (tt / P + ph0 / 6.283) % 1
cr = ease(q, [[0, 0], [0.3, 1], [0.4, 0], [0.75, 0], [0.85, 0.7], [1, 0]])
air = np.sin(np.clip((q - 0.38) / 0.37, 0, 1) * np.pi)
for h, k, ft in legs:
front = h in (5, 8)
c.set(i, h, x=-0.5 * cr + (-0.9 if front else 0.7) * air); c.set(i, k, x=1.0 * cr + 0.3 * air)
c.set(i, 0, x=-0.25 * air * np.cos(np.clip((q - 0.38) / 0.37, 0, 1) * np.pi))
c.set(i, 3, x=-0.3 * air); c.set(i, 17, x=-0.5 * air)
c.root(i, DY, -0.25 * cr + H * air); c.root(i, VZ, (2.5 / FPS) * (air > 0.02))
elif a == "sit":
if i == 0: pt = u(0.6, 0.8)
br = np.sin(6.283 * 0.3 * tt + ph0)
c.set(i, 0, x=-pt)
for h, k, ft in (legs[0], legs[2]): c.set(i, h, x=pt - 1.3); c.set(i, k, x=2.2)
for h, k, ft in (legs[1], legs[3]): c.set(i, h, x=pt)
c.set(i, 3, x=pt * 0.7 + 0.03 * br, y=0.3 * np.sin(6.283 * 0.15 * tt + ph0))
c.set(i, 17, x=pt + 0.4, y=0.4 * np.sin(6.283 * 1.2 * tt))
elif a == "eat":
ch = np.sin(6.283 * 2.5 * tt + ph0)
c.set(i, 3, x=0.95 + 0.05 * ch); c.set(i, 4, x=0.2 + 0.05 * ch)
c.set(i, 5, x=-0.15); c.set(i, 8, x=-0.15); c.set(i, 1, x=0.05)
c.set(i, 17, y=0.3 * np.sin(6.283 * 0.8 * tt))
else:
br = np.sin(6.283 * 0.3 * tt + ph0)
c.set(i, 2, x=0.02 * br); c.set(i, 3, x=-0.05 + 0.03 * br, y=0.35 * np.sin(6.283 * 0.12 * tt + ph0))
c.set(i, 4, z=0.1 * np.sin(6.283 * 0.2 * tt))
c.set(i, 17, y=0.6 * np.sin(6.283 * 1.6 * tt + ph0)); c.set(i, 18, y=0.4 * np.sin(6.283 * 1.6 * tt + ph0 - 0.8))
return c.X
# ------------------------------------------------------------------ bird
def bird(a, r):
c = Clip("bird"); u = r.uniform; ph0 = u(0, 6.28)
def fold(i, open_=0.0):
c.set(i, 4, y=1.25 * (1 - open_), z=-0.3 * (1 - open_)); c.set(i, 5, y=0.5 * (1 - open_))
c.set(i, 7, y=-1.25 * (1 - open_), z=0.3 * (1 - open_)); c.set(i, 8, y=-0.5 * (1 - open_))
for i, tt in enumerate(c.t):
if a == "fly":
if i == 0: f = u(2, 3.5); A = u(0.45, 0.65); alt = u(3, 6); v = u(3, 5) / FPS
ph = 6.283 * f * tt + ph0
c.set(i, 4, z=A * np.sin(ph)); c.set(i, 5, z=0.3 * A * np.sin(ph - 0.6)); c.set(i, 6, z=0.15 * A * np.sin(ph - 1.2))
c.set(i, 7, z=-A * np.sin(ph)); c.set(i, 8, z=-0.3 * A * np.sin(ph - 0.6)); c.set(i, 9, z=-0.15 * A * np.sin(ph - 1.2))
for th, sh in ((10, 11), (13, 14)): c.set(i, th, x=1.3); c.set(i, sh, x=1.0)
c.set(i, 16, x=-0.1 * np.sin(ph)); c.set(i, 2, x=0.2)
c.root(i, DY, alt + 0.15 * np.sin(ph)); c.root(i, VZ, v)
elif a == "glide":
if i == 0: alt = u(4, 7); v = u(3, 4.5) / FPS; turn = u(-0.4, 0.4) / FPS
s = np.sin(6.283 * 0.3 * tt + ph0)
c.set(i, 4, z=0.08 + 0.05 * s); c.set(i, 7, z=-0.08 - 0.05 * s); c.set(i, 0, z=turn * FPS * 0.6)
for th, sh in ((10, 11), (13, 14)): c.set(i, th, x=1.3); c.set(i, sh, x=1.0)
c.root(i, DY, alt + 0.2 * s); c.root(i, VZ, v); c.root(i, YAW, turn)
elif a == "hop":
if i == 0: P = u(0.5, 0.8); H = u(0.3, 0.6)
q = (tt / P + ph0 / 6.283) % 1
cr = ease(q, [[0, 0.2], [0.25, 1], [0.35, 0], [0.75, 0], [0.85, 0.8], [1, 0.2]])
air = np.sin(np.clip((q - 0.33) / 0.42, 0, 1) * np.pi)
fold(i, 0.25 * air)
for th, sh, ft in ((10, 11, 12), (13, 14, 15)): c.set(i, th, x=-0.6 * cr); c.set(i, sh, x=1.0 * cr); c.set(i, ft, x=-0.4 * cr)
c.set(i, 2, x=-0.2 * air); c.root(i, DY, -0.15 * cr + H * air); c.root(i, VZ, (1.2 / FPS) * (air > 0.02))
elif a == "peck":
if i == 0: P = u(0.6, 1.0)
q = (tt / P + ph0 / 6.283) % 1
pk = ease(q, [[0, 0], [0.4, 0], [0.55, 1], [0.7, 0.2], [0.8, 1], [1, 0]])
fold(i)
c.set(i, 0, x=0.35 * pk); c.set(i, 2, x=0.9 * pk)
for th in (10, 13): c.set(i, th, x=-0.35 * pk)
c.set(i, 16, x=-0.3 * pk)
else:
fold(i)
c.set(i, 2, y=0.5 * np.sin(6.283 * 0.25 * tt + ph0) * (np.sin(6.283 * 0.1 * tt) > 0))
c.set(i, 3, z=0.2 * np.sin(6.283 * 0.4 * tt)); c.set(i, 16, x=0.15 * max(0, np.sin(6.283 * 1.0 * tt)))
c.set(i, 1, x=0.02 * np.sin(6.283 * 0.5 * tt))
return c.X
# ------------------------------------------------------------------ fish
def fish(a, r):
c = Clip("fish"); u = r.uniform; ph0 = u(0, 6.28)
if a == "swim": f, A, v = u(1, 2), u(0.8, 1.1), u(1.0, 1.6)
elif a == "dart": f, A, v = u(3, 4), u(1.1, 1.4), u(4, 6)
else: f, A, v = u(0.3, 0.5), u(0.3, 0.5), u(0, 0.08)
for i, tt in enumerate(c.t):
ph = 6.283 * f * tt + ph0
c.set(i, 1, y=-0.06 * A * np.sin(ph + 0.6)); c.set(i, 2, y=-0.05 * A * np.sin(ph + 0.6))
c.set(i, 3, y=0.15 * A * np.sin(ph)); c.set(i, 4, y=0.3 * A * np.sin(ph - 0.8)); c.set(i, 5, y=0.45 * A * np.sin(ph - 1.6))
c.set(i, 6, z=0.3 * np.sin(ph * 0.7), y=0.2); c.set(i, 7, z=-0.3 * np.sin(ph * 0.7), y=-0.2)
c.root(i, DY, 0.05 * np.sin(6.283 * 0.3 * tt + ph0)); c.root(i, VZ, v / FPS)
return c.X
# ------------------------------------------------------------------ snake
def snake(a, r):
c = Clip("snake"); u = r.uniform; ph0 = u(0, 6.28)
f, A, v = (u(0.7, 1.1), u(0.35, 0.5), u(3, 5)) if a == "slither" else (u(0.2, 0.3), u(0.08, 0.15), 0)
for i, tt in enumerate(c.t):
ph = 6.283 * f * tt + ph0
c.set(i, 0, y=-0.4 * A * np.sin(ph + 0.9))
for j in range(1, 10): c.set(i, j, y=A * np.sin(ph - j * 0.9))
c.root(i, VZ, v / FPS)
return c.X
# ------------------------------------------------------------------ insect
def insect(a, r):
c = Clip("insect"); u = r.uniform; ph0 = u(0, 6.28)
legs = [(3, 1, 0), (6, -1, 1), (9, 1, 1), (12, -1, 0), (15, 1, 0), (18, -1, 1)] # (hip, side, tripod group)
if a == "crawl": f, A, L, v = u(2, 3), u(0.35, 0.5), u(0.3, 0.45), u(2, 3)
elif a == "run": f, A, L, v = u(5, 6.5), u(0.4, 0.55), u(0.3, 0.4), u(6, 8)
else: f, A, L, v = 0, 0, 0, 0
for i, tt in enumerate(c.t):
ph = 6.283 * f * tt + ph0
for h, s, g in legs:
p = ph + np.pi * g
c.set(i, h, y=-s * A * np.sin(p), z=s * L * max(0, np.cos(p)))
c.set(i, h + 1, z=-s * 0.2 * L * max(0, np.cos(p)))
if a == "idle":
c.set(i, 1, y=0.25 * np.sin(6.283 * 0.3 * tt + ph0)); c.set(i, 2, x=0.06 * np.sin(6.283 * 0.8 * tt))
for h, s, g in legs: c.set(i, h, z=s * 0.05 * np.sin(6.283 * 0.5 * tt + h))
c.root(i, DY, 0.01 * np.sin(2 * ph)); c.root(i, VZ, v / FPS)
return c.X
GEN = {"biped": biped, "quadruped": quadruped, "bird": bird, "fish": fish, "snake": snake, "insect": insect}
def clip(prompt, r):
f, a = prompt.split(":")
X = GEN[f](a, r)
X[:, :FAM[f][0] * 3] += r.normal(0, 0.01, (CLIP, FAM[f][0] * 3))
return X
# ------------------------------------------------------------------ tokenizer (per body plan)
class Tok:
def fit(self, X, k=K, iters=25, seed=0):
rng = np.random.default_rng(seed)
self.mu, self.sd = X.mean(0), np.maximum(X.std(0), 0.15)
Z = (X - self.mu) / self.sd
C = Z[rng.choice(len(Z), k, replace=False)].copy()
for _ in range(iters):
a = self.assign(Z, C)
for j in range(k):
m = a == j
C[j] = Z[m].mean(0) if m.any() else Z[rng.integers(len(Z))]
self.C = C; return self
@staticmethod
def assign(Z, C):
return ((Z ** 2).sum(1)[:, None] - 2 * Z @ C.T + (C ** 2).sum(1)[None]).argmin(1)
def encode(self, X): return self.assign((X - self.mu) / self.sd, self.C)
def decode(self, t): return self.C[t] * self.sd + self.mu
# ------------------------------------------------------------------ GPT (same design as v1)
class Block:
def __init__(s, d, h):
s.h = h
s.l1w, s.l1b, s.l2w, s.l2b = Param(d, init=1.0), Param(d, init=0.0), Param(d, init=1.0), Param(d, init=0.0)
s.q, s.k, s.v, s.o, s.f1, s.f2 = Linear(d, d), Linear(d, d), Linear(d, d), Linear(d, d), Linear(d, 4 * d), Linear(4 * d, d)
for p in s.o.params + s.f2.params: p.data *= 0.3
s.params = [s.l1w, s.l1b, s.l2w, s.l2b] + [p for m in (s.q, s.k, s.v, s.o, s.f1, s.f2) for p in m.params]
def __call__(s, x, mask):
B, L, d = x.shape; h, dh = s.h, d // s.h
a = x.layernorm(s.l1w, s.l1b)
sp = lambda t: t.reshape(B, L, h, dh).transpose(0, 2, 1, 3)
q, k, v = sp(s.q(a)), sp(s.k(a)), sp(s.v(a))
att = ((q @ k.transpose(0, 1, 3, 2)) * (1 / np.sqrt(dh)) + mask).softmax(-1)
x = x + s.o((att @ v).transpose(0, 2, 1, 3).reshape(B, L, d))
return x + s.f2(s.f1(x.layernorm(s.l2w, s.l2b)).gelu())
class GPT:
def __init__(s, d=96, h=4, layers=4, seed=0):
np.random.seed(seed)
s.tok, s.pos = Param(V, d, scale=0.05), Param(CLIP, d, scale=0.02)
s.blocks = [Block(d, h) for _ in range(layers)]
s.lnw, s.lnb = Param(d, init=1.0), Param(d, init=0.0)
s.head = Linear(d, V)
s.params = [s.tok, s.pos, s.lnw, s.lnb] + [p for b in s.blocks for p in b.params] + s.head.params
s.cfg = dict(d=d, h=h, layers=layers)
def __call__(s, idx):
B, L = idx.shape
x = embed(s.tok, idx) + embed(s.pos, np.arange(L))
mask = T(np.triu(np.full((L, L), -1e9), 1))
for b in s.blocks: x = b(x, mask)
return s.head(x.layernorm(s.lnw, s.lnb))
def make_batch(rng, toks, B):
ps = rng.integers(0, len(PROMPTS), B)
seqs = []
for p in ps:
f = PROMPTS[p].split(":")[0]; fi = FAMS.index(f)
t = toks[f].encode(clip(PROMPTS[p], rng)) + fi * K
seqs.append(np.concatenate([[K * len(FAMS) + p], t]))
return np.array(seqs)
def train(steps=3000, B=24):
rng = np.random.default_rng(0)
toks = {}
for f in FAMS:
acts = FAM[f][1]
X = np.concatenate([clip(f"{f}:{a}", rng) for a in acts for _ in range(250)])
toks[f] = Tok().fit(X)
rec = toks[f].decode(toks[f].encode(X))
print(f, "tokenizer err", np.abs(rec - X)[:, :FAM[f][0] * 3].mean(), flush=True)
vr = np.random.default_rng(123)
val = make_batch(vr, toks, 96)
m = GPT(); opt = Adam(m.params, lr=1e-3, wd=1e-4)
log = []; t0 = time.time()
for it in range(steps):
s = make_batch(rng, toks, B)
loss = cross_entropy(m(s[:, :-1]), s[:, 1:]); loss.backward()
opt.step(1e-3 * min(1, (it + 1) / 200) * 0.5 * (1 + np.cos(np.pi * it / steps)) + 1e-5)
if it % 100 == 0 or it == steps - 1:
vl = float(cross_entropy(m(val[:, :-1]), val[:, 1:]).data)
log.append((it, float(loss.data), vl))
print(f"{it} train {loss.data:.3f} val {vl:.3f} {time.time()-t0:.0f}s", flush=True)
if it % 500 == 0: save(m, toks, log)
save(m, toks, log)
def save(m, toks, log):
np.savez("gpt2.npz", *[p.data for p in m.params])
np.savez("tok2.npz", **{f"{f}_{k}": getattr(toks[f], k) for f in FAMS for k in ("mu", "sd", "C")})
np.save("gpt2_log.npy", np.array(log))
if __name__ == "__main__":
train(int(sys.argv[1]) if len(sys.argv) > 1 else 3000)