File size: 14,687 Bytes
c7ae99a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
"""Real-time, endless text-to-motion with the PartLock CMU model.

The model keeps generating forever; the browser plays it in real time.

Typing a prompt and pressing the button swaps the text at the next step (4 frames), without resetting the context.



    python live/live_server.py          then open http://localhost:8765



Options (environment variables): LIVE_PORT (8765), LIVE_LEAD (6 steps generated ahead of playback = 0.8 s).

Generation is kept only LEAD steps ahead of playback, so a new prompt shows up after about LEAD*4/30 s."""
import os, sys, json, time, threading
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs
import numpy as np, torch
import torch.nn.functional as F
from safetensors.torch import load_file
HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
sys.path.insert(0, os.path.join(REPO, "code"))
import tvq272 as B, tvq272c as C, gen_ar as G
from gen_sample import load, embed, get_codebook

dev = "cuda"
WTS = os.path.join(REPO, "weights")
PORT = int(os.environ.get("LIVE_PORT", "8765"))
LEAD = int(os.environ.get("LIVE_LEAD", "6"))     # steps generated ahead of playback (6 steps = 0.8 s)
DEC_CTX = 32                                      # decoder receptive field is ~16 steps; 32 is safe
KEEP = 30 * 120                                   # frames kept on the server (2 min)

mean = torch.from_numpy(np.load(os.path.join(WTS, "Mean.npy")).astype(np.float32)).to(dev)
std = torch.from_numpy(np.load(os.path.join(WTS, "Std.npy")).astype(np.float32)).to(dev)
h = C.CHierTok().to(dev); h.load_state_dict(load_file(os.path.join(WTS, "tokenizer.safetensors"), device=dev)); h.eval()
g = G.Gen().to(dev); g.load_state_dict(load_file(os.path.join(WTS, "generator.safetensors"), device=dev)); g.eval()
tok, clip = load()
CB = [get_codebook(h.gvq)] + [get_codebook(v) for v in h.vq]
OFFT = torch.tensor(G.OFF, device=dev)
SLOTS = torch.arange(G.L, device=dev) % 7


class State:
    def __init__(self):
        self.lock = threading.Lock()
        self.frames = []          # list of dict per frame (json-ready)
        self.base = 0             # global index of frames[0]
        self.total = 0            # frames produced so far
        self.play = 0             # browser playback frame (reported by polling)
        self.prompt = "Standing still"
        self.pending = None
        self.reset = False
        self.s = 4.0
        self.step_ms = 0.0
        self.log = []             # (frame, prompt)


S = State()


def reset_world():
    return dict(codes=torch.zeros(0, 7, dtype=torch.long, device=dev), toks=torch.zeros(1, 0, dtype=torch.long, device=dev),
                tot=None, inv_prev=None, tr=torch.zeros(3, dtype=torch.float64))


def _attn_in(b, x, pos):
    Bn, T, D = x.shape
    q, k, v = b.qkv(b.n1(x)).view(Bn, T, 3, b.h, D // b.h).permute(2, 0, 3, 1, 4)
    return G.rope(q, pos), G.rope(k, pos), v


def _attn_out(b, x, a):
    Bn, T, D = x.shape
    x = x + b.o(a.transpose(1, 2).reshape(Bn, T, D))
    return x + b.ff(b.n2(x))


LMAX = G.L        # 448 = BOS + 63 past steps + 7 tokens of the current step (most the original ever feeds)


class Engine:
    """CUDA-graph version of the same forward. Static K/V buffers of length LMAX.

    full(): window padded to LMAX at the END (causal -> real positions are unaffected), logits taken at T-1.

    one(): one new token at position p, attends to positions <= p."""
    def __init__(self):
        nh = g.blocks[0].h; dh = 512 // nh
        self.K = [torch.zeros(2, nh, LMAX, dh, device=dev, dtype=torch.bfloat16) for _ in g.blocks]
        self.V = [torch.zeros_like(k) for k in self.K]
        self.ids = torch.full((2, LMAX), G.NV, device=dev, dtype=torch.long)
        self.tc = torch.zeros(2, 1, 512, device=dev)
        self.last = torch.zeros((), device=dev, dtype=torch.long)       # index T-1
        self.tid = torch.zeros(2, 1, device=dev, dtype=torch.long)
        self.p = torch.zeros(1, device=dev, dtype=torch.long)
        self.ar = torch.arange(LMAX, device=dev)
        s = torch.cuda.Stream(); s.wait_stream(torch.cuda.current_stream())
        with torch.cuda.stream(s):
            for _ in range(3):
                self._full(); self._one()
        torch.cuda.current_stream().wait_stream(s)
        self.gf = torch.cuda.CUDAGraph()
        with torch.cuda.graph(self.gf):
            self.out_f = self._full()
        self.go = torch.cuda.CUDAGraph()
        with torch.cuda.graph(self.go):
            self.out_o = self._one()

    @torch.no_grad()
    def _full(self):
        with torch.autocast("cuda", dtype=torch.bfloat16):
            x = g.tok(self.ids) + g.slot(SLOTS)[None] + g.txt(self.tc)
            pos = self.ar
            for b, K, V in zip(g.blocks, self.K, self.V):
                q, k, v = _attn_in(b, x, pos)
                K.copy_(k); V.copy_(v)
                x = _attn_out(b, x, F.scaled_dot_product_attention(q, k, v, is_causal=True))
            xl = x.index_select(1, self.last.view(1))
            return (g.head(g.nf(xl)) + g.mask[SLOTS.index_select(0, self.last.view(1))][None]).float()[:, 0]

    @torch.no_grad()
    def _one(self):
        with torch.autocast("cuda", dtype=torch.bfloat16):
            sl = SLOTS.index_select(0, self.p)
            x = g.tok(self.tid) + g.slot(sl)[None] + g.txt(self.tc)
            msk = (self.ar <= self.p)[None, None, None]            # (1,1,1,LMAX)
            for b, K, V in zip(g.blocks, self.K, self.V):
                q, k, v = _attn_in(b, x, self.p)
                K.index_copy_(2, self.p, k.to(K.dtype)); V.index_copy_(2, self.p, v.to(V.dtype))
                x = _attn_out(b, x, F.scaled_dot_product_attention(q, K, V, attn_mask=msk))
            return (g.head(g.nf(x)) + g.mask[sl][None]).float()[:, 0]

    def full(self, inp, tc):
        T = inp.shape[1]
        self.ids.fill_(G.NV); self.ids[:, :T] = inp; self.tc.copy_(tc); self.last.fill_(T - 1)
        self.gf.replay(); return self.out_f.clone()

    def one(self, tid, p):
        self.tid.copy_(tid); self.p.fill_(p)
        self.go.replay(); return self.out_o.clone()


ENG = None


@torch.no_grad()
def one_step(w, cond, s, check=False):
    """generate the 7 tokens of one step (CFG), decode its 4 frames, integrate world pose."""
    unc = g.null[None]
    cur = w["toks"].shape[1] // 7
    first = max(0, cur - (G.S - 1)) * 7
    inp = torch.cat([torch.full((1, 1), G.NV, device=dev, dtype=torch.long), w["toks"][:, first:]], 1)
    T = inp.shape[1]
    tc = torch.cat([cond, unc])[:, None, :]
    diffs = []
    for k in range(7):
        if ENG is not None:
            lo = ENG.full(torch.cat([inp, inp]), tc) if k == 0 else ENG.one(torch.cat([nxt, nxt]), T + k - 1)
        else:
            with torch.autocast("cuda", dtype=torch.bfloat16):
                lo = g(torch.cat([inp, inp]), SLOTS[:T], tc.expand(-1, T, -1))[:, -1].float() if k == 0 else None
                if k:
                    full = torch.cat([inp, w["toks"][:, w["toks"].shape[1] - k:]], 1)
                    lo = g(torch.cat([full, full]), SLOTS[:full.shape[1]], tc.expand(-1, full.shape[1], -1))[:, -1].float()
        if check:   # compare with the original full forward (valid entries only)
            with torch.autocast("cuda", dtype=torch.bfloat16):
                full = torch.cat([inp, w["toks"][:, w["toks"].shape[1] - k:]], 1) if k else inp
                ref = g(torch.cat([full, full]), SLOTS[:full.shape[1]], tc.expand(-1, full.shape[1], -1))[:, -1].float()
            vv = g.mask[k] == 0
            diffs.append(float((ref[:, vv] - lo[:, vv]).abs().max()))
        lo = lo.float()
        lc, lu = lo[:1], lo[1:]
        valid = (g.mask[k] == 0)[None]
        lc = torch.where(valid, lc, torch.zeros_like(lc)); lu = torch.where(valid, lu, torch.zeros_like(lu))
        lg = torch.where(valid, (1 + s) * lc - s * lu, torch.full_like(lc, float("-inf")))
        nxt = torch.multinomial(F.softmax(lg, -1), 1)
        w["toks"] = torch.cat([w["toks"], nxt], 1)
    if check:
        w["diffs"] = diffs
    w["toks"] = w["toks"][:, -G.S * 7:]                      # only the window is ever used
    code = w["toks"][0, -7:] - OFFT
    w["codes"] = torch.cat([w["codes"], code[None]])[-DEC_CTX:]
    # causal decode of the last DEC_CTX steps, keep the newest 4 frames
    c = w["codes"][None]
    qg = CB[0][c[..., 0]].permute(0, 2, 1)
    y = torch.zeros(1, c.shape[1] * 4, 272, device=dev)
    for p, dims in enumerate(B.PD):
        q = CB[p + 1][c[..., p + 1]].permute(0, 2, 1)
        y[..., dims] = h.dec[p](torch.cat([qg, q], 1)).permute(0, 2, 1)
    x = (y * std + mean)[0, -4:].double().cpu()               # (4,272) -> small 3x3 math on the CPU
    Rd = B.rot6d_to_mat(x[:, 2:8])
    P = B.positions(x)
    Rl = B.local_rots(x).clone()
    out = []
    for t in range(4):
        w["tot"] = Rd[t] if w["tot"] is None else Rd[t] @ w["tot"]
        U, _, Vh = torch.linalg.svd(w["tot"]); w["tot"] = U @ Vh  # keep it a rotation forever
        inv = w["tot"].T
        v = torch.tensor([float(x[t, 0]), 0.0, float(x[t, 1])], dtype=torch.float64)
        if w["inv_prev"] is not None:
            v = w["inv_prev"] @ v
        w["tr"] = w["tr"] + v
        w["inv_prev"] = inv
        Pw = (inv @ P[t].T).T                                 # pelvis-relative in x/z, absolute y
        Rl[t, 0] = inv @ Rl[t, 0]
        out.append((Pw, w["tr"].clone()))
    Gw = B.global_rots(Rl)                                    # (4,22,3,3)
    frames = []
    for t in range(4):
        Pw, tr = out[t]
        frames.append(dict(p=np.round(Pw.numpy().ravel(), 3).tolist(),
                           r=np.round(Gw[t].numpy().ravel(), 3).tolist(),
                           tr=[round(float(tr[0]), 3), round(float(tr[2]), 3)]))
    return frames


def gen_loop():
    w = reset_world()
    cond = embed(tok, clip, [S.prompt]).float()
    torch.manual_seed(1)
    while True:
        with S.lock:
            if S.reset:
                S.reset = False; w = reset_world()
            if S.pending is not None:
                S.prompt, S.pending = S.pending, None
                new = S.prompt
            else:
                new = None
            ahead = S.total - S.play
            s = S.s
        if new is not None:
            cond = embed(tok, clip, [new]).float()
            with S.lock:
                S.log.append([S.total, new]); S.log = S.log[-50:]
        if ahead >= LEAD * 4:
            time.sleep(0.01); continue
        t0 = time.time()
        fr = one_step(w, cond, s)
        dt = (time.time() - t0) * 1000
        with S.lock:
            S.step_ms = dt if S.step_ms == 0 else 0.9 * S.step_ms + 0.1 * dt
            for f in fr:
                f["text"] = S.prompt
            S.frames.extend(fr); S.total += 4
            if len(S.frames) > KEEP:
                cut = len(S.frames) - KEEP; S.frames = S.frames[cut:]; S.base += cut


class H(BaseHTTPRequestHandler):
    def log_message(self, *a):
        pass

    def send(self, code, body, ctype="application/json; charset=utf-8"):
        b = body.encode("utf-8") if isinstance(body, str) else body
        self.send_response(code); self.send_header("Content-Type", ctype)
        self.send_header("Cache-Control", "no-store"); self.send_header("Content-Length", str(len(b)))
        self.end_headers(); self.wfile.write(b)

    def do_GET(self):
        u = urlparse(self.path); q = parse_qs(u.query)
        if u.path in ("/", "/index.html"):
            return self.send(200, open(os.path.join(HERE, "live.html"), encoding="utf-8").read(), "text/html; charset=utf-8")
        if u.path == "/frames":
            frm = int(q.get("from", ["0"])[0]); play = int(q.get("play", ["0"])[0])
            with S.lock:
                if frm < 0:                                   # a page just opened: start at the newest frames
                    frm = max(S.base, S.total - 8)
                    S.play = frm
                else:
                    S.play = max(S.play, min(play, S.total))
                if frm < S.base:
                    frm = S.base
                fr = S.frames[frm - S.base:]
                body = json.dumps(dict(start=frm, frames=fr, total=S.total, prompt=S.prompt, pending=S.pending,
                                       step_ms=round(S.step_ms, 1), realtime_ms=round(4000 / 30, 1), s=S.s,
                                       lead=LEAD, log=S.log[-8:]), separators=(",", ":"))
            return self.send(200, body)
        self.send(404, "{}")

    def do_POST(self):
        u = urlparse(self.path)
        n = int(self.headers.get("Content-Length", "0") or 0)
        d = json.loads(self.rfile.read(n) or b"{}")
        with S.lock:
            if u.path == "/prompt":
                t = str(d.get("text", "")).strip()[:300]
                if t:
                    S.pending = t
            elif u.path == "/reset":
                S.reset = True
                S.frames, S.base, S.total, S.play, S.log = [], 0, 0, 0, []
            elif u.path == "/s":
                S.s = float(max(0.0, min(10.0, float(d.get("s", 4.0)))))
            else:
                return self.send(404, "{}")
        self.send(200, '{"ok":true}')


if __name__ == "__main__":
    # build the CUDA graphs, then an exactness check (bf16 rounding only) and a speed check
    ENG = Engine()
    w = reset_world(); cond = embed(tok, clip, ["walk"]).float()
    for _ in range(3):
        one_step(w, cond, 4.0, check=True)
        print("graph vs original forward, max |logit diff| per token:", ["%.3f" % v for v in w["diffs"]], flush=True)
    torch.cuda.synchronize(); t0 = time.time()
    for _ in range(70):                                       # fills the full 64-step window
        one_step(w, cond, 4.0)
    torch.cuda.synchronize()
    ms = (time.time() - t0) * 1000 / 70
    print("1 step (4 frames) = %.1f ms ; real time needs <= %.1f ms" % (ms, 4000 / 30), flush=True)
    for _ in range(2):                                        # window already sliding here
        one_step(w, cond, 4.0, check=True)
        print("graph vs original forward (sliding window):", ["%.3f" % v for v in w["diffs"]], flush=True)
    threading.Thread(target=gen_loop, daemon=True).start()
    srv = ThreadingHTTPServer(("127.0.0.1", PORT), H)
    print("open http://localhost:%d" % PORT, flush=True)
    srv.serve_forever()