Download live/live_server.py from Yakimask/PartLock-CMU: direct link, hf CLI and curl.
- Browser
- Download file 14.7 kB
-
https://huggingface.co/Yakimask/PartLock-CMU/resolve/main/live/live_server.py
- Command line
-
hf download hf://Yakimask/PartLock-CMU/live/live_server.py
-
curl -L -o live_server.py https://huggingface.co/Yakimask/PartLock-CMU/resolve/main/live/live_server.py
14.7 kB
| """Real-time, endless text-to-motion with the PartLock CMU model. | |
| The model keeps generating forever; the browser plays it in real time. | |
| Typing a prompt and pressing the button swaps the text at the next step (4 frames), without resetting the context. | |
| python live/live_server.py then open http://localhost:8765 | |
| Options (environment variables): LIVE_PORT (8765), LIVE_LEAD (6 steps generated ahead of playback = 0.8 s). | |
| Generation is kept only LEAD steps ahead of playback, so a new prompt shows up after about LEAD*4/30 s.""" | |
| import os, sys, json, time, threading | |
| from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler | |
| from urllib.parse import urlparse, parse_qs | |
| import numpy as np, torch | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| REPO = os.path.dirname(HERE) | |
| sys.path.insert(0, os.path.join(REPO, "code")) | |
| import tvq272 as B, tvq272c as C, gen_ar as G | |
| from gen_sample import load, embed, get_codebook | |
| dev = "cuda" | |
| WTS = os.path.join(REPO, "weights") | |
| PORT = int(os.environ.get("LIVE_PORT", "8765")) | |
| LEAD = int(os.environ.get("LIVE_LEAD", "6")) # steps generated ahead of playback (6 steps = 0.8 s) | |
| DEC_CTX = 32 # decoder receptive field is ~16 steps; 32 is safe | |
| KEEP = 30 * 120 # frames kept on the server (2 min) | |
| mean = torch.from_numpy(np.load(os.path.join(WTS, "Mean.npy")).astype(np.float32)).to(dev) | |
| std = torch.from_numpy(np.load(os.path.join(WTS, "Std.npy")).astype(np.float32)).to(dev) | |
| h = C.CHierTok().to(dev); h.load_state_dict(load_file(os.path.join(WTS, "tokenizer.safetensors"), device=dev)); h.eval() | |
| g = G.Gen().to(dev); g.load_state_dict(load_file(os.path.join(WTS, "generator.safetensors"), device=dev)); g.eval() | |
| tok, clip = load() | |
| CB = [get_codebook(h.gvq)] + [get_codebook(v) for v in h.vq] | |
| OFFT = torch.tensor(G.OFF, device=dev) | |
| SLOTS = torch.arange(G.L, device=dev) % 7 | |
| class State: | |
| def __init__(self): | |
| self.lock = threading.Lock() | |
| self.frames = [] # list of dict per frame (json-ready) | |
| self.base = 0 # global index of frames[0] | |
| self.total = 0 # frames produced so far | |
| self.play = 0 # browser playback frame (reported by polling) | |
| self.prompt = "Standing still" | |
| self.pending = None | |
| self.reset = False | |
| self.s = 4.0 | |
| self.step_ms = 0.0 | |
| self.log = [] # (frame, prompt) | |
| S = State() | |
| def reset_world(): | |
| return dict(codes=torch.zeros(0, 7, dtype=torch.long, device=dev), toks=torch.zeros(1, 0, dtype=torch.long, device=dev), | |
| tot=None, inv_prev=None, tr=torch.zeros(3, dtype=torch.float64)) | |
| def _attn_in(b, x, pos): | |
| Bn, T, D = x.shape | |
| q, k, v = b.qkv(b.n1(x)).view(Bn, T, 3, b.h, D // b.h).permute(2, 0, 3, 1, 4) | |
| return G.rope(q, pos), G.rope(k, pos), v | |
| def _attn_out(b, x, a): | |
| Bn, T, D = x.shape | |
| x = x + b.o(a.transpose(1, 2).reshape(Bn, T, D)) | |
| return x + b.ff(b.n2(x)) | |
| LMAX = G.L # 448 = BOS + 63 past steps + 7 tokens of the current step (most the original ever feeds) | |
| class Engine: | |
| """CUDA-graph version of the same forward. Static K/V buffers of length LMAX. | |
| full(): window padded to LMAX at the END (causal -> real positions are unaffected), logits taken at T-1. | |
| one(): one new token at position p, attends to positions <= p.""" | |
| def __init__(self): | |
| nh = g.blocks[0].h; dh = 512 // nh | |
| self.K = [torch.zeros(2, nh, LMAX, dh, device=dev, dtype=torch.bfloat16) for _ in g.blocks] | |
| self.V = [torch.zeros_like(k) for k in self.K] | |
| self.ids = torch.full((2, LMAX), G.NV, device=dev, dtype=torch.long) | |
| self.tc = torch.zeros(2, 1, 512, device=dev) | |
| self.last = torch.zeros((), device=dev, dtype=torch.long) # index T-1 | |
| self.tid = torch.zeros(2, 1, device=dev, dtype=torch.long) | |
| self.p = torch.zeros(1, device=dev, dtype=torch.long) | |
| self.ar = torch.arange(LMAX, device=dev) | |
| s = torch.cuda.Stream(); s.wait_stream(torch.cuda.current_stream()) | |
| with torch.cuda.stream(s): | |
| for _ in range(3): | |
| self._full(); self._one() | |
| torch.cuda.current_stream().wait_stream(s) | |
| self.gf = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph(self.gf): | |
| self.out_f = self._full() | |
| self.go = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph(self.go): | |
| self.out_o = self._one() | |
| def _full(self): | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| x = g.tok(self.ids) + g.slot(SLOTS)[None] + g.txt(self.tc) | |
| pos = self.ar | |
| for b, K, V in zip(g.blocks, self.K, self.V): | |
| q, k, v = _attn_in(b, x, pos) | |
| K.copy_(k); V.copy_(v) | |
| x = _attn_out(b, x, F.scaled_dot_product_attention(q, k, v, is_causal=True)) | |
| xl = x.index_select(1, self.last.view(1)) | |
| return (g.head(g.nf(xl)) + g.mask[SLOTS.index_select(0, self.last.view(1))][None]).float()[:, 0] | |
| def _one(self): | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| sl = SLOTS.index_select(0, self.p) | |
| x = g.tok(self.tid) + g.slot(sl)[None] + g.txt(self.tc) | |
| msk = (self.ar <= self.p)[None, None, None] # (1,1,1,LMAX) | |
| for b, K, V in zip(g.blocks, self.K, self.V): | |
| q, k, v = _attn_in(b, x, self.p) | |
| K.index_copy_(2, self.p, k.to(K.dtype)); V.index_copy_(2, self.p, v.to(V.dtype)) | |
| x = _attn_out(b, x, F.scaled_dot_product_attention(q, K, V, attn_mask=msk)) | |
| return (g.head(g.nf(x)) + g.mask[sl][None]).float()[:, 0] | |
| def full(self, inp, tc): | |
| T = inp.shape[1] | |
| self.ids.fill_(G.NV); self.ids[:, :T] = inp; self.tc.copy_(tc); self.last.fill_(T - 1) | |
| self.gf.replay(); return self.out_f.clone() | |
| def one(self, tid, p): | |
| self.tid.copy_(tid); self.p.fill_(p) | |
| self.go.replay(); return self.out_o.clone() | |
| ENG = None | |
| def one_step(w, cond, s, check=False): | |
| """generate the 7 tokens of one step (CFG), decode its 4 frames, integrate world pose.""" | |
| unc = g.null[None] | |
| cur = w["toks"].shape[1] // 7 | |
| first = max(0, cur - (G.S - 1)) * 7 | |
| inp = torch.cat([torch.full((1, 1), G.NV, device=dev, dtype=torch.long), w["toks"][:, first:]], 1) | |
| T = inp.shape[1] | |
| tc = torch.cat([cond, unc])[:, None, :] | |
| diffs = [] | |
| for k in range(7): | |
| if ENG is not None: | |
| lo = ENG.full(torch.cat([inp, inp]), tc) if k == 0 else ENG.one(torch.cat([nxt, nxt]), T + k - 1) | |
| else: | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| lo = g(torch.cat([inp, inp]), SLOTS[:T], tc.expand(-1, T, -1))[:, -1].float() if k == 0 else None | |
| if k: | |
| full = torch.cat([inp, w["toks"][:, w["toks"].shape[1] - k:]], 1) | |
| lo = g(torch.cat([full, full]), SLOTS[:full.shape[1]], tc.expand(-1, full.shape[1], -1))[:, -1].float() | |
| if check: # compare with the original full forward (valid entries only) | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| full = torch.cat([inp, w["toks"][:, w["toks"].shape[1] - k:]], 1) if k else inp | |
| ref = g(torch.cat([full, full]), SLOTS[:full.shape[1]], tc.expand(-1, full.shape[1], -1))[:, -1].float() | |
| vv = g.mask[k] == 0 | |
| diffs.append(float((ref[:, vv] - lo[:, vv]).abs().max())) | |
| lo = lo.float() | |
| lc, lu = lo[:1], lo[1:] | |
| valid = (g.mask[k] == 0)[None] | |
| lc = torch.where(valid, lc, torch.zeros_like(lc)); lu = torch.where(valid, lu, torch.zeros_like(lu)) | |
| lg = torch.where(valid, (1 + s) * lc - s * lu, torch.full_like(lc, float("-inf"))) | |
| nxt = torch.multinomial(F.softmax(lg, -1), 1) | |
| w["toks"] = torch.cat([w["toks"], nxt], 1) | |
| if check: | |
| w["diffs"] = diffs | |
| w["toks"] = w["toks"][:, -G.S * 7:] # only the window is ever used | |
| code = w["toks"][0, -7:] - OFFT | |
| w["codes"] = torch.cat([w["codes"], code[None]])[-DEC_CTX:] | |
| # causal decode of the last DEC_CTX steps, keep the newest 4 frames | |
| c = w["codes"][None] | |
| qg = CB[0][c[..., 0]].permute(0, 2, 1) | |
| y = torch.zeros(1, c.shape[1] * 4, 272, device=dev) | |
| for p, dims in enumerate(B.PD): | |
| q = CB[p + 1][c[..., p + 1]].permute(0, 2, 1) | |
| y[..., dims] = h.dec[p](torch.cat([qg, q], 1)).permute(0, 2, 1) | |
| x = (y * std + mean)[0, -4:].double().cpu() # (4,272) -> small 3x3 math on the CPU | |
| Rd = B.rot6d_to_mat(x[:, 2:8]) | |
| P = B.positions(x) | |
| Rl = B.local_rots(x).clone() | |
| out = [] | |
| for t in range(4): | |
| w["tot"] = Rd[t] if w["tot"] is None else Rd[t] @ w["tot"] | |
| U, _, Vh = torch.linalg.svd(w["tot"]); w["tot"] = U @ Vh # keep it a rotation forever | |
| inv = w["tot"].T | |
| v = torch.tensor([float(x[t, 0]), 0.0, float(x[t, 1])], dtype=torch.float64) | |
| if w["inv_prev"] is not None: | |
| v = w["inv_prev"] @ v | |
| w["tr"] = w["tr"] + v | |
| w["inv_prev"] = inv | |
| Pw = (inv @ P[t].T).T # pelvis-relative in x/z, absolute y | |
| Rl[t, 0] = inv @ Rl[t, 0] | |
| out.append((Pw, w["tr"].clone())) | |
| Gw = B.global_rots(Rl) # (4,22,3,3) | |
| frames = [] | |
| for t in range(4): | |
| Pw, tr = out[t] | |
| frames.append(dict(p=np.round(Pw.numpy().ravel(), 3).tolist(), | |
| r=np.round(Gw[t].numpy().ravel(), 3).tolist(), | |
| tr=[round(float(tr[0]), 3), round(float(tr[2]), 3)])) | |
| return frames | |
| def gen_loop(): | |
| w = reset_world() | |
| cond = embed(tok, clip, [S.prompt]).float() | |
| torch.manual_seed(1) | |
| while True: | |
| with S.lock: | |
| if S.reset: | |
| S.reset = False; w = reset_world() | |
| if S.pending is not None: | |
| S.prompt, S.pending = S.pending, None | |
| new = S.prompt | |
| else: | |
| new = None | |
| ahead = S.total - S.play | |
| s = S.s | |
| if new is not None: | |
| cond = embed(tok, clip, [new]).float() | |
| with S.lock: | |
| S.log.append([S.total, new]); S.log = S.log[-50:] | |
| if ahead >= LEAD * 4: | |
| time.sleep(0.01); continue | |
| t0 = time.time() | |
| fr = one_step(w, cond, s) | |
| dt = (time.time() - t0) * 1000 | |
| with S.lock: | |
| S.step_ms = dt if S.step_ms == 0 else 0.9 * S.step_ms + 0.1 * dt | |
| for f in fr: | |
| f["text"] = S.prompt | |
| S.frames.extend(fr); S.total += 4 | |
| if len(S.frames) > KEEP: | |
| cut = len(S.frames) - KEEP; S.frames = S.frames[cut:]; S.base += cut | |
| class H(BaseHTTPRequestHandler): | |
| def log_message(self, *a): | |
| pass | |
| def send(self, code, body, ctype="application/json; charset=utf-8"): | |
| b = body.encode("utf-8") if isinstance(body, str) else body | |
| self.send_response(code); self.send_header("Content-Type", ctype) | |
| self.send_header("Cache-Control", "no-store"); self.send_header("Content-Length", str(len(b))) | |
| self.end_headers(); self.wfile.write(b) | |
| def do_GET(self): | |
| u = urlparse(self.path); q = parse_qs(u.query) | |
| if u.path in ("/", "/index.html"): | |
| return self.send(200, open(os.path.join(HERE, "live.html"), encoding="utf-8").read(), "text/html; charset=utf-8") | |
| if u.path == "/frames": | |
| frm = int(q.get("from", ["0"])[0]); play = int(q.get("play", ["0"])[0]) | |
| with S.lock: | |
| if frm < 0: # a page just opened: start at the newest frames | |
| frm = max(S.base, S.total - 8) | |
| S.play = frm | |
| else: | |
| S.play = max(S.play, min(play, S.total)) | |
| if frm < S.base: | |
| frm = S.base | |
| fr = S.frames[frm - S.base:] | |
| body = json.dumps(dict(start=frm, frames=fr, total=S.total, prompt=S.prompt, pending=S.pending, | |
| step_ms=round(S.step_ms, 1), realtime_ms=round(4000 / 30, 1), s=S.s, | |
| lead=LEAD, log=S.log[-8:]), separators=(",", ":")) | |
| return self.send(200, body) | |
| self.send(404, "{}") | |
| def do_POST(self): | |
| u = urlparse(self.path) | |
| n = int(self.headers.get("Content-Length", "0") or 0) | |
| d = json.loads(self.rfile.read(n) or b"{}") | |
| with S.lock: | |
| if u.path == "/prompt": | |
| t = str(d.get("text", "")).strip()[:300] | |
| if t: | |
| S.pending = t | |
| elif u.path == "/reset": | |
| S.reset = True | |
| S.frames, S.base, S.total, S.play, S.log = [], 0, 0, 0, [] | |
| elif u.path == "/s": | |
| S.s = float(max(0.0, min(10.0, float(d.get("s", 4.0))))) | |
| else: | |
| return self.send(404, "{}") | |
| self.send(200, '{"ok":true}') | |
| if __name__ == "__main__": | |
| # build the CUDA graphs, then an exactness check (bf16 rounding only) and a speed check | |
| ENG = Engine() | |
| w = reset_world(); cond = embed(tok, clip, ["walk"]).float() | |
| for _ in range(3): | |
| one_step(w, cond, 4.0, check=True) | |
| print("graph vs original forward, max |logit diff| per token:", ["%.3f" % v for v in w["diffs"]], flush=True) | |
| torch.cuda.synchronize(); t0 = time.time() | |
| for _ in range(70): # fills the full 64-step window | |
| one_step(w, cond, 4.0) | |
| torch.cuda.synchronize() | |
| ms = (time.time() - t0) * 1000 / 70 | |
| print("1 step (4 frames) = %.1f ms ; real time needs <= %.1f ms" % (ms, 4000 / 30), flush=True) | |
| for _ in range(2): # window already sliding here | |
| one_step(w, cond, 4.0, check=True) | |
| print("graph vs original forward (sliding window):", ["%.3f" % v for v in w["diffs"]], flush=True) | |
| threading.Thread(target=gen_loop, daemon=True).start() | |
| srv = ThreadingHTTPServer(("127.0.0.1", PORT), H) | |
| print("open http://localhost:%d" % PORT, flush=True) | |
| srv.serve_forever() | |