metafeed commited on
Commit
458c218
·
verified ·
1 Parent(s): b8a8f12

MotionDNA v0: checkpoint, sample, training pipeline

Browse files
README.md ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ tags:
4
+ - motion-generation
5
+ - human-pose
6
+ - music-video
7
+ - transformer
8
+ - boostify
9
+ - motiondna
10
+ library_name: pytorch
11
+ pipeline_tag: other
12
+ ---
13
+
14
+ # Boostify MotionDNA v0
15
+
16
+ **MotionDNA** is Boostify's human-motion model: a compact autoregressive Transformer
17
+ that learns the *DNA* of how performers move in real music videos and can generate
18
+ new, plausible motion sequences (2D skeletal pose) from a short seed.
19
+
20
+ This is the **v0** checkpoint, trained end-to-end on motion extracted from real
21
+ music-video clips. It is intentionally lightweight so it runs (and trains) on a
22
+ single NVIDIA L4 (23 GB) — in fact training uses under 1 GB of VRAM.
23
+
24
+ ## What it does
25
+ - Input: a sequence of 2D human poses (17 COCO keypoints, x/y).
26
+ - Output: the predicted next pose, applied autoregressively to roll out new motion.
27
+ - Use cases: motion previews, choreography ideation, b-roll motion guides,
28
+ music-video editing assistance inside Boostify Dataset Studio.
29
+
30
+ ## Architecture
31
+ - Causal Transformer encoder (next-pose prediction).
32
+ - `d_model=256`, `n_head=8`, `n_layer=6`, window `WIN=64`, input dim `34` (17×2).
33
+ - ~4.77M parameters. Smooth-L1 loss. AdamW + cosine schedule.
34
+
35
+ ## Training data
36
+ Pose sequences extracted with YOLOv8-pose from real music-video clips
37
+ (46 clips, 11,038 frames). Poses are hip-centered and torso-scaled, then
38
+ standardized (mean/std stored inside the checkpoint).
39
+
40
+ > Source footage is used internally for research/demo. The released artifact
41
+ > contains only the trained weights and normalization stats — no source video.
42
+
43
+ ## Files
44
+ | File | Description |
45
+ |------|-------------|
46
+ | `motiondna_v0_best.pt` | Best checkpoint (lowest train loss). |
47
+ | `motiondna_v0_final.pt` | Final-epoch checkpoint. |
48
+ | `motiondna_sample.npy` | Example generated motion `[136, 17, 2]`. |
49
+ | `loss_history.json` | Per-epoch training loss. |
50
+ | `extract_poses.py` | Pose-extraction pipeline (YOLOv8-pose → .npy). |
51
+ | `train_motiondna.py` | Training script. |
52
+
53
+ ## Inference
54
+ ```python
55
+ import torch, numpy as np
56
+ ckpt = torch.load("motiondna_v0_best.pt", map_location="cpu")
57
+ cfg = ckpt["cfg"]; mean = ckpt["mean"]; std = ckpt["std"]
58
+ # rebuild the MotionDNA module from train_motiondna.py, load_state_dict(ckpt["model"]),
59
+ # seed with N normalized poses and roll out model(x)[:, -1:] autoregressively,
60
+ # then de-normalize with: pose = gen * std + mean
61
+ ```
62
+
63
+ ## Limitations
64
+ - v0 trained on a small clip set → motion is coherent but limited in style range.
65
+ - 2D pose only (no 3D, no global translation, no audio conditioning yet).
66
+ - Roadmap: more clips, music/beat conditioning, 3D lift, style control.
67
+
68
+ ## Citation
69
+ Boostify MotionDNA (2026). Trained with the Boostify Dataset Studio pipeline.
extract_poses.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MotionDNA — extrae secuencias de pose 2D (17 keypoints COCO) de los clips de
2
+ music video usando YOLO-pose. Guarda un .npy [T,17,2] (xy normalizado) por clip."""
3
+ import os, glob, sys
4
+ import numpy as np
5
+ import cv2
6
+ from ultralytics import YOLO
7
+
8
+ VIDEOS = os.path.expanduser("~/cosmos-predict2/datasets/boostify_mv_demo/videos")
9
+ OUT = os.path.expanduser("~/motiondna/poses")
10
+ os.makedirs(OUT, exist_ok=True)
11
+
12
+ model = YOLO("yolov8n-pose.pt") # se descarga solo la 1a vez
13
+ clips = sorted(glob.glob(os.path.join(VIDEOS, "*.mp4")))
14
+ print(f"Clips: {len(clips)}", flush=True)
15
+
16
+ def biggest_person(res):
17
+ if res.keypoints is None or res.keypoints.xyn is None: return None
18
+ kp = res.keypoints.xyn.cpu().numpy() # [n,17,2]
19
+ if kp.shape[0] == 0: return None
20
+ if res.boxes is not None and res.boxes.conf is not None and len(res.boxes.conf)==kp.shape[0]:
21
+ idx = int(res.boxes.conf.cpu().numpy().argmax())
22
+ else:
23
+ idx = 0
24
+ return kp[idx] # [17,2]
25
+
26
+ total_frames = 0
27
+ for ci, path in enumerate(clips):
28
+ cap = cv2.VideoCapture(path)
29
+ seq = []; last = None
30
+ while True:
31
+ ok, frame = cap.read()
32
+ if not ok: break
33
+ res = model.predict(frame, verbose=False, device=0)[0]
34
+ kp = biggest_person(res)
35
+ if kp is None:
36
+ if last is None: continue
37
+ kp = last
38
+ last = kp
39
+ seq.append(kp.astype(np.float32))
40
+ cap.release()
41
+ if len(seq) >= 64:
42
+ arr = np.stack(seq) # [T,17,2]
43
+ np.save(os.path.join(OUT, os.path.basename(path).replace(".mp4",".npy")), arr)
44
+ total_frames += len(seq)
45
+ print(f" [{ci+1}/{len(clips)}] {os.path.basename(path)} -> {len(seq)} frames", flush=True)
46
+ print(f"LISTO. Secuencias guardadas. Frames totales: {total_frames}", flush=True)
loss_history.json ADDED
@@ -0,0 +1 @@
 
 
1
+ [0.2717815339565277, 0.16170065104961395, 0.15134050324559212, 0.13297879695892334, 0.10486973077058792, 0.1119859591126442, 0.10408504121005535, 0.097188550978899, 0.09335055388510227, 0.07721971906721592, 0.06102317292243242, 0.08365507610142231, 0.0811763247475028, 0.07989647053182125, 0.07774336636066437, 0.07627369184046984, 0.07516010105609894, 0.0731716938316822, 0.06191330496221781, 0.0722947521135211, 0.07149569038301706, 0.0709458738565445, 0.07037000358104706, 0.06927726045250893, 0.05760655086487532, 0.06750039663165808, 0.06795270182192326, 0.06836127210408449, 0.055791727267205715, 0.06718302983790636, 0.06699226144701242, 0.05669730808585882, 0.06608119606971741, 0.0552947660908103, 0.0658682631328702, 0.06554433889687061, 0.06535841152071953, 0.05380233284085989, 0.06463884469121695, 0.06472351308912039, 0.06384623516350985, 0.043290598317980766, 0.05420905631035566, 0.06335244048386812, 0.06382613070309162, 0.06408144626766443, 0.06418973486870527, 0.06370448786765337, 0.063649564050138, 0.0634511411190033, 0.06308642029762268, 0.06297928467392921, 0.0638816338032484, 0.06335235293954611, 0.06301694270223379, 0.06371666677296162, 0.05220216326415539, 0.06294357124716043, 0.0628390982747078, 0.06346037331968546]
motiondna_sample.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f91b46758542c16b243f9b4a21122d2bd1b9490ac06a511a98d5e0f6c5156aa7
3
+ size 18624
motiondna_v0_best.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dee5047ea7ba3669dafd6568c1646c8e0f7ee565dd8247d80e19a87c0de0c520
3
+ size 19119705
motiondna_v0_final.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b76d73f75587e6cc706a6f79de83dcf4fe51ae9bb2850ad393090d58c003b743
3
+ size 19119786
train_motiondna.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MotionDNA v0 — modelo generativo de movimiento humano entrenado con la pose
2
+ extraida de music videos. Transformer causal que predice la pose del frame t+1
3
+ dada la secuencia 0..t. Genera movimiento nuevo por rollout autoregresivo.
4
+ Pensado para caber holgado en una L4 (23GB)."""
5
+ import os, glob, math, time, json
6
+ import numpy as np
7
+ import torch, torch.nn as nn
8
+ from torch.utils.data import Dataset, DataLoader
9
+
10
+ POSES = os.path.expanduser("~/motiondna/poses")
11
+ CKPT = os.path.expanduser("~/motiondna/checkpoints")
12
+ SAMP = os.path.expanduser("~/motiondna/samples")
13
+ os.makedirs(CKPT, exist_ok=True); os.makedirs(SAMP, exist_ok=True)
14
+ DEV = "cuda" if torch.cuda.is_available() else "cpu"
15
+ WIN = 64 # ventana temporal
16
+ DIM = 34 # 17 keypoints * 2
17
+ D_MODEL = 256; N_HEAD = 8; N_LAYER = 6; DROP = 0.1
18
+ BATCH = 64; EPOCHS = 60; LR = 3e-4
19
+
20
+ def normalize(seq):
21
+ """seq [T,17,2] xy normalizado a frame -> centrado en cadera + escala torso."""
22
+ s = seq.copy().astype(np.float32)
23
+ hip = (s[:,11,:] + s[:,12,:]) / 2.0 # centro caderas
24
+ s = s - hip[:,None,:]
25
+ sh = (s[:,5,:] + s[:,6,:]) / 2.0 # centro hombros
26
+ torso = np.linalg.norm(sh - 0.0, axis=1) # dist hombros->cadera(0)
27
+ scale = np.clip(np.median(torso), 1e-3, None)
28
+ s = s / scale
29
+ return s.reshape(s.shape[0], -1) # [T,34]
30
+
31
+ class MotionDS(Dataset):
32
+ def __init__(self):
33
+ self.windows = []
34
+ files = sorted(glob.glob(os.path.join(POSES, "*.npy")))
35
+ for f in files:
36
+ seq = np.load(f)
37
+ if seq.shape[0] < WIN + 1: continue
38
+ n = normalize(seq)
39
+ for i in range(0, n.shape[0] - WIN - 1, WIN // 2):
40
+ self.windows.append(n[i:i+WIN+1])
41
+ if self.windows:
42
+ allp = np.concatenate(self.windows, 0)
43
+ self.mean = allp.mean(0, keepdims=True); self.std = allp.std(0, keepdims=True) + 1e-6
44
+ self.windows = [(w - self.mean) / self.std for w in self.windows]
45
+ print(f"Ventanas de entrenamiento: {len(self.windows)} | clips: {len(files)}", flush=True)
46
+ def __len__(self): return len(self.windows)
47
+ def __getitem__(self, i):
48
+ w = torch.from_numpy(self.windows[i].astype(np.float32))
49
+ return w[:-1], w[1:] # input, target (shift 1)
50
+
51
+ class MotionDNA(nn.Module):
52
+ def __init__(self):
53
+ super().__init__()
54
+ self.inp = nn.Linear(DIM, D_MODEL)
55
+ self.pos = nn.Parameter(torch.randn(1, WIN, D_MODEL) * 0.02)
56
+ layer = nn.TransformerEncoderLayer(D_MODEL, N_HEAD, D_MODEL*4, DROP, batch_first=True, activation="gelu")
57
+ self.enc = nn.TransformerEncoder(layer, N_LAYER)
58
+ self.out = nn.Linear(D_MODEL, DIM)
59
+ def forward(self, x):
60
+ T = x.size(1)
61
+ h = self.inp(x) + self.pos[:, :T]
62
+ mask = torch.triu(torch.ones(T, T, device=x.device), 1).bool()
63
+ h = self.enc(h, mask=mask)
64
+ return self.out(h)
65
+
66
+ def main():
67
+ ds = MotionDS()
68
+ if len(ds) == 0:
69
+ print("ERROR: no hay ventanas. Corre extract_poses.py primero."); return
70
+ dl = DataLoader(ds, batch_size=BATCH, shuffle=True, num_workers=4, drop_last=True)
71
+ model = MotionDNA().to(DEV)
72
+ nparam = sum(p.numel() for p in model.parameters())
73
+ print(f"Device: {DEV} | Parametros: {nparam/1e6:.2f}M", flush=True)
74
+ opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
75
+ sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, EPOCHS)
76
+ lossf = nn.SmoothL1Loss()
77
+ best = 1e9; hist = []
78
+ for ep in range(1, EPOCHS+1):
79
+ model.train(); tot = 0; nb = 0
80
+ for x, y in dl:
81
+ x, y = x.to(DEV), y.to(DEV)
82
+ pred = model(x)
83
+ loss = lossf(pred, y)
84
+ opt.zero_grad(); loss.backward()
85
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
86
+ opt.step()
87
+ tot += loss.item(); nb += 1
88
+ sched.step()
89
+ avg = tot / max(nb,1); hist.append(avg)
90
+ mem = torch.cuda.max_memory_allocated()/1e9 if DEV=="cuda" else 0
91
+ print(f"Epoch {ep:03d}/{EPOCHS} loss={avg:.5f} lr={sched.get_last_lr()[0]:.2e} vram={mem:.2f}GB", flush=True)
92
+ if avg < best:
93
+ best = avg
94
+ torch.save({"model": model.state_dict(), "mean": ds.mean, "std": ds.std,
95
+ "cfg": {"WIN":WIN,"DIM":DIM,"D_MODEL":D_MODEL,"N_HEAD":N_HEAD,"N_LAYER":N_LAYER},
96
+ "epoch": ep, "loss": best}, os.path.join(CKPT, "motiondna_v0_best.pt"))
97
+ torch.save({"model": model.state_dict(), "mean": ds.mean, "std": ds.std,
98
+ "cfg": {"WIN":WIN,"DIM":DIM,"D_MODEL":D_MODEL,"N_HEAD":N_HEAD,"N_LAYER":N_LAYER},
99
+ "epoch": EPOCHS, "loss": hist[-1]}, os.path.join(CKPT, "motiondna_v0_final.pt"))
100
+ json.dump(hist, open(os.path.join(CKPT, "loss_history.json"), "w"))
101
+ # --- rollout de muestra: genera 120 frames de movimiento nuevo ---
102
+ model.eval()
103
+ with torch.no_grad():
104
+ seed = ds[0][0][:16].unsqueeze(0).to(DEV) # 16 frames semilla
105
+ gen = seed.clone()
106
+ for _ in range(120):
107
+ nxt = model(gen[:, -WIN:])[:, -1:]
108
+ gen = torch.cat([gen, nxt], 1)
109
+ gen = gen[0].cpu().numpy() * ds.std + ds.mean # des-normaliza
110
+ np.save(os.path.join(SAMP, "motiondna_sample.npy"), gen.reshape(-1,17,2))
111
+ print(f"LISTO. best_loss={best:.5f} ckpt={CKPT}/motiondna_v0_best.pt sample={SAMP}/motiondna_sample.npy", flush=True)
112
+
113
+ if __name__ == "__main__":
114
+ main()