MotionDNA v0: checkpoint, sample, training pipeline
Browse files- README.md +69 -0
- extract_poses.py +46 -0
- loss_history.json +1 -0
- motiondna_sample.npy +3 -0
- motiondna_v0_best.pt +3 -0
- motiondna_v0_final.pt +3 -0
- train_motiondna.py +114 -0
README.md
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
tags:
|
| 4 |
+
- motion-generation
|
| 5 |
+
- human-pose
|
| 6 |
+
- music-video
|
| 7 |
+
- transformer
|
| 8 |
+
- boostify
|
| 9 |
+
- motiondna
|
| 10 |
+
library_name: pytorch
|
| 11 |
+
pipeline_tag: other
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# Boostify MotionDNA v0
|
| 15 |
+
|
| 16 |
+
**MotionDNA** is Boostify's human-motion model: a compact autoregressive Transformer
|
| 17 |
+
that learns the *DNA* of how performers move in real music videos and can generate
|
| 18 |
+
new, plausible motion sequences (2D skeletal pose) from a short seed.
|
| 19 |
+
|
| 20 |
+
This is the **v0** checkpoint, trained end-to-end on motion extracted from real
|
| 21 |
+
music-video clips. It is intentionally lightweight so it runs (and trains) on a
|
| 22 |
+
single NVIDIA L4 (23 GB) — in fact training uses under 1 GB of VRAM.
|
| 23 |
+
|
| 24 |
+
## What it does
|
| 25 |
+
- Input: a sequence of 2D human poses (17 COCO keypoints, x/y).
|
| 26 |
+
- Output: the predicted next pose, applied autoregressively to roll out new motion.
|
| 27 |
+
- Use cases: motion previews, choreography ideation, b-roll motion guides,
|
| 28 |
+
music-video editing assistance inside Boostify Dataset Studio.
|
| 29 |
+
|
| 30 |
+
## Architecture
|
| 31 |
+
- Causal Transformer encoder (next-pose prediction).
|
| 32 |
+
- `d_model=256`, `n_head=8`, `n_layer=6`, window `WIN=64`, input dim `34` (17×2).
|
| 33 |
+
- ~4.77M parameters. Smooth-L1 loss. AdamW + cosine schedule.
|
| 34 |
+
|
| 35 |
+
## Training data
|
| 36 |
+
Pose sequences extracted with YOLOv8-pose from real music-video clips
|
| 37 |
+
(46 clips, 11,038 frames). Poses are hip-centered and torso-scaled, then
|
| 38 |
+
standardized (mean/std stored inside the checkpoint).
|
| 39 |
+
|
| 40 |
+
> Source footage is used internally for research/demo. The released artifact
|
| 41 |
+
> contains only the trained weights and normalization stats — no source video.
|
| 42 |
+
|
| 43 |
+
## Files
|
| 44 |
+
| File | Description |
|
| 45 |
+
|------|-------------|
|
| 46 |
+
| `motiondna_v0_best.pt` | Best checkpoint (lowest train loss). |
|
| 47 |
+
| `motiondna_v0_final.pt` | Final-epoch checkpoint. |
|
| 48 |
+
| `motiondna_sample.npy` | Example generated motion `[136, 17, 2]`. |
|
| 49 |
+
| `loss_history.json` | Per-epoch training loss. |
|
| 50 |
+
| `extract_poses.py` | Pose-extraction pipeline (YOLOv8-pose → .npy). |
|
| 51 |
+
| `train_motiondna.py` | Training script. |
|
| 52 |
+
|
| 53 |
+
## Inference
|
| 54 |
+
```python
|
| 55 |
+
import torch, numpy as np
|
| 56 |
+
ckpt = torch.load("motiondna_v0_best.pt", map_location="cpu")
|
| 57 |
+
cfg = ckpt["cfg"]; mean = ckpt["mean"]; std = ckpt["std"]
|
| 58 |
+
# rebuild the MotionDNA module from train_motiondna.py, load_state_dict(ckpt["model"]),
|
| 59 |
+
# seed with N normalized poses and roll out model(x)[:, -1:] autoregressively,
|
| 60 |
+
# then de-normalize with: pose = gen * std + mean
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
## Limitations
|
| 64 |
+
- v0 trained on a small clip set → motion is coherent but limited in style range.
|
| 65 |
+
- 2D pose only (no 3D, no global translation, no audio conditioning yet).
|
| 66 |
+
- Roadmap: more clips, music/beat conditioning, 3D lift, style control.
|
| 67 |
+
|
| 68 |
+
## Citation
|
| 69 |
+
Boostify MotionDNA (2026). Trained with the Boostify Dataset Studio pipeline.
|
extract_poses.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MotionDNA — extrae secuencias de pose 2D (17 keypoints COCO) de los clips de
|
| 2 |
+
music video usando YOLO-pose. Guarda un .npy [T,17,2] (xy normalizado) por clip."""
|
| 3 |
+
import os, glob, sys
|
| 4 |
+
import numpy as np
|
| 5 |
+
import cv2
|
| 6 |
+
from ultralytics import YOLO
|
| 7 |
+
|
| 8 |
+
VIDEOS = os.path.expanduser("~/cosmos-predict2/datasets/boostify_mv_demo/videos")
|
| 9 |
+
OUT = os.path.expanduser("~/motiondna/poses")
|
| 10 |
+
os.makedirs(OUT, exist_ok=True)
|
| 11 |
+
|
| 12 |
+
model = YOLO("yolov8n-pose.pt") # se descarga solo la 1a vez
|
| 13 |
+
clips = sorted(glob.glob(os.path.join(VIDEOS, "*.mp4")))
|
| 14 |
+
print(f"Clips: {len(clips)}", flush=True)
|
| 15 |
+
|
| 16 |
+
def biggest_person(res):
|
| 17 |
+
if res.keypoints is None or res.keypoints.xyn is None: return None
|
| 18 |
+
kp = res.keypoints.xyn.cpu().numpy() # [n,17,2]
|
| 19 |
+
if kp.shape[0] == 0: return None
|
| 20 |
+
if res.boxes is not None and res.boxes.conf is not None and len(res.boxes.conf)==kp.shape[0]:
|
| 21 |
+
idx = int(res.boxes.conf.cpu().numpy().argmax())
|
| 22 |
+
else:
|
| 23 |
+
idx = 0
|
| 24 |
+
return kp[idx] # [17,2]
|
| 25 |
+
|
| 26 |
+
total_frames = 0
|
| 27 |
+
for ci, path in enumerate(clips):
|
| 28 |
+
cap = cv2.VideoCapture(path)
|
| 29 |
+
seq = []; last = None
|
| 30 |
+
while True:
|
| 31 |
+
ok, frame = cap.read()
|
| 32 |
+
if not ok: break
|
| 33 |
+
res = model.predict(frame, verbose=False, device=0)[0]
|
| 34 |
+
kp = biggest_person(res)
|
| 35 |
+
if kp is None:
|
| 36 |
+
if last is None: continue
|
| 37 |
+
kp = last
|
| 38 |
+
last = kp
|
| 39 |
+
seq.append(kp.astype(np.float32))
|
| 40 |
+
cap.release()
|
| 41 |
+
if len(seq) >= 64:
|
| 42 |
+
arr = np.stack(seq) # [T,17,2]
|
| 43 |
+
np.save(os.path.join(OUT, os.path.basename(path).replace(".mp4",".npy")), arr)
|
| 44 |
+
total_frames += len(seq)
|
| 45 |
+
print(f" [{ci+1}/{len(clips)}] {os.path.basename(path)} -> {len(seq)} frames", flush=True)
|
| 46 |
+
print(f"LISTO. Secuencias guardadas. Frames totales: {total_frames}", flush=True)
|
loss_history.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[0.2717815339565277, 0.16170065104961395, 0.15134050324559212, 0.13297879695892334, 0.10486973077058792, 0.1119859591126442, 0.10408504121005535, 0.097188550978899, 0.09335055388510227, 0.07721971906721592, 0.06102317292243242, 0.08365507610142231, 0.0811763247475028, 0.07989647053182125, 0.07774336636066437, 0.07627369184046984, 0.07516010105609894, 0.0731716938316822, 0.06191330496221781, 0.0722947521135211, 0.07149569038301706, 0.0709458738565445, 0.07037000358104706, 0.06927726045250893, 0.05760655086487532, 0.06750039663165808, 0.06795270182192326, 0.06836127210408449, 0.055791727267205715, 0.06718302983790636, 0.06699226144701242, 0.05669730808585882, 0.06608119606971741, 0.0552947660908103, 0.0658682631328702, 0.06554433889687061, 0.06535841152071953, 0.05380233284085989, 0.06463884469121695, 0.06472351308912039, 0.06384623516350985, 0.043290598317980766, 0.05420905631035566, 0.06335244048386812, 0.06382613070309162, 0.06408144626766443, 0.06418973486870527, 0.06370448786765337, 0.063649564050138, 0.0634511411190033, 0.06308642029762268, 0.06297928467392921, 0.0638816338032484, 0.06335235293954611, 0.06301694270223379, 0.06371666677296162, 0.05220216326415539, 0.06294357124716043, 0.0628390982747078, 0.06346037331968546]
|
motiondna_sample.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f91b46758542c16b243f9b4a21122d2bd1b9490ac06a511a98d5e0f6c5156aa7
|
| 3 |
+
size 18624
|
motiondna_v0_best.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dee5047ea7ba3669dafd6568c1646c8e0f7ee565dd8247d80e19a87c0de0c520
|
| 3 |
+
size 19119705
|
motiondna_v0_final.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b76d73f75587e6cc706a6f79de83dcf4fe51ae9bb2850ad393090d58c003b743
|
| 3 |
+
size 19119786
|
train_motiondna.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MotionDNA v0 — modelo generativo de movimiento humano entrenado con la pose
|
| 2 |
+
extraida de music videos. Transformer causal que predice la pose del frame t+1
|
| 3 |
+
dada la secuencia 0..t. Genera movimiento nuevo por rollout autoregresivo.
|
| 4 |
+
Pensado para caber holgado en una L4 (23GB)."""
|
| 5 |
+
import os, glob, math, time, json
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch, torch.nn as nn
|
| 8 |
+
from torch.utils.data import Dataset, DataLoader
|
| 9 |
+
|
| 10 |
+
POSES = os.path.expanduser("~/motiondna/poses")
|
| 11 |
+
CKPT = os.path.expanduser("~/motiondna/checkpoints")
|
| 12 |
+
SAMP = os.path.expanduser("~/motiondna/samples")
|
| 13 |
+
os.makedirs(CKPT, exist_ok=True); os.makedirs(SAMP, exist_ok=True)
|
| 14 |
+
DEV = "cuda" if torch.cuda.is_available() else "cpu"
|
| 15 |
+
WIN = 64 # ventana temporal
|
| 16 |
+
DIM = 34 # 17 keypoints * 2
|
| 17 |
+
D_MODEL = 256; N_HEAD = 8; N_LAYER = 6; DROP = 0.1
|
| 18 |
+
BATCH = 64; EPOCHS = 60; LR = 3e-4
|
| 19 |
+
|
| 20 |
+
def normalize(seq):
|
| 21 |
+
"""seq [T,17,2] xy normalizado a frame -> centrado en cadera + escala torso."""
|
| 22 |
+
s = seq.copy().astype(np.float32)
|
| 23 |
+
hip = (s[:,11,:] + s[:,12,:]) / 2.0 # centro caderas
|
| 24 |
+
s = s - hip[:,None,:]
|
| 25 |
+
sh = (s[:,5,:] + s[:,6,:]) / 2.0 # centro hombros
|
| 26 |
+
torso = np.linalg.norm(sh - 0.0, axis=1) # dist hombros->cadera(0)
|
| 27 |
+
scale = np.clip(np.median(torso), 1e-3, None)
|
| 28 |
+
s = s / scale
|
| 29 |
+
return s.reshape(s.shape[0], -1) # [T,34]
|
| 30 |
+
|
| 31 |
+
class MotionDS(Dataset):
|
| 32 |
+
def __init__(self):
|
| 33 |
+
self.windows = []
|
| 34 |
+
files = sorted(glob.glob(os.path.join(POSES, "*.npy")))
|
| 35 |
+
for f in files:
|
| 36 |
+
seq = np.load(f)
|
| 37 |
+
if seq.shape[0] < WIN + 1: continue
|
| 38 |
+
n = normalize(seq)
|
| 39 |
+
for i in range(0, n.shape[0] - WIN - 1, WIN // 2):
|
| 40 |
+
self.windows.append(n[i:i+WIN+1])
|
| 41 |
+
if self.windows:
|
| 42 |
+
allp = np.concatenate(self.windows, 0)
|
| 43 |
+
self.mean = allp.mean(0, keepdims=True); self.std = allp.std(0, keepdims=True) + 1e-6
|
| 44 |
+
self.windows = [(w - self.mean) / self.std for w in self.windows]
|
| 45 |
+
print(f"Ventanas de entrenamiento: {len(self.windows)} | clips: {len(files)}", flush=True)
|
| 46 |
+
def __len__(self): return len(self.windows)
|
| 47 |
+
def __getitem__(self, i):
|
| 48 |
+
w = torch.from_numpy(self.windows[i].astype(np.float32))
|
| 49 |
+
return w[:-1], w[1:] # input, target (shift 1)
|
| 50 |
+
|
| 51 |
+
class MotionDNA(nn.Module):
|
| 52 |
+
def __init__(self):
|
| 53 |
+
super().__init__()
|
| 54 |
+
self.inp = nn.Linear(DIM, D_MODEL)
|
| 55 |
+
self.pos = nn.Parameter(torch.randn(1, WIN, D_MODEL) * 0.02)
|
| 56 |
+
layer = nn.TransformerEncoderLayer(D_MODEL, N_HEAD, D_MODEL*4, DROP, batch_first=True, activation="gelu")
|
| 57 |
+
self.enc = nn.TransformerEncoder(layer, N_LAYER)
|
| 58 |
+
self.out = nn.Linear(D_MODEL, DIM)
|
| 59 |
+
def forward(self, x):
|
| 60 |
+
T = x.size(1)
|
| 61 |
+
h = self.inp(x) + self.pos[:, :T]
|
| 62 |
+
mask = torch.triu(torch.ones(T, T, device=x.device), 1).bool()
|
| 63 |
+
h = self.enc(h, mask=mask)
|
| 64 |
+
return self.out(h)
|
| 65 |
+
|
| 66 |
+
def main():
|
| 67 |
+
ds = MotionDS()
|
| 68 |
+
if len(ds) == 0:
|
| 69 |
+
print("ERROR: no hay ventanas. Corre extract_poses.py primero."); return
|
| 70 |
+
dl = DataLoader(ds, batch_size=BATCH, shuffle=True, num_workers=4, drop_last=True)
|
| 71 |
+
model = MotionDNA().to(DEV)
|
| 72 |
+
nparam = sum(p.numel() for p in model.parameters())
|
| 73 |
+
print(f"Device: {DEV} | Parametros: {nparam/1e6:.2f}M", flush=True)
|
| 74 |
+
opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
|
| 75 |
+
sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, EPOCHS)
|
| 76 |
+
lossf = nn.SmoothL1Loss()
|
| 77 |
+
best = 1e9; hist = []
|
| 78 |
+
for ep in range(1, EPOCHS+1):
|
| 79 |
+
model.train(); tot = 0; nb = 0
|
| 80 |
+
for x, y in dl:
|
| 81 |
+
x, y = x.to(DEV), y.to(DEV)
|
| 82 |
+
pred = model(x)
|
| 83 |
+
loss = lossf(pred, y)
|
| 84 |
+
opt.zero_grad(); loss.backward()
|
| 85 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 86 |
+
opt.step()
|
| 87 |
+
tot += loss.item(); nb += 1
|
| 88 |
+
sched.step()
|
| 89 |
+
avg = tot / max(nb,1); hist.append(avg)
|
| 90 |
+
mem = torch.cuda.max_memory_allocated()/1e9 if DEV=="cuda" else 0
|
| 91 |
+
print(f"Epoch {ep:03d}/{EPOCHS} loss={avg:.5f} lr={sched.get_last_lr()[0]:.2e} vram={mem:.2f}GB", flush=True)
|
| 92 |
+
if avg < best:
|
| 93 |
+
best = avg
|
| 94 |
+
torch.save({"model": model.state_dict(), "mean": ds.mean, "std": ds.std,
|
| 95 |
+
"cfg": {"WIN":WIN,"DIM":DIM,"D_MODEL":D_MODEL,"N_HEAD":N_HEAD,"N_LAYER":N_LAYER},
|
| 96 |
+
"epoch": ep, "loss": best}, os.path.join(CKPT, "motiondna_v0_best.pt"))
|
| 97 |
+
torch.save({"model": model.state_dict(), "mean": ds.mean, "std": ds.std,
|
| 98 |
+
"cfg": {"WIN":WIN,"DIM":DIM,"D_MODEL":D_MODEL,"N_HEAD":N_HEAD,"N_LAYER":N_LAYER},
|
| 99 |
+
"epoch": EPOCHS, "loss": hist[-1]}, os.path.join(CKPT, "motiondna_v0_final.pt"))
|
| 100 |
+
json.dump(hist, open(os.path.join(CKPT, "loss_history.json"), "w"))
|
| 101 |
+
# --- rollout de muestra: genera 120 frames de movimiento nuevo ---
|
| 102 |
+
model.eval()
|
| 103 |
+
with torch.no_grad():
|
| 104 |
+
seed = ds[0][0][:16].unsqueeze(0).to(DEV) # 16 frames semilla
|
| 105 |
+
gen = seed.clone()
|
| 106 |
+
for _ in range(120):
|
| 107 |
+
nxt = model(gen[:, -WIN:])[:, -1:]
|
| 108 |
+
gen = torch.cat([gen, nxt], 1)
|
| 109 |
+
gen = gen[0].cpu().numpy() * ds.std + ds.mean # des-normaliza
|
| 110 |
+
np.save(os.path.join(SAMP, "motiondna_sample.npy"), gen.reshape(-1,17,2))
|
| 111 |
+
print(f"LISTO. best_loss={best:.5f} ckpt={CKPT}/motiondna_v0_best.pt sample={SAMP}/motiondna_sample.npy", flush=True)
|
| 112 |
+
|
| 113 |
+
if __name__ == "__main__":
|
| 114 |
+
main()
|