bimanual-pose-vae / train.py
WhynotGPT's picture
Release joint_mix40 VAE weights, training and GT neighborhood sampling code
34988af verified
Raw History Blame Contribute Delete
5.83 kB
"""Training algorithm preserved from the original 2026-09-27 experiment.
Only environment/path plumbing and optional smoke-run lengths are adapted.
"""
from pathlib import Path
import json, hashlib
import numpy as np
from mano_compat import enable_legacy_mano_loading
enable_legacy_mano_loading()
def sha(path):
with Path(path).open('rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()
def write(path,value):
Path(path).parent.mkdir(parents=True,exist_ok=True)
Path(path).write_text(json.dumps(value,indent=2,allow_nan=False)+'\n')
import torch,time,smplx,argparse
from bimanual_vae import *
torch.set_num_threads(2)
def main(variant, data, mano_dir, output, device, epochs, steps_per_epoch):
output.mkdir(parents=True,exist_ok=True)
seed=20260927;torch.manual_seed(seed);rng=np.random.default_rng(seed);a=np.load(data,allow_pickle=False);local=aa_to_matrix(torch.tensor(a['pose_aa'],device=device)).reshape(-1,30,3,3);rot=torch.cat([local,torch.tensor(a['relative_rotation'],device=device)[:,None]],1);pos=torch.tensor(a['relative_wrist_m'],device=device);ds=a['dataset'];split=a['split'];seq=a['sequence'];included=[0,1] if variant=='joint_ego40' else [0,1,2];train=np.flatnonzero((split==0)&np.isin(ds,included));val=np.flatnonzero((split==1)&np.isin(ds,included));model=BimanualVAE(pose_only=variant=='pose_mix40').to(device);x=model.features(rot[train],pos[train]);model.input_mean.copy_(x.mean(0));model.input_std.copy_(x.std(0).clamp_min(.03));opt=torch.optim.AdamW(model.parameters(),lr=.001,weight_decay=1e-5)
# Balance datasets, then recordings; long recordings cannot dominate.
groups={d:{int(s):np.flatnonzero((split==0)&(ds==d)&(seq==s)) for s in np.unique(seq[(split==0)&(ds==d)])} for d in included}
# Simple FK of native 16 MANO skeletal joints, differentiated without rendering meshes.
offsets=[];parents=None
for s in range(2):
m=smplx.MANO(str(mano_dir),is_rhand=s==1,use_pca=False,flat_hand_mean=True);j=m.J_regressor@m.v_template;par=m.parents.numpy();off=j.clone();off[1:]-=j[par[1:]];off[0]=0;offsets.append(off);parents=par
off=torch.stack(offsets).to(device)
def fk(q):
q=q[:,:30].reshape(-1,2,15,3,3);n=len(q);g=[torch.eye(3,device=device).expand(n,2,3,3)];j=[torch.zeros(n,2,3,device=device)]
for k in range(1,16):
p=int(parents[k]);j.append(j[p]+(g[p]@off[None,:,k,:,None]).squeeze(-1));g.append(g[p]@q[:,:,k-1])
return torch.stack(j,2)
with torch.no_grad():ref=fk(rot)
cfg=dict(variant=variant,seed=seed,epochs=epochs,steps_per_epoch=steps_per_epoch,batch=192,lr=.001,kl_beta=.0005,joint_weight=.25,position_scale_m=.1,balanced_dataset_and_sequence=True,included_datasets=included,selection='equal-dataset validation local FK mm + 0.25 relative position mm + relative rotation deg; test not used',data_sha256=sha(data))
write(output/f'{variant}_config.json',cfg);best=float('inf');hist=[];start=time.monotonic()
for epoch in range(1,cfg['epochs']+1):
model.train();sums=[]
for step in range(cfg['steps_per_epoch']):
ids=[]
for d in included:
keys=list(groups[d]);ss=rng.choice(keys,192//len(included));ids.extend(int(rng.choice(groups[d][int(s)])) for s in ss)
ids=torch.tensor(ids,device=device);rr,pp,mu,lv=model(rot[ids],pos[ids]);rec=(rr-rot[ids,:model.nrot]).square().sum((-1,-2)).mean();jloss=((fk(rr)-ref[ids])/.02).square().sum(-1).mean();ploss=rec*0 if model.pose_only else ((pp-pos[ids])/.1).square().sum(-1).mean();kl=.5*(mu.square()+lv.exp()-lv-1).mean();loss=rec+.25*jloss+ploss+.0005*min(1,epoch/20)*kl;opt.zero_grad();loss.backward();torch.nn.utils.clip_grad_norm_(model.parameters(),5);opt.step();sums.append(float(loss.detach()))
if epoch==1 or epoch%5==0:
model.eval();met=[]
with torch.no_grad():
for d in included:
ids=val[ds[val]==d];errs=[];pe=[];re=[]
for t in range(0,len(ids),1024):
ix=ids[t:t+1024];rr,pp,mu,lv=model(rot[ix],pos[ix]);errs.append((fk(rr)-ref[ix]).norm(dim=-1).mean((1,2))*1000)
if not model.pose_only:pe.append((pp-pos[ix]).norm(dim=-1)*1000);rel=rr[:,30]@rot[ix,30].transpose(-1,-2);re.append(torch.acos(((rel.diagonal(dim1=-2,dim2=-1).sum(-1)-1)/2).clamp(-1+1e-7,1-1e-7))*180/torch.pi)
met.append(dict(dataset=['ARCTIC','H2O','GRAB'][d],local_fk_mm=float(torch.cat(errs).mean()),relative_wrist_mm=float(torch.cat(pe).mean()) if pe else None,relative_rotation_deg=float(torch.cat(re).mean()) if re else None))
score=float(np.mean([v['local_fk_mm']+.25*(v['relative_wrist_mm'] or 0)+(v['relative_rotation_deg'] or 0) for v in met]));entry=dict(epoch=epoch,train_loss=float(np.mean(sums)),validation=met,selection_score=score,seconds=time.monotonic()-start);hist.append(entry);print(variant,json.dumps(entry),flush=True)
if score<best:
best=score;torch.save(dict(architecture=dict(latent_dim=40,width=384,pose_only=model.pose_only),state_dict=model.state_dict(),config=cfg,epoch=epoch,validation=met),output/f'{variant}.pt')
write(output/f'{variant}_history.json',hist)
print('COMPLETE',variant,sha(output/f'{variant}.pt'),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser(description='Original paired-hand VAE recipe with portable paths.')
p.add_argument('--variant',choices=['joint_ego40','joint_mix40','pose_mix40'],default='joint_mix40')
p.add_argument('--data',type=Path,required=True)
p.add_argument('--mano-dir',type=Path,required=True)
p.add_argument('--output',type=Path,required=True)
p.add_argument('--device',default='cuda')
p.add_argument('--epochs',type=int,default=60)
p.add_argument('--steps-per-epoch',type=int,default=48)
a=p.parse_args()
if a.epochs<1 or a.steps_per_epoch<1:p.error('Training counts must be positive.')
if (a.output/(a.variant+'.pt')).exists():p.error('Output checkpoint already exists; choose a new output folder.')
main(a.variant,a.data,a.mano_dir,a.output,a.device,a.epochs,a.steps_per_epoch)