File size: 6,380 Bytes
c7ae99a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | """Causal hierarchical tokenizer (CHierTok, unchanged) trained on the CMU 272-dim data.
Differences from tvq272c.py (training loop only, model unchanged):
- all train frames are used; windows (64 frames) are cut at random from the clips on the fly
(the old code stacked every window on the GPU, which does not fit for this much data)
- stops by wall-clock (--seconds); lr: 5% warm-up, then cosine over elapsed / budget
- overfitting guard, same rule as gen_ar.py: every --val-every steps measure validation
reconstruction loss; keep the best checkpoint; stop when it has not improved for --patience checks
Validation = "val" split, fixed windows (stride 64).
"""
import os, sys, json, time, math, argparse
import numpy as np
import torch
import torch.nn.functional as F
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import tvq272 as B
import tvq272c as C
def load_clips(split):
mdir = B.find_motion_dir()
ids = open(os.path.join(B.ROOT, "split", split + ".txt")).read().split()
out = []
for i in ids:
p = os.path.join(mdir, i + ".npy")
if os.path.exists(p):
a = np.load(p).astype(np.float32)
if a.shape[0] >= B.T:
out.append(a)
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--seconds", type=float, default=7200)
ap.add_argument("--bs", type=int, default=256)
ap.add_argument("--lr", type=float, default=4e-4)
ap.add_argument("--beta", type=float, default=0.25)
ap.add_argument("--val-every", type=int, default=1000)
ap.add_argument("--patience", type=int, default=3)
ap.add_argument("--tag", default="cmu_tok_v1")
a = ap.parse_args()
dev = "cuda"
t0 = time.time()
mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy"))).to(dev)
std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy"))).to(dev)
tr = load_clips("train")
lens = np.array([c.shape[0] for c in tr])
starts = np.concatenate([[0], np.cumsum(lens)[:-1]])
# normalise on the CPU so only ONE 2 GB copy ever lives on the GPU
big_np = np.concatenate(tr); del tr
big_np -= mean.cpu().numpy(); big_np /= std.cpu().numpy()
big = torch.from_numpy(big_np).to(dev); del big_np
valid = torch.from_numpy(np.concatenate([s + np.arange(n - B.T + 1) for s, n in zip(starts, lens)])).to(dev)
va = load_clips("val")
Wva = np.stack([c[s:s + B.T] for c in va for s in range(0, c.shape[0] - B.T + 1, B.T)])
Xva = torch.from_numpy(Wva).to(dev)
Zva = (Xva - mean) / std
print("train clips %d, %d frames (%.2f h), %d window starts | val %d windows | load %.0fs"
% (len(lens), big.shape[0], big.shape[0] / 30 / 3600, valid.numel(), Zva.shape[0], time.time() - t0), flush=True)
mt = B.Metrics(mean, std, torch.stack([big[s:s + B.T] for s in valid[:: max(1, valid.numel() // 2000)][:2000]]) * std + mean)
ar = torch.arange(B.T, device=dev)
torch.cuda.empty_cache()
print("gpu memory after load: allocated %.2f GB, reserved %.2f GB"
% (torch.cuda.memory_allocated() / 1e9, torch.cuda.memory_reserved() / 1e9), flush=True)
torch.manual_seed(0)
m = C.CHierTok().to(dev).train()
print("params %d" % B.nparams(m), flush=True)
opt = torch.optim.AdamW(m.parameters(), lr=a.lr)
@torch.no_grad()
def val_loss():
m.eval()
tot = 0.0
for i in range(0, Zva.shape[0], 256):
x = Zva[i:i + 256]
tot += float(F.mse_loss(m(x)[0], x, reduction="sum"))
m.train()
return tot / Zva.numel()
best, bad, log, step = [float("inf"), 0], 0, [], 0
tt = time.time()
while True:
el = time.time() - tt
if el > a.seconds:
print(" time budget reached", flush=True)
break
frac = el / a.seconds
lr = a.lr * min(1.0, frac / 0.05) if frac < 0.05 else a.lr * 0.5 * (1 + math.cos(math.pi * (frac - 0.05) / 0.95))
for g in opt.param_groups:
g["lr"] = max(lr, a.lr * 1e-3)
s = valid[torch.randint(0, valid.numel(), (a.bs,), device=dev)]
xb = big[s[:, None] + ar[None]]
y, commit = m(xb)
loss = F.mse_loss(y, xb) + a.beta * commit
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
opt.step()
step += 1
if step == 20:
torch.cuda.synchronize()
print(" speed %.1f step/s -> ~%d steps in budget | gpu peak %.2f GB" % (20 / (time.time() - tt), a.seconds * 20 / (time.time() - tt), torch.cuda.max_memory_allocated() / 1e9), flush=True)
if step % a.val_every == 0:
vl = val_loss()
row = dict(step=step, sec=round(time.time() - tt), lr=round(lr, 7), train_loss=round(loss.item(), 5),
val_recon=round(vl, 5))
if vl < best[0]:
best, bad = [vl, step], 0
torch.save(m.state_dict(), os.path.join(B.CKPT_DIR, a.tag + "_H.pt"))
row["saved"] = True
else:
bad += 1
log.append(row)
print(" " + json.dumps(row), flush=True)
if bad >= a.patience:
print(" early stop: val not improved for %d checks" % a.patience, flush=True)
break
secs = time.time() - tt
m.load_state_dict(torch.load(os.path.join(B.CKPT_DIR, a.tag + "_H.pt"), map_location=dev))
r = B.evaluate(m, Zva, mt)
r.update(params=B.nparams(m), steps=step, best_step=best[1], best_val_recon=best[0], train_seconds=round(secs, 1),
causality_maxdiff=C.causality_check(m, Zva))
json.dump(dict(result=r, args=vars(a), log=log), open(os.path.join(B.OUTDIR, a.tag + ".json"), "w", encoding="utf-8"),
ensure_ascii=False, indent=1)
print("best step %d | pos %.2f mm | rot local %.2f / global %.2f deg | seam %.2f mm | consist %.2f (gt %.2f) | causal %.1e | %.0fs"
% (best[1], r["mpjpe_mm"], r["rot_local_deg"], r["rot_global_deg"], r["seam_bone_mm"], r["consistency_deg"],
r["consistency_gt_deg"], r["causality_maxdiff"], secs), flush=True)
for pn, v in r["per_part"].items():
print(" %-6s pos %7.2f mm rot %6.2f deg" % (pn, v["pos_mm"], v["rot_global_deg"]), flush=True)
if __name__ == "__main__":
main()
|