File size: 6,380 Bytes
c7ae99a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
"""Causal hierarchical tokenizer (CHierTok, unchanged) trained on the CMU 272-dim data.

Differences from tvq272c.py (training loop only, model unchanged):
- all train frames are used; windows (64 frames) are cut at random from the clips on the fly
  (the old code stacked every window on the GPU, which does not fit for this much data)
- stops by wall-clock (--seconds); lr: 5% warm-up, then cosine over elapsed / budget
- overfitting guard, same rule as gen_ar.py: every --val-every steps measure validation
  reconstruction loss; keep the best checkpoint; stop when it has not improved for --patience checks
Validation = "val" split, fixed windows (stride 64).
"""
import os, sys, json, time, math, argparse
import numpy as np
import torch
import torch.nn.functional as F
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import tvq272 as B
import tvq272c as C


def load_clips(split):
    mdir = B.find_motion_dir()
    ids = open(os.path.join(B.ROOT, "split", split + ".txt")).read().split()
    out = []
    for i in ids:
        p = os.path.join(mdir, i + ".npy")
        if os.path.exists(p):
            a = np.load(p).astype(np.float32)
            if a.shape[0] >= B.T:
                out.append(a)
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--seconds", type=float, default=7200)
    ap.add_argument("--bs", type=int, default=256)
    ap.add_argument("--lr", type=float, default=4e-4)
    ap.add_argument("--beta", type=float, default=0.25)
    ap.add_argument("--val-every", type=int, default=1000)
    ap.add_argument("--patience", type=int, default=3)
    ap.add_argument("--tag", default="cmu_tok_v1")
    a = ap.parse_args()
    dev = "cuda"
    t0 = time.time()
    mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy"))).to(dev)
    std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy"))).to(dev)

    tr = load_clips("train")
    lens = np.array([c.shape[0] for c in tr])
    starts = np.concatenate([[0], np.cumsum(lens)[:-1]])
    # normalise on the CPU so only ONE 2 GB copy ever lives on the GPU
    big_np = np.concatenate(tr); del tr
    big_np -= mean.cpu().numpy(); big_np /= std.cpu().numpy()
    big = torch.from_numpy(big_np).to(dev); del big_np
    valid = torch.from_numpy(np.concatenate([s + np.arange(n - B.T + 1) for s, n in zip(starts, lens)])).to(dev)
    va = load_clips("val")
    Wva = np.stack([c[s:s + B.T] for c in va for s in range(0, c.shape[0] - B.T + 1, B.T)])
    Xva = torch.from_numpy(Wva).to(dev)
    Zva = (Xva - mean) / std
    print("train clips %d, %d frames (%.2f h), %d window starts | val %d windows | load %.0fs"
          % (len(lens), big.shape[0], big.shape[0] / 30 / 3600, valid.numel(), Zva.shape[0], time.time() - t0), flush=True)
    mt = B.Metrics(mean, std, torch.stack([big[s:s + B.T] for s in valid[:: max(1, valid.numel() // 2000)][:2000]]) * std + mean)
    ar = torch.arange(B.T, device=dev)
    torch.cuda.empty_cache()
    print("gpu memory after load: allocated %.2f GB, reserved %.2f GB"
          % (torch.cuda.memory_allocated() / 1e9, torch.cuda.memory_reserved() / 1e9), flush=True)

    torch.manual_seed(0)
    m = C.CHierTok().to(dev).train()
    print("params %d" % B.nparams(m), flush=True)
    opt = torch.optim.AdamW(m.parameters(), lr=a.lr)

    @torch.no_grad()
    def val_loss():
        m.eval()
        tot = 0.0
        for i in range(0, Zva.shape[0], 256):
            x = Zva[i:i + 256]
            tot += float(F.mse_loss(m(x)[0], x, reduction="sum"))
        m.train()
        return tot / Zva.numel()

    best, bad, log, step = [float("inf"), 0], 0, [], 0
    tt = time.time()
    while True:
        el = time.time() - tt
        if el > a.seconds:
            print("  time budget reached", flush=True)
            break
        frac = el / a.seconds
        lr = a.lr * min(1.0, frac / 0.05) if frac < 0.05 else a.lr * 0.5 * (1 + math.cos(math.pi * (frac - 0.05) / 0.95))
        for g in opt.param_groups:
            g["lr"] = max(lr, a.lr * 1e-3)
        s = valid[torch.randint(0, valid.numel(), (a.bs,), device=dev)]
        xb = big[s[:, None] + ar[None]]
        y, commit = m(xb)
        loss = F.mse_loss(y, xb) + a.beta * commit
        opt.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
        opt.step()
        step += 1
        if step == 20:
            torch.cuda.synchronize()
            print("  speed %.1f step/s -> ~%d steps in budget | gpu peak %.2f GB" % (20 / (time.time() - tt), a.seconds * 20 / (time.time() - tt), torch.cuda.max_memory_allocated() / 1e9), flush=True)
        if step % a.val_every == 0:
            vl = val_loss()
            row = dict(step=step, sec=round(time.time() - tt), lr=round(lr, 7), train_loss=round(loss.item(), 5),
                       val_recon=round(vl, 5))
            if vl < best[0]:
                best, bad = [vl, step], 0
                torch.save(m.state_dict(), os.path.join(B.CKPT_DIR, a.tag + "_H.pt"))
                row["saved"] = True
            else:
                bad += 1
            log.append(row)
            print("  " + json.dumps(row), flush=True)
            if bad >= a.patience:
                print("  early stop: val not improved for %d checks" % a.patience, flush=True)
                break
    secs = time.time() - tt
    m.load_state_dict(torch.load(os.path.join(B.CKPT_DIR, a.tag + "_H.pt"), map_location=dev))
    r = B.evaluate(m, Zva, mt)
    r.update(params=B.nparams(m), steps=step, best_step=best[1], best_val_recon=best[0], train_seconds=round(secs, 1),
             causality_maxdiff=C.causality_check(m, Zva))
    json.dump(dict(result=r, args=vars(a), log=log), open(os.path.join(B.OUTDIR, a.tag + ".json"), "w", encoding="utf-8"),
              ensure_ascii=False, indent=1)
    print("best step %d | pos %.2f mm | rot local %.2f / global %.2f deg | seam %.2f mm | consist %.2f (gt %.2f) | causal %.1e | %.0fs"
          % (best[1], r["mpjpe_mm"], r["rot_local_deg"], r["rot_global_deg"], r["seam_bone_mm"], r["consistency_deg"],
             r["consistency_gt_deg"], r["causality_maxdiff"], secs), flush=True)
    for pn, v in r["per_part"].items():
        print("    %-6s pos %7.2f mm  rot %6.2f deg" % (pn, v["pos_mm"], v["rot_global_deg"]), flush=True)


if __name__ == "__main__":
    main()