bimanual-pose-vae / augment.py
WhynotGPT's picture
Release joint_mix40 VAE weights, training and GT neighborhood sampling code
34988af verified
Raw History Blame Contribute Delete
7.59 kB
"""Encode training GT and sample distinct-recording latent neighborhoods.
Inputs are numeric NPZ arrays, described in DATA_FORMAT.md. Mesh generation is
external; the output contains full MANO rotations and actual wrist positions.
"""
import argparse
import hashlib
from pathlib import Path
import numpy as np
import torch
from hand_vae import aa_to_matrix, matrix_to_aa
from load_model import load_model
def encode_pose(model, pose, relative_rotation, relative_wrist):
local = aa_to_matrix(torch.as_tensor(pose, dtype=torch.float32)).reshape(-1, 30, 3, 3)
rotations = torch.cat([local, torch.as_tensor(relative_rotation, dtype=torch.float32)[:, None]], 1)
return model.encode(rotations, torch.as_tensor(relative_wrist, dtype=torch.float32))
@torch.inference_mode()
def encode(data_path, output):
model, _ = load_model()
with np.load(data_path, allow_pickle=False) as data:
rows = np.flatnonzero(data["split"] == 0)
if not len(rows):
raise ValueError("No split=0 training rows.")
payload = {name: data[name][rows] for name in ("sequence", "dataset")}
if "frame_ids" in data:
payload["frame_ids"] = data["frame_ids"][rows]
poses = {name: data[name][rows] for name in ("pose_aa", "relative_rotation", "relative_wrist_m")}
means, logvars = [], []
for start in range(0, len(rows), 2048):
ix = slice(start, start + 2048)
mu, logvar = encode_pose(model, poses["pose_aa"][ix], poses["relative_rotation"][ix], poses["relative_wrist_m"][ix])
means.append(mu.numpy())
logvars.append(logvar.numpy())
with data_path.open("rb") as stream:
source_hash = hashlib.file_digest(stream, "sha256").hexdigest()
np.savez_compressed(output, mu=np.concatenate(means), logvar=np.concatenate(logvars),
source_row=rows, split=np.zeros(len(rows), dtype=np.int8),
source_sha256=np.array(source_hash), **payload)
print(f"Encoded {len(rows)} training GT rows into {output}")
@torch.inference_mode()
def sample(pool_path, query_path, output, neighbor_count, directions, scales, seed):
model, _ = load_model()
rng = np.random.default_rng(seed)
with np.load(pool_path, allow_pickle=False) as data:
pool = {key: data[key] for key in data.files}
if not np.all(pool["split"] == 0):
raise ValueError("The candidate pool must contain only training GT.")
with np.load(query_path, allow_pickle=False) as queries:
# Queries identify difficult GT regions; held-out test queries are excluded.
if not np.isin(queries["split"], [0, 1]).all():
raise ValueError("Queries must use train/validation GT, never test rows.")
query_datasets = queries["dataset"].copy()
query_mu, _ = encode_pose(model, queries["pose_aa"], queries["relative_rotation"], queries["relative_wrist_m"])
query_mu = query_mu.numpy()
if not len(query_mu):
raise ValueError("The query file is empty.")
if not np.isfinite(pool["mu"]).all() or not np.isfinite(query_mu).all():
raise ValueError("Latents must be finite.")
if neighbor_count < 1 or directions < 1 or not len(scales) or np.any((scales < 0) | (scales > 1)) or not np.isfinite(scales).all():
raise ValueError("Use positive counts and finite interpolation scales in [0,1].")
anchors, neighbors, pairs, weights, latent_batches = [], [], [], [], []
for query, dataset in zip(query_mu, query_datasets):
ids = np.flatnonzero(pool["dataset"] == dataset)
if not len(ids):
raise ValueError(f"No training rows for dataset {dataset}.")
anchor = int(ids[np.argmin(np.square(pool["mu"][ids] - query).sum(1))])
order = np.argsort(np.square(pool["mu"][ids] - pool["mu"][anchor]).sum(1))
seen = {int(pool["sequence"][anchor])}
nearest = []
for row in ids[order]:
sequence = int(pool["sequence"][row])
if sequence not in seen:
nearest.append(int(row))
seen.add(sequence)
if len(nearest) == neighbor_count:
break
if len(nearest) != neighbor_count:
raise ValueError(f"Dataset {dataset} needs {neighbor_count + 1} distinct training recordings.")
nearest = np.asarray(nearest)
pair = rng.choice(neighbor_count, (directions, 2), replace=True)
weight = rng.uniform(.2, .8, directions).astype(np.float32)
endpoint = weight[:, None] * pool["mu"][nearest[pair[:, 0]]] + (1 - weight[:, None]) * pool["mu"][nearest[pair[:, 1]]]
z = (1 - scales[:, None, None]) * pool["mu"][anchor] + scales[:, None, None] * endpoint[None]
latent_batches.append(z.reshape(-1, model.latent_dim))
anchors.append(anchor)
neighbors.append(nearest)
pairs.append(pair)
weights.append(weight)
z = np.concatenate(latent_batches).astype(np.float32)
rot, pos = model.decode(torch.from_numpy(z))
full = torch.zeros(len(z), 2, 16, 3, 3)
full[:, 0, 0] = torch.eye(3)
full[:, 1, 0] = rot[:, 30]
full[:, :, 1:] = rot[:, :30].reshape(-1, 2, 15, 3, 3)
wrist = torch.zeros(len(z), 2, 3)
wrist[:, 1] = pos
anchors = np.asarray(anchors)
query_index = np.repeat(np.arange(len(anchors)), len(scales) * directions)
np.savez_compressed(
output, z=z, mano_rotation_matrices=full.numpy(), mano_pose_axis_angle=matrix_to_aa(full).numpy(),
wrist_positions_m=wrist.numpy(), mano_shape=np.zeros((len(z), 2, 10), np.float32),
query_index=query_index, train_pool_row=anchors[query_index],
train_source_row=pool["source_row"][anchors[query_index]],
train_sequence=pool["sequence"][anchors[query_index]],
train_dataset=pool["dataset"][anchors[query_index]],
temperature=np.tile(np.repeat(scales, directions), len(anchors)),
draw=np.tile(np.arange(directions), len(anchors) * len(scales)),
anchor_pool_rows=anchors, neighbor_pool_rows=np.asarray(neighbors),
endpoint_neighbor_indices=np.asarray(pairs), endpoint_weights=np.asarray(weights),
seed=np.array(seed), coordinate_system=np.array("left_wrist_local"),
)
print(f"Saved {len(z)} synthetic bimanual pose candidates to {output}")
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
enc = commands.add_parser("encode")
enc.add_argument("--data", type=Path, required=True)
enc.add_argument("--output", type=Path, required=True)
sam = commands.add_parser("sample")
sam.add_argument("--pool", type=Path, required=True)
sam.add_argument("--queries", type=Path, required=True)
sam.add_argument("--output", type=Path, required=True)
sam.add_argument("--neighbors", type=int, default=16)
sam.add_argument("--directions", type=int, default=8)
sam.add_argument("--scales", type=float, nargs="+", default=[.25, .5, 1.])
sam.add_argument("--seed", type=int, default=20261110)
args = parser.parse_args()
if args.output.exists():
parser.error("Output already exists; choose a new path.")
args.output.parent.mkdir(parents=True, exist_ok=True)
torch.set_num_threads(4)
if args.command == "encode":
encode(args.data, args.output)
else:
sample(args.pool, args.queries, args.output, args.neighbors, args.directions,
np.asarray(args.scales, dtype=np.float32), args.seed)
if __name__ == "__main__":
main()