Download augment.py from WhynotGPT/bimanual-pose-vae: direct link, hf CLI and curl.
- Browser
- Download file 7.59 kB
-
https://huggingface.co/WhynotGPT/bimanual-pose-vae/resolve/main/augment.py
- Command line
-
hf download hf://WhynotGPT/bimanual-pose-vae/augment.py
-
curl -L -o augment.py https://huggingface.co/WhynotGPT/bimanual-pose-vae/resolve/main/augment.py
7.59 kB
| """Encode training GT and sample distinct-recording latent neighborhoods. | |
| Inputs are numeric NPZ arrays, described in DATA_FORMAT.md. Mesh generation is | |
| external; the output contains full MANO rotations and actual wrist positions. | |
| """ | |
| import argparse | |
| import hashlib | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from hand_vae import aa_to_matrix, matrix_to_aa | |
| from load_model import load_model | |
| def encode_pose(model, pose, relative_rotation, relative_wrist): | |
| local = aa_to_matrix(torch.as_tensor(pose, dtype=torch.float32)).reshape(-1, 30, 3, 3) | |
| rotations = torch.cat([local, torch.as_tensor(relative_rotation, dtype=torch.float32)[:, None]], 1) | |
| return model.encode(rotations, torch.as_tensor(relative_wrist, dtype=torch.float32)) | |
| def encode(data_path, output): | |
| model, _ = load_model() | |
| with np.load(data_path, allow_pickle=False) as data: | |
| rows = np.flatnonzero(data["split"] == 0) | |
| if not len(rows): | |
| raise ValueError("No split=0 training rows.") | |
| payload = {name: data[name][rows] for name in ("sequence", "dataset")} | |
| if "frame_ids" in data: | |
| payload["frame_ids"] = data["frame_ids"][rows] | |
| poses = {name: data[name][rows] for name in ("pose_aa", "relative_rotation", "relative_wrist_m")} | |
| means, logvars = [], [] | |
| for start in range(0, len(rows), 2048): | |
| ix = slice(start, start + 2048) | |
| mu, logvar = encode_pose(model, poses["pose_aa"][ix], poses["relative_rotation"][ix], poses["relative_wrist_m"][ix]) | |
| means.append(mu.numpy()) | |
| logvars.append(logvar.numpy()) | |
| with data_path.open("rb") as stream: | |
| source_hash = hashlib.file_digest(stream, "sha256").hexdigest() | |
| np.savez_compressed(output, mu=np.concatenate(means), logvar=np.concatenate(logvars), | |
| source_row=rows, split=np.zeros(len(rows), dtype=np.int8), | |
| source_sha256=np.array(source_hash), **payload) | |
| print(f"Encoded {len(rows)} training GT rows into {output}") | |
| def sample(pool_path, query_path, output, neighbor_count, directions, scales, seed): | |
| model, _ = load_model() | |
| rng = np.random.default_rng(seed) | |
| with np.load(pool_path, allow_pickle=False) as data: | |
| pool = {key: data[key] for key in data.files} | |
| if not np.all(pool["split"] == 0): | |
| raise ValueError("The candidate pool must contain only training GT.") | |
| with np.load(query_path, allow_pickle=False) as queries: | |
| # Queries identify difficult GT regions; held-out test queries are excluded. | |
| if not np.isin(queries["split"], [0, 1]).all(): | |
| raise ValueError("Queries must use train/validation GT, never test rows.") | |
| query_datasets = queries["dataset"].copy() | |
| query_mu, _ = encode_pose(model, queries["pose_aa"], queries["relative_rotation"], queries["relative_wrist_m"]) | |
| query_mu = query_mu.numpy() | |
| if not len(query_mu): | |
| raise ValueError("The query file is empty.") | |
| if not np.isfinite(pool["mu"]).all() or not np.isfinite(query_mu).all(): | |
| raise ValueError("Latents must be finite.") | |
| if neighbor_count < 1 or directions < 1 or not len(scales) or np.any((scales < 0) | (scales > 1)) or not np.isfinite(scales).all(): | |
| raise ValueError("Use positive counts and finite interpolation scales in [0,1].") | |
| anchors, neighbors, pairs, weights, latent_batches = [], [], [], [], [] | |
| for query, dataset in zip(query_mu, query_datasets): | |
| ids = np.flatnonzero(pool["dataset"] == dataset) | |
| if not len(ids): | |
| raise ValueError(f"No training rows for dataset {dataset}.") | |
| anchor = int(ids[np.argmin(np.square(pool["mu"][ids] - query).sum(1))]) | |
| order = np.argsort(np.square(pool["mu"][ids] - pool["mu"][anchor]).sum(1)) | |
| seen = {int(pool["sequence"][anchor])} | |
| nearest = [] | |
| for row in ids[order]: | |
| sequence = int(pool["sequence"][row]) | |
| if sequence not in seen: | |
| nearest.append(int(row)) | |
| seen.add(sequence) | |
| if len(nearest) == neighbor_count: | |
| break | |
| if len(nearest) != neighbor_count: | |
| raise ValueError(f"Dataset {dataset} needs {neighbor_count + 1} distinct training recordings.") | |
| nearest = np.asarray(nearest) | |
| pair = rng.choice(neighbor_count, (directions, 2), replace=True) | |
| weight = rng.uniform(.2, .8, directions).astype(np.float32) | |
| endpoint = weight[:, None] * pool["mu"][nearest[pair[:, 0]]] + (1 - weight[:, None]) * pool["mu"][nearest[pair[:, 1]]] | |
| z = (1 - scales[:, None, None]) * pool["mu"][anchor] + scales[:, None, None] * endpoint[None] | |
| latent_batches.append(z.reshape(-1, model.latent_dim)) | |
| anchors.append(anchor) | |
| neighbors.append(nearest) | |
| pairs.append(pair) | |
| weights.append(weight) | |
| z = np.concatenate(latent_batches).astype(np.float32) | |
| rot, pos = model.decode(torch.from_numpy(z)) | |
| full = torch.zeros(len(z), 2, 16, 3, 3) | |
| full[:, 0, 0] = torch.eye(3) | |
| full[:, 1, 0] = rot[:, 30] | |
| full[:, :, 1:] = rot[:, :30].reshape(-1, 2, 15, 3, 3) | |
| wrist = torch.zeros(len(z), 2, 3) | |
| wrist[:, 1] = pos | |
| anchors = np.asarray(anchors) | |
| query_index = np.repeat(np.arange(len(anchors)), len(scales) * directions) | |
| np.savez_compressed( | |
| output, z=z, mano_rotation_matrices=full.numpy(), mano_pose_axis_angle=matrix_to_aa(full).numpy(), | |
| wrist_positions_m=wrist.numpy(), mano_shape=np.zeros((len(z), 2, 10), np.float32), | |
| query_index=query_index, train_pool_row=anchors[query_index], | |
| train_source_row=pool["source_row"][anchors[query_index]], | |
| train_sequence=pool["sequence"][anchors[query_index]], | |
| train_dataset=pool["dataset"][anchors[query_index]], | |
| temperature=np.tile(np.repeat(scales, directions), len(anchors)), | |
| draw=np.tile(np.arange(directions), len(anchors) * len(scales)), | |
| anchor_pool_rows=anchors, neighbor_pool_rows=np.asarray(neighbors), | |
| endpoint_neighbor_indices=np.asarray(pairs), endpoint_weights=np.asarray(weights), | |
| seed=np.array(seed), coordinate_system=np.array("left_wrist_local"), | |
| ) | |
| print(f"Saved {len(z)} synthetic bimanual pose candidates to {output}") | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| commands = parser.add_subparsers(dest="command", required=True) | |
| enc = commands.add_parser("encode") | |
| enc.add_argument("--data", type=Path, required=True) | |
| enc.add_argument("--output", type=Path, required=True) | |
| sam = commands.add_parser("sample") | |
| sam.add_argument("--pool", type=Path, required=True) | |
| sam.add_argument("--queries", type=Path, required=True) | |
| sam.add_argument("--output", type=Path, required=True) | |
| sam.add_argument("--neighbors", type=int, default=16) | |
| sam.add_argument("--directions", type=int, default=8) | |
| sam.add_argument("--scales", type=float, nargs="+", default=[.25, .5, 1.]) | |
| sam.add_argument("--seed", type=int, default=20261110) | |
| args = parser.parse_args() | |
| if args.output.exists(): | |
| parser.error("Output already exists; choose a new path.") | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| torch.set_num_threads(4) | |
| if args.command == "encode": | |
| encode(args.data, args.output) | |
| else: | |
| sample(args.pool, args.queries, args.output, args.neighbors, args.directions, | |
| np.asarray(args.scales, dtype=np.float32), args.seed) | |
| if __name__ == "__main__": | |
| main() | |