"""Encode training GT and sample distinct-recording latent neighborhoods. Inputs are numeric NPZ arrays, described in DATA_FORMAT.md. Mesh generation is external; the output contains full MANO rotations and actual wrist positions. """ import argparse import hashlib from pathlib import Path import numpy as np import torch from hand_vae import aa_to_matrix, matrix_to_aa from load_model import load_model def encode_pose(model, pose, relative_rotation, relative_wrist): local = aa_to_matrix(torch.as_tensor(pose, dtype=torch.float32)).reshape(-1, 30, 3, 3) rotations = torch.cat([local, torch.as_tensor(relative_rotation, dtype=torch.float32)[:, None]], 1) return model.encode(rotations, torch.as_tensor(relative_wrist, dtype=torch.float32)) @torch.inference_mode() def encode(data_path, output): model, _ = load_model() with np.load(data_path, allow_pickle=False) as data: rows = np.flatnonzero(data["split"] == 0) if not len(rows): raise ValueError("No split=0 training rows.") payload = {name: data[name][rows] for name in ("sequence", "dataset")} if "frame_ids" in data: payload["frame_ids"] = data["frame_ids"][rows] poses = {name: data[name][rows] for name in ("pose_aa", "relative_rotation", "relative_wrist_m")} means, logvars = [], [] for start in range(0, len(rows), 2048): ix = slice(start, start + 2048) mu, logvar = encode_pose(model, poses["pose_aa"][ix], poses["relative_rotation"][ix], poses["relative_wrist_m"][ix]) means.append(mu.numpy()) logvars.append(logvar.numpy()) with data_path.open("rb") as stream: source_hash = hashlib.file_digest(stream, "sha256").hexdigest() np.savez_compressed(output, mu=np.concatenate(means), logvar=np.concatenate(logvars), source_row=rows, split=np.zeros(len(rows), dtype=np.int8), source_sha256=np.array(source_hash), **payload) print(f"Encoded {len(rows)} training GT rows into {output}") @torch.inference_mode() def sample(pool_path, query_path, output, neighbor_count, directions, scales, seed): model, _ = load_model() rng = np.random.default_rng(seed) with np.load(pool_path, allow_pickle=False) as data: pool = {key: data[key] for key in data.files} if not np.all(pool["split"] == 0): raise ValueError("The candidate pool must contain only training GT.") with np.load(query_path, allow_pickle=False) as queries: # Queries identify difficult GT regions; held-out test queries are excluded. if not np.isin(queries["split"], [0, 1]).all(): raise ValueError("Queries must use train/validation GT, never test rows.") query_datasets = queries["dataset"].copy() query_mu, _ = encode_pose(model, queries["pose_aa"], queries["relative_rotation"], queries["relative_wrist_m"]) query_mu = query_mu.numpy() if not len(query_mu): raise ValueError("The query file is empty.") if not np.isfinite(pool["mu"]).all() or not np.isfinite(query_mu).all(): raise ValueError("Latents must be finite.") if neighbor_count < 1 or directions < 1 or not len(scales) or np.any((scales < 0) | (scales > 1)) or not np.isfinite(scales).all(): raise ValueError("Use positive counts and finite interpolation scales in [0,1].") anchors, neighbors, pairs, weights, latent_batches = [], [], [], [], [] for query, dataset in zip(query_mu, query_datasets): ids = np.flatnonzero(pool["dataset"] == dataset) if not len(ids): raise ValueError(f"No training rows for dataset {dataset}.") anchor = int(ids[np.argmin(np.square(pool["mu"][ids] - query).sum(1))]) order = np.argsort(np.square(pool["mu"][ids] - pool["mu"][anchor]).sum(1)) seen = {int(pool["sequence"][anchor])} nearest = [] for row in ids[order]: sequence = int(pool["sequence"][row]) if sequence not in seen: nearest.append(int(row)) seen.add(sequence) if len(nearest) == neighbor_count: break if len(nearest) != neighbor_count: raise ValueError(f"Dataset {dataset} needs {neighbor_count + 1} distinct training recordings.") nearest = np.asarray(nearest) pair = rng.choice(neighbor_count, (directions, 2), replace=True) weight = rng.uniform(.2, .8, directions).astype(np.float32) endpoint = weight[:, None] * pool["mu"][nearest[pair[:, 0]]] + (1 - weight[:, None]) * pool["mu"][nearest[pair[:, 1]]] z = (1 - scales[:, None, None]) * pool["mu"][anchor] + scales[:, None, None] * endpoint[None] latent_batches.append(z.reshape(-1, model.latent_dim)) anchors.append(anchor) neighbors.append(nearest) pairs.append(pair) weights.append(weight) z = np.concatenate(latent_batches).astype(np.float32) rot, pos = model.decode(torch.from_numpy(z)) full = torch.zeros(len(z), 2, 16, 3, 3) full[:, 0, 0] = torch.eye(3) full[:, 1, 0] = rot[:, 30] full[:, :, 1:] = rot[:, :30].reshape(-1, 2, 15, 3, 3) wrist = torch.zeros(len(z), 2, 3) wrist[:, 1] = pos anchors = np.asarray(anchors) query_index = np.repeat(np.arange(len(anchors)), len(scales) * directions) np.savez_compressed( output, z=z, mano_rotation_matrices=full.numpy(), mano_pose_axis_angle=matrix_to_aa(full).numpy(), wrist_positions_m=wrist.numpy(), mano_shape=np.zeros((len(z), 2, 10), np.float32), query_index=query_index, train_pool_row=anchors[query_index], train_source_row=pool["source_row"][anchors[query_index]], train_sequence=pool["sequence"][anchors[query_index]], train_dataset=pool["dataset"][anchors[query_index]], temperature=np.tile(np.repeat(scales, directions), len(anchors)), draw=np.tile(np.arange(directions), len(anchors) * len(scales)), anchor_pool_rows=anchors, neighbor_pool_rows=np.asarray(neighbors), endpoint_neighbor_indices=np.asarray(pairs), endpoint_weights=np.asarray(weights), seed=np.array(seed), coordinate_system=np.array("left_wrist_local"), ) print(f"Saved {len(z)} synthetic bimanual pose candidates to {output}") def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) enc = commands.add_parser("encode") enc.add_argument("--data", type=Path, required=True) enc.add_argument("--output", type=Path, required=True) sam = commands.add_parser("sample") sam.add_argument("--pool", type=Path, required=True) sam.add_argument("--queries", type=Path, required=True) sam.add_argument("--output", type=Path, required=True) sam.add_argument("--neighbors", type=int, default=16) sam.add_argument("--directions", type=int, default=8) sam.add_argument("--scales", type=float, nargs="+", default=[.25, .5, 1.]) sam.add_argument("--seed", type=int, default=20261110) args = parser.parse_args() if args.output.exists(): parser.error("Output already exists; choose a new path.") args.output.parent.mkdir(parents=True, exist_ok=True) torch.set_num_threads(4) if args.command == "encode": encode(args.data, args.output) else: sample(args.pool, args.queries, args.output, args.neighbors, args.directions, np.asarray(args.scales, dtype=np.float32), args.seed) if __name__ == "__main__": main()