File size: 7,588 Bytes
34988af | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """Encode training GT and sample distinct-recording latent neighborhoods.
Inputs are numeric NPZ arrays, described in DATA_FORMAT.md. Mesh generation is
external; the output contains full MANO rotations and actual wrist positions.
"""
import argparse
import hashlib
from pathlib import Path
import numpy as np
import torch
from hand_vae import aa_to_matrix, matrix_to_aa
from load_model import load_model
def encode_pose(model, pose, relative_rotation, relative_wrist):
local = aa_to_matrix(torch.as_tensor(pose, dtype=torch.float32)).reshape(-1, 30, 3, 3)
rotations = torch.cat([local, torch.as_tensor(relative_rotation, dtype=torch.float32)[:, None]], 1)
return model.encode(rotations, torch.as_tensor(relative_wrist, dtype=torch.float32))
@torch.inference_mode()
def encode(data_path, output):
model, _ = load_model()
with np.load(data_path, allow_pickle=False) as data:
rows = np.flatnonzero(data["split"] == 0)
if not len(rows):
raise ValueError("No split=0 training rows.")
payload = {name: data[name][rows] for name in ("sequence", "dataset")}
if "frame_ids" in data:
payload["frame_ids"] = data["frame_ids"][rows]
poses = {name: data[name][rows] for name in ("pose_aa", "relative_rotation", "relative_wrist_m")}
means, logvars = [], []
for start in range(0, len(rows), 2048):
ix = slice(start, start + 2048)
mu, logvar = encode_pose(model, poses["pose_aa"][ix], poses["relative_rotation"][ix], poses["relative_wrist_m"][ix])
means.append(mu.numpy())
logvars.append(logvar.numpy())
with data_path.open("rb") as stream:
source_hash = hashlib.file_digest(stream, "sha256").hexdigest()
np.savez_compressed(output, mu=np.concatenate(means), logvar=np.concatenate(logvars),
source_row=rows, split=np.zeros(len(rows), dtype=np.int8),
source_sha256=np.array(source_hash), **payload)
print(f"Encoded {len(rows)} training GT rows into {output}")
@torch.inference_mode()
def sample(pool_path, query_path, output, neighbor_count, directions, scales, seed):
model, _ = load_model()
rng = np.random.default_rng(seed)
with np.load(pool_path, allow_pickle=False) as data:
pool = {key: data[key] for key in data.files}
if not np.all(pool["split"] == 0):
raise ValueError("The candidate pool must contain only training GT.")
with np.load(query_path, allow_pickle=False) as queries:
# Queries identify difficult GT regions; held-out test queries are excluded.
if not np.isin(queries["split"], [0, 1]).all():
raise ValueError("Queries must use train/validation GT, never test rows.")
query_datasets = queries["dataset"].copy()
query_mu, _ = encode_pose(model, queries["pose_aa"], queries["relative_rotation"], queries["relative_wrist_m"])
query_mu = query_mu.numpy()
if not len(query_mu):
raise ValueError("The query file is empty.")
if not np.isfinite(pool["mu"]).all() or not np.isfinite(query_mu).all():
raise ValueError("Latents must be finite.")
if neighbor_count < 1 or directions < 1 or not len(scales) or np.any((scales < 0) | (scales > 1)) or not np.isfinite(scales).all():
raise ValueError("Use positive counts and finite interpolation scales in [0,1].")
anchors, neighbors, pairs, weights, latent_batches = [], [], [], [], []
for query, dataset in zip(query_mu, query_datasets):
ids = np.flatnonzero(pool["dataset"] == dataset)
if not len(ids):
raise ValueError(f"No training rows for dataset {dataset}.")
anchor = int(ids[np.argmin(np.square(pool["mu"][ids] - query).sum(1))])
order = np.argsort(np.square(pool["mu"][ids] - pool["mu"][anchor]).sum(1))
seen = {int(pool["sequence"][anchor])}
nearest = []
for row in ids[order]:
sequence = int(pool["sequence"][row])
if sequence not in seen:
nearest.append(int(row))
seen.add(sequence)
if len(nearest) == neighbor_count:
break
if len(nearest) != neighbor_count:
raise ValueError(f"Dataset {dataset} needs {neighbor_count + 1} distinct training recordings.")
nearest = np.asarray(nearest)
pair = rng.choice(neighbor_count, (directions, 2), replace=True)
weight = rng.uniform(.2, .8, directions).astype(np.float32)
endpoint = weight[:, None] * pool["mu"][nearest[pair[:, 0]]] + (1 - weight[:, None]) * pool["mu"][nearest[pair[:, 1]]]
z = (1 - scales[:, None, None]) * pool["mu"][anchor] + scales[:, None, None] * endpoint[None]
latent_batches.append(z.reshape(-1, model.latent_dim))
anchors.append(anchor)
neighbors.append(nearest)
pairs.append(pair)
weights.append(weight)
z = np.concatenate(latent_batches).astype(np.float32)
rot, pos = model.decode(torch.from_numpy(z))
full = torch.zeros(len(z), 2, 16, 3, 3)
full[:, 0, 0] = torch.eye(3)
full[:, 1, 0] = rot[:, 30]
full[:, :, 1:] = rot[:, :30].reshape(-1, 2, 15, 3, 3)
wrist = torch.zeros(len(z), 2, 3)
wrist[:, 1] = pos
anchors = np.asarray(anchors)
query_index = np.repeat(np.arange(len(anchors)), len(scales) * directions)
np.savez_compressed(
output, z=z, mano_rotation_matrices=full.numpy(), mano_pose_axis_angle=matrix_to_aa(full).numpy(),
wrist_positions_m=wrist.numpy(), mano_shape=np.zeros((len(z), 2, 10), np.float32),
query_index=query_index, train_pool_row=anchors[query_index],
train_source_row=pool["source_row"][anchors[query_index]],
train_sequence=pool["sequence"][anchors[query_index]],
train_dataset=pool["dataset"][anchors[query_index]],
temperature=np.tile(np.repeat(scales, directions), len(anchors)),
draw=np.tile(np.arange(directions), len(anchors) * len(scales)),
anchor_pool_rows=anchors, neighbor_pool_rows=np.asarray(neighbors),
endpoint_neighbor_indices=np.asarray(pairs), endpoint_weights=np.asarray(weights),
seed=np.array(seed), coordinate_system=np.array("left_wrist_local"),
)
print(f"Saved {len(z)} synthetic bimanual pose candidates to {output}")
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
enc = commands.add_parser("encode")
enc.add_argument("--data", type=Path, required=True)
enc.add_argument("--output", type=Path, required=True)
sam = commands.add_parser("sample")
sam.add_argument("--pool", type=Path, required=True)
sam.add_argument("--queries", type=Path, required=True)
sam.add_argument("--output", type=Path, required=True)
sam.add_argument("--neighbors", type=int, default=16)
sam.add_argument("--directions", type=int, default=8)
sam.add_argument("--scales", type=float, nargs="+", default=[.25, .5, 1.])
sam.add_argument("--seed", type=int, default=20261110)
args = parser.parse_args()
if args.output.exists():
parser.error("Output already exists; choose a new path.")
args.output.parent.mkdir(parents=True, exist_ok=True)
torch.set_num_threads(4)
if args.command == "encode":
encode(args.data, args.output)
else:
sample(args.pool, args.queries, args.output, args.neighbors, args.directions,
np.asarray(args.scales, dtype=np.float32), args.seed)
if __name__ == "__main__":
main()
|