Download scripts/data/calc_seed_textop_stats.py from Tinghong-Ye22/code-Psi0-yth-v23: direct link, hf CLI and curl.
- Browser
- Download file 16.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v23/resolve/main/scripts/data/calc_seed_textop_stats.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v23/scripts/data/calc_seed_textop_stats.py
-
curl -L -o calc_seed_textop_stats.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v23/resolve/main/scripts/data/calc_seed_textop_stats.py
16.1 kB
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| from tqdm import tqdm | |
| from psi.config.transform import SeedPosttrainRepackTransform | |
| from psi.data.seed.motion_repr import build_seed_textop_action | |
| from psi.data.seed.seed_motion_dataset import SeedMotionDataset | |
| def stats_dict(array: np.ndarray) -> dict[str, list[float]]: | |
| return { | |
| "mean": np.mean(array, axis=0).tolist(), | |
| "std": np.std(array, axis=0).tolist(), | |
| "min": np.min(array, axis=0).tolist(), | |
| "max": np.max(array, axis=0).tolist(), | |
| "q01": np.quantile(array, 0.01, axis=0).tolist(), | |
| "q99": np.quantile(array, 0.99, axis=0).tolist(), | |
| } | |
| def stats_from_sample( | |
| *, | |
| count: int, | |
| total: np.ndarray, | |
| square_total: np.ndarray, | |
| min_value: np.ndarray, | |
| max_value: np.ndarray, | |
| quantile_sample: np.ndarray, | |
| ) -> dict[str, list[float]]: | |
| mean = total / float(count) | |
| variance = square_total / float(count) - mean * mean | |
| variance = np.maximum(variance, 0.0) | |
| return { | |
| "mean": mean.tolist(), | |
| "std": np.sqrt(variance).tolist(), | |
| "min": min_value.tolist(), | |
| "max": max_value.tolist(), | |
| "q01": np.quantile(quantile_sample, 0.01, axis=0).tolist(), | |
| "q99": np.quantile(quantile_sample, 0.99, axis=0).tolist(), | |
| } | |
| def take_sample_rows(array: np.ndarray, sample_indices: np.ndarray, start: int, end: int) -> np.ndarray: | |
| left = np.searchsorted(sample_indices, start, side="left") | |
| right = np.searchsorted(sample_indices, end, side="left") | |
| if right <= left: | |
| return np.zeros((0, array.shape[1]), dtype=np.float32) | |
| local_indices = sample_indices[left:right] - start | |
| return array[local_indices].astype(np.float32) | |
| def map_seed_motion_dir(root_dir: Path, move_g1_path: str) -> Path: | |
| rel = str(move_g1_path).strip().replace("\\", "/") | |
| rel = rel.replace("g1/csv/", "g1/csv_sonic/") | |
| rel = rel.replace("/csv/", "/csv_sonic/") | |
| if rel.endswith(".csv"): | |
| rel = rel[:-4] | |
| if rel.startswith("dataset/seed/"): | |
| rel = rel[len("dataset/seed/") :] | |
| if rel.startswith("seed/"): | |
| rel = rel[len("seed/") :] | |
| p = Path(rel) | |
| if p.is_absolute(): | |
| return p | |
| return root_dir / p | |
| def load_seed_motion_array(motion_dir: Path) -> np.ndarray: | |
| joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32) | |
| body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32) | |
| body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32) | |
| length = min(joint.shape[0], body_pos.shape[0], body_quat.shape[0]) | |
| return np.concatenate([joint[:length], body_pos[:length], body_quat[:length]], axis=1).astype(np.float32) | |
| def count_csv_data_rows(path: Path) -> int: | |
| with path.open("r", encoding="utf-8", errors="ignore") as f: | |
| return max(0, sum(1 for _ in f) - 1) | |
| def calculate_frame_level_stats_from_metadata( | |
| *, | |
| root_dir: str, | |
| metadata_csv: str, | |
| action_chunk_size: int, | |
| num_past_frames: int, | |
| quantile_sample_size: int, | |
| seed: int, | |
| ) -> dict: | |
| root = Path(root_dir) | |
| metadata_path = Path(metadata_csv) | |
| if not metadata_path.is_absolute(): | |
| metadata_path = root / metadata_path | |
| metadata = pd.read_csv(metadata_path, usecols=["move_g1_path"]) | |
| motion_dirs = [map_seed_motion_dir(root, p) for p in metadata["move_g1_path"].tolist()] | |
| lengths = [] | |
| valid_motion_dirs = [] | |
| for motion_dir in tqdm(motion_dirs, desc="Counting SEED frame rows"): | |
| joint_path = motion_dir / "joint_pos.csv" | |
| if not joint_path.exists(): | |
| continue | |
| n = count_csv_data_rows(joint_path) | |
| if n <= 0: | |
| continue | |
| lengths.append(n) | |
| valid_motion_dirs.append(motion_dir) | |
| total_action_rows = int(sum(max(0, n - 1) for n in lengths)) | |
| total_state_rows = int(sum(lengths)) | |
| rng = np.random.RandomState(seed) | |
| action_sample_size = min(int(quantile_sample_size), total_action_rows) | |
| state_sample_size = min(int(quantile_sample_size), total_state_rows) | |
| action_sample_indices = np.sort(rng.choice(total_action_rows, size=action_sample_size, replace=False)) | |
| state_sample_indices = np.sort(rng.choice(total_state_rows, size=state_sample_size, replace=False)) | |
| action_sum = np.zeros((70,), dtype=np.float64) | |
| action_sumsq = np.zeros((70,), dtype=np.float64) | |
| action_min = np.full((70,), np.inf, dtype=np.float64) | |
| action_max = np.full((70,), -np.inf, dtype=np.float64) | |
| state_sum = np.zeros((36,), dtype=np.float64) | |
| state_sumsq = np.zeros((36,), dtype=np.float64) | |
| state_min = np.full((36,), np.inf, dtype=np.float64) | |
| state_max = np.full((36,), -np.inf, dtype=np.float64) | |
| action_samples = [] | |
| state_samples = [] | |
| action_offset = 0 | |
| state_offset = 0 | |
| for motion_dir in tqdm(valid_motion_dirs, desc="Collecting frame-level SEED TextOp stats"): | |
| motion = load_seed_motion_array(motion_dir) | |
| if motion.shape[0] > 1: | |
| textop_action = build_seed_textop_action(motion[1:], motion[:1]) | |
| action_sum += textop_action.sum(axis=0, dtype=np.float64) | |
| action_sumsq += np.square(textop_action.astype(np.float64)).sum(axis=0) | |
| action_min = np.minimum(action_min, textop_action.min(axis=0)) | |
| action_max = np.maximum(action_max, textop_action.max(axis=0)) | |
| action_end = action_offset + textop_action.shape[0] | |
| sampled = take_sample_rows(textop_action, action_sample_indices, action_offset, action_end) | |
| if sampled.shape[0] > 0: | |
| action_samples.append(sampled) | |
| action_offset = action_end | |
| state_sum += motion.sum(axis=0, dtype=np.float64) | |
| state_sumsq += np.square(motion.astype(np.float64)).sum(axis=0) | |
| state_min = np.minimum(state_min, motion.min(axis=0)) | |
| state_max = np.maximum(state_max, motion.max(axis=0)) | |
| state_end = state_offset + motion.shape[0] | |
| sampled = take_sample_rows(motion, state_sample_indices, state_offset, state_end) | |
| if sampled.shape[0] > 0: | |
| state_samples.append(sampled) | |
| state_offset = state_end | |
| action_sample = np.concatenate(action_samples, axis=0).astype(np.float32) | |
| state_sample = np.concatenate(state_samples, axis=0).astype(np.float32) | |
| return { | |
| "action": stats_from_sample( | |
| count=total_action_rows, | |
| total=action_sum, | |
| square_total=action_sumsq, | |
| min_value=action_min, | |
| max_value=action_max, | |
| quantile_sample=action_sample, | |
| ), | |
| "states": stats_from_sample( | |
| count=total_state_rows, | |
| total=state_sum, | |
| square_total=state_sumsq, | |
| min_value=state_min, | |
| max_value=state_max, | |
| quantile_sample=state_sample, | |
| ), | |
| "meta": { | |
| "representation": "joint_abs29_joint_delta29_root_delta_local3_root_rot6d_delta6_root_height1_foot_contact2", | |
| "stat_mode": "frame_level_direct_metadata", | |
| "action_dim": 70, | |
| "state_dim": 36, | |
| "action_chunk_size": int(action_chunk_size), | |
| "num_past_frames": int(num_past_frames), | |
| "num_metadata_rows": int(len(metadata)), | |
| "num_motions": int(len(valid_motion_dirs)), | |
| "num_action_rows": int(total_action_rows), | |
| "num_state_rows": int(total_state_rows), | |
| "action_quantile_sample_rows": int(action_sample.shape[0]), | |
| "state_quantile_sample_rows": int(state_sample.shape[0]), | |
| "quantile_sample_seed": int(seed), | |
| }, | |
| } | |
| def calculate_frame_level_stats( | |
| dataset: SeedMotionDataset, | |
| *, | |
| action_chunk_size: int, | |
| num_past_frames: int, | |
| quantile_sample_size: int, | |
| seed: int, | |
| ) -> dict: | |
| total_action_rows = int(sum(max(0, int(r["n_frames"]) - 1) for r in dataset.motion_records)) | |
| total_state_rows = int(sum(int(r["n_frames"]) for r in dataset.motion_records)) | |
| rng = np.random.RandomState(seed) | |
| action_sample_size = min(int(quantile_sample_size), total_action_rows) | |
| state_sample_size = min(int(quantile_sample_size), total_state_rows) | |
| action_sample_indices = np.sort(rng.choice(total_action_rows, size=action_sample_size, replace=False)) | |
| state_sample_indices = np.sort(rng.choice(total_state_rows, size=state_sample_size, replace=False)) | |
| action_sum = np.zeros((70,), dtype=np.float64) | |
| action_sumsq = np.zeros((70,), dtype=np.float64) | |
| action_min = np.full((70,), np.inf, dtype=np.float64) | |
| action_max = np.full((70,), -np.inf, dtype=np.float64) | |
| state_sum = np.zeros((36,), dtype=np.float64) | |
| state_sumsq = np.zeros((36,), dtype=np.float64) | |
| state_min = np.full((36,), np.inf, dtype=np.float64) | |
| state_max = np.full((36,), -np.inf, dtype=np.float64) | |
| action_samples = [] | |
| state_samples = [] | |
| action_offset = 0 | |
| state_offset = 0 | |
| for record in tqdm(dataset.motion_records, desc="Collecting frame-level SEED TextOp stats"): | |
| motion = dataset._load_motion_array(record["motion_dir"]) | |
| motion = np.asarray(motion[:, :36], dtype=np.float32) | |
| if motion.shape[0] > 1: | |
| textop_action = build_seed_textop_action(motion[1:], motion[:1]) | |
| action_sum += textop_action.sum(axis=0, dtype=np.float64) | |
| action_sumsq += np.square(textop_action.astype(np.float64)).sum(axis=0) | |
| action_min = np.minimum(action_min, textop_action.min(axis=0)) | |
| action_max = np.maximum(action_max, textop_action.max(axis=0)) | |
| action_end = action_offset + textop_action.shape[0] | |
| sampled = take_sample_rows(textop_action, action_sample_indices, action_offset, action_end) | |
| if sampled.shape[0] > 0: | |
| action_samples.append(sampled) | |
| action_offset = action_end | |
| state_sum += motion.sum(axis=0, dtype=np.float64) | |
| state_sumsq += np.square(motion.astype(np.float64)).sum(axis=0) | |
| state_min = np.minimum(state_min, motion.min(axis=0)) | |
| state_max = np.maximum(state_max, motion.max(axis=0)) | |
| state_end = state_offset + motion.shape[0] | |
| sampled = take_sample_rows(motion, state_sample_indices, state_offset, state_end) | |
| if sampled.shape[0] > 0: | |
| state_samples.append(sampled) | |
| state_offset = state_end | |
| action_sample = np.concatenate(action_samples, axis=0).astype(np.float32) | |
| state_sample = np.concatenate(state_samples, axis=0).astype(np.float32) | |
| return { | |
| "action": stats_from_sample( | |
| count=total_action_rows, | |
| total=action_sum, | |
| square_total=action_sumsq, | |
| min_value=action_min, | |
| max_value=action_max, | |
| quantile_sample=action_sample, | |
| ), | |
| "states": stats_from_sample( | |
| count=total_state_rows, | |
| total=state_sum, | |
| square_total=state_sumsq, | |
| min_value=state_min, | |
| max_value=state_max, | |
| quantile_sample=state_sample, | |
| ), | |
| "meta": { | |
| "representation": "joint_abs29_joint_delta29_root_delta_local3_root_rot6d_delta6_root_height1_foot_contact2", | |
| "stat_mode": "frame_level", | |
| "action_dim": 70, | |
| "state_dim": 36, | |
| "action_chunk_size": int(action_chunk_size), | |
| "num_past_frames": int(num_past_frames), | |
| "num_motions": int(len(dataset.motion_records)), | |
| "num_action_rows": int(total_action_rows), | |
| "num_state_rows": int(total_state_rows), | |
| "action_quantile_sample_rows": int(action_sample.shape[0]), | |
| "state_quantile_sample_rows": int(state_sample.shape[0]), | |
| "quantile_sample_seed": int(seed), | |
| }, | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="Calculate SEED TextOp-style action statistics.") | |
| parser.add_argument("--root-dir", type=str, required=True) | |
| parser.add_argument("--metadata-csv", type=str, default="metadata/seed_metadata_v004_100.csv") | |
| parser.add_argument("--output-path", type=str, required=True) | |
| parser.add_argument("--action-chunk-size", type=int, default=30) | |
| parser.add_argument("--num-past-frames", type=int, default=3) | |
| parser.add_argument("--frame-level", action="store_true", default=False) | |
| parser.add_argument("--direct-metadata", action="store_true", default=False) | |
| parser.add_argument("--quantile-sample-size", type=int, default=1000000) | |
| parser.add_argument("--seed", type=int, default=292285) | |
| args = parser.parse_args() | |
| if args.frame_level and args.direct_metadata: | |
| stats = calculate_frame_level_stats_from_metadata( | |
| root_dir=args.root_dir, | |
| metadata_csv=args.metadata_csv, | |
| action_chunk_size=args.action_chunk_size, | |
| num_past_frames=args.num_past_frames, | |
| quantile_sample_size=args.quantile_sample_size, | |
| seed=args.seed, | |
| ) | |
| output_path = Path(args.output_path) | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| output_path.write_text(json.dumps(stats, indent=2), encoding="utf-8") | |
| print(f"Wrote {output_path}") | |
| print(json.dumps(stats["meta"], indent=2)) | |
| return | |
| dataset = SeedMotionDataset( | |
| root_dir=args.root_dir, | |
| metadata_csv=args.metadata_csv, | |
| split="all", | |
| action_chunk_size=args.action_chunk_size, | |
| num_past_frames=args.num_past_frames, | |
| ) | |
| if args.frame_level: | |
| stats = calculate_frame_level_stats( | |
| dataset, | |
| action_chunk_size=args.action_chunk_size, | |
| num_past_frames=args.num_past_frames, | |
| quantile_sample_size=args.quantile_sample_size, | |
| seed=args.seed, | |
| ) | |
| output_path = Path(args.output_path) | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| output_path.write_text(json.dumps(stats, indent=2), encoding="utf-8") | |
| print(f"Wrote {output_path}") | |
| print(json.dumps(stats["meta"], indent=2)) | |
| return | |
| repack = SeedPosttrainRepackTransform( | |
| textop_actions=True, | |
| action_chunk_size=args.action_chunk_size, | |
| num_past_frames=args.num_past_frames, | |
| pad_action_dim=70, | |
| pad_state_dim=36, | |
| ) | |
| actions = [] | |
| states = [] | |
| for i in tqdm(range(len(dataset)), desc="Collecting SEED TextOp stats"): | |
| item = repack(dataset[i]) | |
| # <modify_textop_repr> 只统计有效action帧,避免chunk尾部padding重复最后一帧影响q01/q99。 | |
| action_mask = np.asarray(item["actions_mask"], dtype=np.float32) | |
| valid_rows = np.any(action_mask > 0.5, axis=1) | |
| actions.append(item["actions"][valid_rows].reshape(-1, item["actions"].shape[-1])) | |
| states.append(item["states"].reshape(-1, item["states"].shape[-1])) | |
| action_array = np.concatenate(actions, axis=0).astype(np.float32) | |
| state_array = np.concatenate(states, axis=0).astype(np.float32) | |
| stats = { | |
| "action": stats_dict(action_array), | |
| "states": stats_dict(state_array), | |
| "meta": { | |
| "representation": "joint_abs29_joint_delta29_root_delta_local3_root_rot6d_delta6_root_height1_foot_contact2", | |
| "stat_mode": "chunk_window", | |
| "action_dim": 70, | |
| "state_dim": 36, | |
| "action_chunk_size": int(args.action_chunk_size), | |
| "num_past_frames": int(args.num_past_frames), | |
| "num_samples": int(len(dataset)), | |
| "num_action_rows": int(action_array.shape[0]), | |
| "num_state_rows": int(state_array.shape[0]), | |
| }, | |
| } | |
| output_path = Path(args.output_path) | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| output_path.write_text(json.dumps(stats, indent=2), encoding="utf-8") | |
| print(f"Wrote {output_path}") | |
| print(f"action shape: {action_array.shape}, states shape: {state_array.shape}") | |
| if __name__ == "__main__": | |
| main() | |