Download scripts/data/calc_modality_stats.py from Tinghong-Ye22/code-Psi0-yth-v15: direct link, hf CLI and curl.
- Browser
- Download file 2.91 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v15/resolve/main/scripts/data/calc_modality_stats.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v15/scripts/data/calc_modality_stats.py
-
curl -L -o calc_modality_stats.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v15/resolve/main/scripts/data/calc_modality_stats.py
2.91 kB
| # adapted from gr00t/data/dataset.py::calculate_dataset_statistics | |
| import numpy as np | |
| from tqdm import tqdm | |
| from pathlib import Path | |
| import pandas as pd | |
| def calculate_dataset_statistics(parquet_paths: list[Path]) -> dict: | |
| """Calculate the dataset statistics of all columns for a list of parquet files.""" | |
| # Dataset statistics | |
| all_low_dim_data_list = [] | |
| # Collect all the data | |
| for parquet_path in tqdm( | |
| sorted(list(parquet_paths)), | |
| desc="Collecting all parquet files...", | |
| ): | |
| # Load the parquet file | |
| parquet_data = pd.read_parquet(parquet_path) | |
| # parquet_data = parquet_data | |
| all_low_dim_data_list.append(parquet_data) | |
| all_low_dim_data = pd.concat(all_low_dim_data_list, axis=0) | |
| # Compute dataset statistics | |
| dataset_statistics = {} | |
| for le_modality in all_low_dim_data.columns: # type:ignore | |
| print(f"Computing statistics for {le_modality}...") | |
| # check if the data is the modality is actually a list of numbers | |
| # skip if it is a string | |
| if isinstance(all_low_dim_data[le_modality].iloc[0], str): | |
| print(f"Skipping {le_modality} because it is a string") | |
| continue | |
| np_data = np.vstack( | |
| [np.asarray(x, dtype=np.float32) for x in all_low_dim_data[le_modality]] | |
| ) | |
| dataset_statistics[le_modality] = { | |
| "mean": np.mean(np_data, axis=0).tolist(), | |
| "std": np.std(np_data, axis=0).tolist(), | |
| "min": np.min(np_data, axis=0).tolist(), | |
| "max": np.max(np_data, axis=0).tolist(), | |
| "q01": np.quantile(np_data, 0.01, axis=0).tolist(), | |
| "q99": np.quantile(np_data, 0.99, axis=0).tolist(), | |
| } | |
| return dataset_statistics | |
| if __name__ == "__main__": | |
| import argparse | |
| import json | |
| LE_ROBOT_DATA_FILENAME = "data/*/*.parquet" | |
| LE_ROBOT_STATS_FILENAME = "meta/stats.json" | |
| parser = argparse.ArgumentParser(description="Calculate dataset statistics for LeRobot parquet files.") | |
| # parser.add_argument("--dataset_path", type=str, required=True, help="Path to the root dataset directory.") | |
| parser.add_argument("--work-dir", type=str, default="_lerobot_build") | |
| parser.add_argument("--task", type=str, default=None, help="Process only this specific task (category/task_name)") | |
| parser.add_argument("--task-dir", type=str, default=None, help="Path to the lerobot data dir.") | |
| args = parser.parse_args() | |
| # dataset_path = Path(f"{args.work_dir}/{args.task}") | |
| dataset_path = Path(args.task_dir) if args.task_dir else Path(f"{args.work_dir}/{args.task}") | |
| stats_path = dataset_path / LE_ROBOT_STATS_FILENAME | |
| parquet_files = list((dataset_path).glob(LE_ROBOT_DATA_FILENAME)) | |
| le_statistics = calculate_dataset_statistics(parquet_files) | |
| with open(stats_path, "w") as f: | |
| json.dump(le_statistics, f, indent=4) |