File size: 2,906 Bytes
062e462 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | # adapted from gr00t/data/dataset.py::calculate_dataset_statistics
import numpy as np
from tqdm import tqdm
from pathlib import Path
import pandas as pd
def calculate_dataset_statistics(parquet_paths: list[Path]) -> dict:
"""Calculate the dataset statistics of all columns for a list of parquet files."""
# Dataset statistics
all_low_dim_data_list = []
# Collect all the data
for parquet_path in tqdm(
sorted(list(parquet_paths)),
desc="Collecting all parquet files...",
):
# Load the parquet file
parquet_data = pd.read_parquet(parquet_path)
# parquet_data = parquet_data
all_low_dim_data_list.append(parquet_data)
all_low_dim_data = pd.concat(all_low_dim_data_list, axis=0)
# Compute dataset statistics
dataset_statistics = {}
for le_modality in all_low_dim_data.columns: # type:ignore
print(f"Computing statistics for {le_modality}...")
# check if the data is the modality is actually a list of numbers
# skip if it is a string
if isinstance(all_low_dim_data[le_modality].iloc[0], str):
print(f"Skipping {le_modality} because it is a string")
continue
np_data = np.vstack(
[np.asarray(x, dtype=np.float32) for x in all_low_dim_data[le_modality]]
)
dataset_statistics[le_modality] = {
"mean": np.mean(np_data, axis=0).tolist(),
"std": np.std(np_data, axis=0).tolist(),
"min": np.min(np_data, axis=0).tolist(),
"max": np.max(np_data, axis=0).tolist(),
"q01": np.quantile(np_data, 0.01, axis=0).tolist(),
"q99": np.quantile(np_data, 0.99, axis=0).tolist(),
}
return dataset_statistics
if __name__ == "__main__":
import argparse
import json
LE_ROBOT_DATA_FILENAME = "data/*/*.parquet"
LE_ROBOT_STATS_FILENAME = "meta/stats.json"
parser = argparse.ArgumentParser(description="Calculate dataset statistics for LeRobot parquet files.")
# parser.add_argument("--dataset_path", type=str, required=True, help="Path to the root dataset directory.")
parser.add_argument("--work-dir", type=str, default="_lerobot_build")
parser.add_argument("--task", type=str, default=None, help="Process only this specific task (category/task_name)")
parser.add_argument("--task-dir", type=str, default=None, help="Path to the lerobot data dir.")
args = parser.parse_args()
# dataset_path = Path(f"{args.work_dir}/{args.task}")
dataset_path = Path(args.task_dir) if args.task_dir else Path(f"{args.work_dir}/{args.task}")
stats_path = dataset_path / LE_ROBOT_STATS_FILENAME
parquet_files = list((dataset_path).glob(LE_ROBOT_DATA_FILENAME))
le_statistics = calculate_dataset_statistics(parquet_files)
with open(stats_path, "w") as f:
json.dump(le_statistics, f, indent=4) |