"""Decode the target masks only for the sampled frame (speed fix, no change to openpi / LeRobot / mask_overlay.py). Why: openpi asks LeRobot for a 50-step action chunk; LeRobot then reads the 50 future rows *with all columns*, so every sample decoded 3 cameras x 50 mask PNGs (~0.9 s of the ~1.1 s per sample). The overlay only needs the current frame. How: the observation.masks.* columns are removed from the LeRobot table (so chunk queries never touch them) and kept as a separate, undecoded table; __getitem__ decodes the current frame's 3 PNGs and adds them back under the same keys, so train.mask_overlay.MaskOverlay sees exactly what it saw before (HWC uint8, channel 0 > 127 = target). Installed by patching openpi.training.data_loader.create_torch_dataset (looked up at call time by openpi's code). """ import cv2 import numpy as np from datasets import Image as HFImage from lerobot.datasets import lerobot_dataset from lerobot.datasets.utils import hf_transform_to_torch from openpi import transforms as _transforms from openpi.training import data_loader as _dl MASK_PREFIX = "observation.masks." cv2.setNumThreads(1) # imported in every data worker; avoid per-worker thread pools class LazyMaskDataset: def __init__(self, ds: lerobot_dataset.LeRobotDataset): cols = [c for c in ds.hf_dataset.column_names if c.startswith(MASK_PREFIX)] masks = ds.hf_dataset.select_columns(cols) masks.reset_format() for c in cols: masks = masks.cast_column(c, HFImage(decode=False)) rest = ds.hf_dataset.remove_columns(cols) rest.set_transform(hf_transform_to_torch) ds.hf_dataset = rest self._ds, self._masks, self._cols = ds, masks, cols def __len__(self): return len(self._ds) def __getitem__(self, idx): item = self._ds[idx] row = self._masks[int(idx)] for c in self._cols: m = cv2.imdecode(np.frombuffer(row[c]["bytes"], np.uint8), cv2.IMREAD_UNCHANGED) if m.ndim == 2: m = np.repeat(m[..., None], 3, axis=-1) else: # OpenCV gives BGR(A); PIL (the original path) gives RGB -> channel 0 must be R m = cv2.cvtColor(m[..., :3], cv2.COLOR_BGR2RGB) item[c] = m return item def create_torch_dataset(data_config, action_horizon, model_config): """openpi's create_torch_dataset (same arguments and steps) with the lazy-mask wrapper around LeRobotDataset. repo_id "a+b" concatenates several LeRobot datasets (each with its own task table, so prompts stay correct) instead of copying them into one merged dataset; frames are then sampled uniformly over all of them.""" repo_id = data_config.repo_id if repo_id is None: raise ValueError("Repo ID is not set. Cannot create dataset.") if repo_id == "fake": return _dl.FakeDataset(model_config, num_samples=1024) if "+" in repo_id: import dataclasses import torch.utils.data parts = [create_torch_dataset(dataclasses.replace(data_config, repo_id=r), action_horizon, model_config) for r in repo_id.split("+")] print("concatenated:", {r: len(p) for r, p in zip(repo_id.split("+"), parts)}, flush=True) return torch.utils.data.ConcatDataset(parts) meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id) ds = lerobot_dataset.LeRobotDataset( repo_id, delta_timestamps={k: [t / meta.fps for t in range(action_horizon)] for k in data_config.action_sequence_keys}, video_backend=data_config.video_backend, ) dataset = LazyMaskDataset(ds) if any(c.startswith(MASK_PREFIX) for c in ds.hf_dataset.column_names) else ds if data_config.prompt_from_task: dataset = _dl.TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(meta.tasks)]) return dataset def install(): _dl.create_torch_dataset = create_torch_dataset class RemoveStrings(_transforms.DataTransformFn): """Copy of openpi scripts/compute_norm_stats.py RemoveStrings, in an importable module so that DataLoader worker processes can unpickle it (the script itself is loaded by path and is not importable in workers).""" def __call__(self, x: dict) -> dict: return {k: v for k, v in x.items() if not np.issubdtype(np.asarray(v).dtype, np.str_)}