Download code/runner/lazy_masks.py from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 4.34 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/lazy_masks.py
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/runner/lazy_masks.py
-
curl -L -o lazy_masks.py https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/lazy_masks.py
4.34 kB
| """Decode the target masks only for the sampled frame (speed fix, no change to openpi / LeRobot / mask_overlay.py). | |
| Why: openpi asks LeRobot for a 50-step action chunk; LeRobot then reads the 50 future rows *with all columns*, so every | |
| sample decoded 3 cameras x 50 mask PNGs (~0.9 s of the ~1.1 s per sample). The overlay only needs the current frame. | |
| How: the observation.masks.* columns are removed from the LeRobot table (so chunk queries never touch them) and kept | |
| as a separate, undecoded table; __getitem__ decodes the current frame's 3 PNGs and adds them back under the same keys, | |
| so train.mask_overlay.MaskOverlay sees exactly what it saw before (HWC uint8, channel 0 > 127 = target). | |
| Installed by patching openpi.training.data_loader.create_torch_dataset (looked up at call time by openpi's code). | |
| """ | |
| import cv2 | |
| import numpy as np | |
| from datasets import Image as HFImage | |
| from lerobot.datasets import lerobot_dataset | |
| from lerobot.datasets.utils import hf_transform_to_torch | |
| from openpi import transforms as _transforms | |
| from openpi.training import data_loader as _dl | |
| MASK_PREFIX = "observation.masks." | |
| cv2.setNumThreads(1) # imported in every data worker; avoid per-worker thread pools | |
| class LazyMaskDataset: | |
| def __init__(self, ds: lerobot_dataset.LeRobotDataset): | |
| cols = [c for c in ds.hf_dataset.column_names if c.startswith(MASK_PREFIX)] | |
| masks = ds.hf_dataset.select_columns(cols) | |
| masks.reset_format() | |
| for c in cols: | |
| masks = masks.cast_column(c, HFImage(decode=False)) | |
| rest = ds.hf_dataset.remove_columns(cols) | |
| rest.set_transform(hf_transform_to_torch) | |
| ds.hf_dataset = rest | |
| self._ds, self._masks, self._cols = ds, masks, cols | |
| def __len__(self): | |
| return len(self._ds) | |
| def __getitem__(self, idx): | |
| item = self._ds[idx] | |
| row = self._masks[int(idx)] | |
| for c in self._cols: | |
| m = cv2.imdecode(np.frombuffer(row[c]["bytes"], np.uint8), cv2.IMREAD_UNCHANGED) | |
| if m.ndim == 2: | |
| m = np.repeat(m[..., None], 3, axis=-1) | |
| else: # OpenCV gives BGR(A); PIL (the original path) gives RGB -> channel 0 must be R | |
| m = cv2.cvtColor(m[..., :3], cv2.COLOR_BGR2RGB) | |
| item[c] = m | |
| return item | |
| def create_torch_dataset(data_config, action_horizon, model_config): | |
| """openpi's create_torch_dataset (same arguments and steps) with the lazy-mask wrapper around LeRobotDataset. | |
| repo_id "a+b" concatenates several LeRobot datasets (each with its own task table, so prompts stay correct) | |
| instead of copying them into one merged dataset; frames are then sampled uniformly over all of them.""" | |
| repo_id = data_config.repo_id | |
| if repo_id is None: | |
| raise ValueError("Repo ID is not set. Cannot create dataset.") | |
| if repo_id == "fake": | |
| return _dl.FakeDataset(model_config, num_samples=1024) | |
| if "+" in repo_id: | |
| import dataclasses | |
| import torch.utils.data | |
| parts = [create_torch_dataset(dataclasses.replace(data_config, repo_id=r), action_horizon, model_config) | |
| for r in repo_id.split("+")] | |
| print("concatenated:", {r: len(p) for r, p in zip(repo_id.split("+"), parts)}, flush=True) | |
| return torch.utils.data.ConcatDataset(parts) | |
| meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id) | |
| ds = lerobot_dataset.LeRobotDataset( | |
| repo_id, | |
| delta_timestamps={k: [t / meta.fps for t in range(action_horizon)] for k in data_config.action_sequence_keys}, | |
| video_backend=data_config.video_backend, | |
| ) | |
| dataset = LazyMaskDataset(ds) if any(c.startswith(MASK_PREFIX) for c in ds.hf_dataset.column_names) else ds | |
| if data_config.prompt_from_task: | |
| dataset = _dl.TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(meta.tasks)]) | |
| return dataset | |
| def install(): | |
| _dl.create_torch_dataset = create_torch_dataset | |
| class RemoveStrings(_transforms.DataTransformFn): | |
| """Copy of openpi scripts/compute_norm_stats.py RemoveStrings, in an importable module so that DataLoader worker | |
| processes can unpickle it (the script itself is loaded by path and is not importable in workers).""" | |
| def __call__(self, x: dict) -> dict: | |
| return {k: v for k, v in x.items() if not np.issubdtype(np.asarray(v).dtype, np.str_)} | |