dexhand-train-kit / code /runner /lazy_masks.py
Noctis12138's picture
Add files using upload-large-folder tool
6a08a28 verified
Raw History Blame Contribute Delete
4.34 kB
"""Decode the target masks only for the sampled frame (speed fix, no change to openpi / LeRobot / mask_overlay.py).
Why: openpi asks LeRobot for a 50-step action chunk; LeRobot then reads the 50 future rows *with all columns*, so every
sample decoded 3 cameras x 50 mask PNGs (~0.9 s of the ~1.1 s per sample). The overlay only needs the current frame.
How: the observation.masks.* columns are removed from the LeRobot table (so chunk queries never touch them) and kept
as a separate, undecoded table; __getitem__ decodes the current frame's 3 PNGs and adds them back under the same keys,
so train.mask_overlay.MaskOverlay sees exactly what it saw before (HWC uint8, channel 0 > 127 = target).
Installed by patching openpi.training.data_loader.create_torch_dataset (looked up at call time by openpi's code).
"""
import cv2
import numpy as np
from datasets import Image as HFImage
from lerobot.datasets import lerobot_dataset
from lerobot.datasets.utils import hf_transform_to_torch
from openpi import transforms as _transforms
from openpi.training import data_loader as _dl
MASK_PREFIX = "observation.masks."
cv2.setNumThreads(1) # imported in every data worker; avoid per-worker thread pools
class LazyMaskDataset:
def __init__(self, ds: lerobot_dataset.LeRobotDataset):
cols = [c for c in ds.hf_dataset.column_names if c.startswith(MASK_PREFIX)]
masks = ds.hf_dataset.select_columns(cols)
masks.reset_format()
for c in cols:
masks = masks.cast_column(c, HFImage(decode=False))
rest = ds.hf_dataset.remove_columns(cols)
rest.set_transform(hf_transform_to_torch)
ds.hf_dataset = rest
self._ds, self._masks, self._cols = ds, masks, cols
def __len__(self):
return len(self._ds)
def __getitem__(self, idx):
item = self._ds[idx]
row = self._masks[int(idx)]
for c in self._cols:
m = cv2.imdecode(np.frombuffer(row[c]["bytes"], np.uint8), cv2.IMREAD_UNCHANGED)
if m.ndim == 2:
m = np.repeat(m[..., None], 3, axis=-1)
else: # OpenCV gives BGR(A); PIL (the original path) gives RGB -> channel 0 must be R
m = cv2.cvtColor(m[..., :3], cv2.COLOR_BGR2RGB)
item[c] = m
return item
def create_torch_dataset(data_config, action_horizon, model_config):
"""openpi's create_torch_dataset (same arguments and steps) with the lazy-mask wrapper around LeRobotDataset.
repo_id "a+b" concatenates several LeRobot datasets (each with its own task table, so prompts stay correct)
instead of copying them into one merged dataset; frames are then sampled uniformly over all of them."""
repo_id = data_config.repo_id
if repo_id is None:
raise ValueError("Repo ID is not set. Cannot create dataset.")
if repo_id == "fake":
return _dl.FakeDataset(model_config, num_samples=1024)
if "+" in repo_id:
import dataclasses
import torch.utils.data
parts = [create_torch_dataset(dataclasses.replace(data_config, repo_id=r), action_horizon, model_config)
for r in repo_id.split("+")]
print("concatenated:", {r: len(p) for r, p in zip(repo_id.split("+"), parts)}, flush=True)
return torch.utils.data.ConcatDataset(parts)
meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
ds = lerobot_dataset.LeRobotDataset(
repo_id,
delta_timestamps={k: [t / meta.fps for t in range(action_horizon)] for k in data_config.action_sequence_keys},
video_backend=data_config.video_backend,
)
dataset = LazyMaskDataset(ds) if any(c.startswith(MASK_PREFIX) for c in ds.hf_dataset.column_names) else ds
if data_config.prompt_from_task:
dataset = _dl.TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(meta.tasks)])
return dataset
def install():
_dl.create_torch_dataset = create_torch_dataset
class RemoveStrings(_transforms.DataTransformFn):
"""Copy of openpi scripts/compute_norm_stats.py RemoveStrings, in an importable module so that DataLoader worker
processes can unpickle it (the script itself is loaded by path and is not importable in workers)."""
def __call__(self, x: dict) -> dict:
return {k: v for k, v in x.items() if not np.issubdtype(np.asarray(v).dtype, np.str_)}