"""openpi (XPolicyLab's vendored copy, not edited) configuration for the dual Marvin + Wuji robot, sub-task data. Data: LeRobot v3.0 shards of gen/to_lerobot.py (HF Noctis12138/dexhand-cover-data, lerobot/cover_v2_7seg_*): observation.state / action 54-D (left arm 7, left hand 20, right arm 7, right hand 20; action = commanded joint targets), observation.images.cam_high / cam_left_wrist / cam_right_wrist (480x640), task = sub-task instruction. Model: pi0.5 from pi05_base with the action dim kept at 32 (user decision: every pretrained weight is reused, no widened head). The 54-D action chunk is interleaved left / right ("Ego-Pi style"): (T, 54) -> left [arm 7 + hand 20] and right [arm 7 + hand 20], each zero-padded 27 -> 32 -> L0, R0, L1, R1, ... = (2T, 32); the model's action horizon is 2T, the data loader reads T future actions. The 54-D state is not interleaved: pi0.5 only uses it as discretised text in the prompt (TokenizePrompt runs before the interleave), and the policy hands the transformed state to the output transforms (AbsoluteActions needs all 54). So the model's state spec is 54-D (Pi05Interleaved.inputs_spec); the network never reads obs.state in pi0.5. Everything else follows XPolicyLab's own pi05_wuji_marvin_54d config (same robot): WujiInputs / WujiOutputs camera mapping, arms as deltas and hands absolute (make_bool_mask(7, -20, 7, -20)), quantile normalisation. Difference: action_dim 32 + interleave instead of action_dim 54, the full (not partial) pi05_base weights, cam_high as base camera. """ import dataclasses import glob import os import pathlib import numpy as np SPLIT, PAD = 27, 32 # per-side dims (arm 7 + hand 20), padded per-side width = model action dim ASSET_ID = "dual_marvin_wuji" # ------------------------------------------------------------------ transforms def _transforms(): from openpi import transforms as _t return _t @dataclasses.dataclass(frozen=True) class InterleaveLR: """actions (..., T, 54) -> (..., 2T, 32): L0, R0, L1, R1, ... (each side zero-padded 27 -> 32).""" def __call__(self, data): if "actions" not in data: return data a = np.asarray(data["actions"]) assert a.shape[-1] == 2 * SPLIT, a.shape pad = [(0, 0)] * (a.ndim - 1) + [(0, PAD - SPLIT)] lr = np.stack([np.pad(a[..., :SPLIT], pad), np.pad(a[..., SPLIT:], pad)], axis=-2) # (..., T, 2, 32) data["actions"] = lr.reshape(*a.shape[:-2], 2 * a.shape[-2], PAD) return data @dataclasses.dataclass(frozen=True) class DeinterleaveLR: """actions (..., 2T, 32) -> (..., T, 54); inverse of InterleaveLR (padding dropped).""" def __call__(self, data): a = np.asarray(data["actions"]) assert a.shape[-2] % 2 == 0 and a.shape[-1] >= SPLIT, a.shape lr = a.reshape(*a.shape[:-2], a.shape[-2] // 2, 2, a.shape[-1]) data["actions"] = np.concatenate([lr[..., 0, :SPLIT], lr[..., 1, :SPLIT]], axis=-1) return data class RemoveStrings: """Drop string fields (prompt) before batching numeric stats (module level: picklable for spawn workers).""" def __call__(self, x): return {k: v for k, v in x.items() if not np.issubdtype(np.asarray(v).dtype, np.str_)} # ------------------------------------------------------------------ model / data configs def _classes(): """Built lazily: openpi is only importable inside its venv.""" import jax import jax.numpy as jnp from openpi.models import pi0_config from openpi.policies import wuji_policy from openpi.training import config as _config from openpi.shared import array_typing as at _t = _transforms() @dataclasses.dataclass(frozen=True) class Pi05Interleaved(pi0_config.Pi0Config): """Pi0Config whose observation state is state_dim wide (54) while actions stay action_dim (32) wide.""" state_dim: int = 54 def inputs_spec(self, *, batch_size: int = 1): obs, act = super().inputs_spec(batch_size=batch_size) with at.disable_typechecking(): obs = dataclasses.replace(obs, state=jax.ShapeDtypeStruct([batch_size, self.state_dim], jnp.float32)) return obs, act @dataclasses.dataclass(frozen=True) class DexhandDataConfig(_config.DataConfigFactory): base_image_key: str = "observation.images.cam_high" action_sequence_keys: tuple = ("action",) def create(self, assets_dirs: pathlib.Path, model_config) -> _config.DataConfig: repack = _t.Group(inputs=[_t.RepackTransform({ "observation/image": self.base_image_key, "observation/left_wrist_image": "observation.images.cam_left_wrist", "observation/right_wrist_image": "observation.images.cam_right_wrist", "observation/state": "observation.state", "actions": "action", "prompt": "prompt", })]) mask = _t.make_bool_mask(7, -20, 7, -20) # arms delta, hands absolute (PD past contact) data_tf = _t.Group( inputs=[wuji_policy.WujiInputs(model_type=model_config.model_type), _t.DeltaActions(mask)], outputs=[_t.AbsoluteActions(mask), wuji_policy.WujiOutputs(action_dim=2 * SPLIT)], ) base = _config.ModelTransformFactory()(model_config) ins = list(base.inputs) k = next(i for i, f in enumerate(ins) if type(f).__name__ == "TokenizePrompt") + 1 model_tf = _t.Group(inputs=[*ins[:k], InterleaveLR(), *ins[k:]], # tokenise the 54-D state first outputs=[DeinterleaveLR(), *base.outputs]) return dataclasses.replace( self.create_base_config(assets_dirs, model_config), repack_transforms=repack, data_transforms=data_tf, model_transforms=model_tf, action_sequence_keys=self.action_sequence_keys, ) return Pi05Interleaved, DexhandDataConfig def get_config(repo_id, *, T=50, max_token_len=288, assets_base_dir=None, **overrides): """TrainConfig 'dexhand_ilv'. repo_id: a LeRobot repo id under $HF_LEROBOT_HOME, 'a+b+...', or 'glob:' (e.g. 'glob:dexhand/cover_v2_7seg_*'), expanded by the dataset patch below.""" from openpi.training import config as _config from openpi.training import weight_loaders Pi05Interleaved, DexhandDataConfig = _classes() model = Pi05Interleaved(pi05=True, action_dim=PAD, action_horizon=2 * T, max_token_len=max_token_len) data = DexhandDataConfig( repo_id=repo_id, assets=_config.AssetsConfig(assets_dir=assets_base_dir, asset_id=ASSET_ID) if assets_base_dir else _config.AssetsConfig(asset_id=ASSET_ID), base_config=_config.DataConfig(prompt_from_task=True, video_backend="pyav"), ) kw = dict(seed=0, batch_size=256) kw.update(overrides) return _config.TrainConfig( name="dexhand_ilv", model=model, data=data, weight_loader=weight_loaders.CheckpointWeightLoader("gs://openpi-assets/checkpoints/pi05_base/params"), **kw) # ------------------------------------------------------------------ data loading patch def expand_repo_ids(repo_id): home = pathlib.Path(os.environ["HF_LEROBOT_HOME"]) out = [] for part in repo_id.split("+"): if part.startswith("glob:"): hits = sorted(str(pathlib.Path(p).relative_to(home)) for p in glob.glob(str(home / part[5:]))) assert hits, f"no dataset matches {part}" out += hits else: out.append(part) return out def install_dataset_patch(): """openpi's create_torch_dataset, except: (1) several repos ('a+b', 'glob:...') are concatenated, each keeping its own task table; (2) the interleaved model's action horizon is 2T, but each sample needs T future actions.""" import torch.utils.data from lerobot.datasets import lerobot_dataset from openpi import transforms as _t from openpi.training import data_loader as _dl def create_torch_dataset(data_config, action_horizon, model_config): repo_id = data_config.repo_id if repo_id is None: raise ValueError("Repo ID is not set. Cannot create dataset.") if repo_id == "fake": return _dl.FakeDataset(model_config, num_samples=1024) T = action_horizon // 2 if getattr(model_config, "state_dim", None) else action_horizon parts = [] for r in expand_repo_ids(repo_id): meta = lerobot_dataset.LeRobotDatasetMetadata(r) ds = lerobot_dataset.LeRobotDataset( r, delta_timestamps={k: [t / meta.fps for t in range(T)] for k in data_config.action_sequence_keys}, video_backend=data_config.video_backend) if data_config.prompt_from_task: ds = _dl.TransformedDataset(ds, [_t.PromptFromLeRobotTask(meta.tasks)]) parts.append(ds) print(f"datasets: {len(parts)}, frames {sum(len(p) for p in parts)}, future actions per sample {T}", flush=True) if BALANCE_TASKS: return TaskBalancedDataset(expand_repo_ids(repo_id), parts) return parts[0] if len(parts) == 1 else torch.utils.data.ConcatDataset(parts) _dl.create_torch_dataset = create_torch_dataset BALANCE_TASKS = False # set by run.py --balance-tasks: sample tasks uniformly, frames uniformly within a task TASK_OF = {"pickup_v3": "pickup_v2", "pickup_m3b": "pickup_m3a"} # batches that are the same task (general_pickup, # two object sets) def task_key(repo): """'dexhand/pens_v2_seg_p0_s003' -> 'pens_v2'; pickup_v3 counts as pickup_v2 (one task).""" name = repo.split("/")[-1] for tag in ("_seg_", "_fseg_", "_7seg_"): if tag in name: name = name.split(tag)[0] break return TASK_OF.get(name, name) class TaskBalancedDataset: """Mid-training mixture: index i -> task i % n_tasks, frame (i // n_tasks) spread uniformly over that task's frames. With the loader's shuffle=True every task gets the same share of each batch whatever its size (cover_m2 700 and dolls_v2 577 segments vs align_v2 200: a plain concatenation gave them 37 % of the samples, 2026-10-24). Length = n_tasks x the largest task, so the biggest task is seen once per pass and the others are oversampled.""" def __init__(self, repos, parts): import torch.utils.data groups = {} for r, ds in zip(repos, parts): groups.setdefault(task_key(r), []).append(ds) self.tasks = sorted(groups) self.sets = [torch.utils.data.ConcatDataset(groups[t]) for t in self.tasks] self.sizes = [len(d) for d in self.sets] self.per_task = max(self.sizes) print("task-balanced mixture:", {t: n for t, n in zip(self.tasks, self.sizes)}, "-> per task", self.per_task, flush=True) def __len__(self): return self.per_task * len(self.tasks) def __getitem__(self, i): t, j = i % len(self.tasks), i // len(self.tasks) n = self.sizes[t] k = (j * n) // self.per_task if n < self.per_task else j # stride-map the slot onto this task's frames return self.sets[t][min(k, n - 1)]