Download code/dexhand/train/dexhand_openpi.py from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/dexhand/train/dexhand_openpi.py
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/dexhand/train/dexhand_openpi.py
-
curl -L -o dexhand_openpi.py https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/dexhand/train/dexhand_openpi.py
11.4 kB
| """openpi (XPolicyLab's vendored copy, not edited) configuration for the dual Marvin + Wuji robot, sub-task data. | |
| Data: LeRobot v3.0 shards of gen/to_lerobot.py (HF Noctis12138/dexhand-cover-data, lerobot/cover_v2_7seg_*): | |
| observation.state / action 54-D (left arm 7, left hand 20, right arm 7, right hand 20; action = commanded joint | |
| targets), observation.images.cam_high / cam_left_wrist / cam_right_wrist (480x640), task = sub-task instruction. | |
| Model: pi0.5 from pi05_base with the action dim kept at 32 (user decision: every pretrained weight is reused, no | |
| widened head). The 54-D action chunk is interleaved left / right ("Ego-Pi style"): | |
| (T, 54) -> left [arm 7 + hand 20] and right [arm 7 + hand 20], each zero-padded 27 -> 32 -> L0, R0, L1, R1, ... | |
| = (2T, 32); the model's action horizon is 2T, the data loader reads T future actions. | |
| The 54-D state is not interleaved: pi0.5 only uses it as discretised text in the prompt (TokenizePrompt runs before | |
| the interleave), and the policy hands the transformed state to the output transforms (AbsoluteActions needs all 54). | |
| So the model's state spec is 54-D (Pi05Interleaved.inputs_spec); the network never reads obs.state in pi0.5. | |
| Everything else follows XPolicyLab's own pi05_wuji_marvin_54d config (same robot): WujiInputs / WujiOutputs camera | |
| mapping, arms as deltas and hands absolute (make_bool_mask(7, -20, 7, -20)), quantile normalisation. Difference: | |
| action_dim 32 + interleave instead of action_dim 54, the full (not partial) pi05_base weights, cam_high as base camera. | |
| """ | |
| import dataclasses | |
| import glob | |
| import os | |
| import pathlib | |
| import numpy as np | |
| SPLIT, PAD = 27, 32 # per-side dims (arm 7 + hand 20), padded per-side width = model action dim | |
| ASSET_ID = "dual_marvin_wuji" | |
| # ------------------------------------------------------------------ transforms | |
| def _transforms(): | |
| from openpi import transforms as _t | |
| return _t | |
| class InterleaveLR: | |
| """actions (..., T, 54) -> (..., 2T, 32): L0, R0, L1, R1, ... (each side zero-padded 27 -> 32).""" | |
| def __call__(self, data): | |
| if "actions" not in data: | |
| return data | |
| a = np.asarray(data["actions"]) | |
| assert a.shape[-1] == 2 * SPLIT, a.shape | |
| pad = [(0, 0)] * (a.ndim - 1) + [(0, PAD - SPLIT)] | |
| lr = np.stack([np.pad(a[..., :SPLIT], pad), np.pad(a[..., SPLIT:], pad)], axis=-2) # (..., T, 2, 32) | |
| data["actions"] = lr.reshape(*a.shape[:-2], 2 * a.shape[-2], PAD) | |
| return data | |
| class DeinterleaveLR: | |
| """actions (..., 2T, 32) -> (..., T, 54); inverse of InterleaveLR (padding dropped).""" | |
| def __call__(self, data): | |
| a = np.asarray(data["actions"]) | |
| assert a.shape[-2] % 2 == 0 and a.shape[-1] >= SPLIT, a.shape | |
| lr = a.reshape(*a.shape[:-2], a.shape[-2] // 2, 2, a.shape[-1]) | |
| data["actions"] = np.concatenate([lr[..., 0, :SPLIT], lr[..., 1, :SPLIT]], axis=-1) | |
| return data | |
| class RemoveStrings: | |
| """Drop string fields (prompt) before batching numeric stats (module level: picklable for spawn workers).""" | |
| def __call__(self, x): | |
| return {k: v for k, v in x.items() if not np.issubdtype(np.asarray(v).dtype, np.str_)} | |
| # ------------------------------------------------------------------ model / data configs | |
| def _classes(): | |
| """Built lazily: openpi is only importable inside its venv.""" | |
| import jax | |
| import jax.numpy as jnp | |
| from openpi.models import pi0_config | |
| from openpi.policies import wuji_policy | |
| from openpi.training import config as _config | |
| from openpi.shared import array_typing as at | |
| _t = _transforms() | |
| class Pi05Interleaved(pi0_config.Pi0Config): | |
| """Pi0Config whose observation state is state_dim wide (54) while actions stay action_dim (32) wide.""" | |
| state_dim: int = 54 | |
| def inputs_spec(self, *, batch_size: int = 1): | |
| obs, act = super().inputs_spec(batch_size=batch_size) | |
| with at.disable_typechecking(): | |
| obs = dataclasses.replace(obs, state=jax.ShapeDtypeStruct([batch_size, self.state_dim], jnp.float32)) | |
| return obs, act | |
| class DexhandDataConfig(_config.DataConfigFactory): | |
| base_image_key: str = "observation.images.cam_high" | |
| action_sequence_keys: tuple = ("action",) | |
| def create(self, assets_dirs: pathlib.Path, model_config) -> _config.DataConfig: | |
| repack = _t.Group(inputs=[_t.RepackTransform({ | |
| "observation/image": self.base_image_key, | |
| "observation/left_wrist_image": "observation.images.cam_left_wrist", | |
| "observation/right_wrist_image": "observation.images.cam_right_wrist", | |
| "observation/state": "observation.state", | |
| "actions": "action", | |
| "prompt": "prompt", | |
| })]) | |
| mask = _t.make_bool_mask(7, -20, 7, -20) # arms delta, hands absolute (PD past contact) | |
| data_tf = _t.Group( | |
| inputs=[wuji_policy.WujiInputs(model_type=model_config.model_type), _t.DeltaActions(mask)], | |
| outputs=[_t.AbsoluteActions(mask), wuji_policy.WujiOutputs(action_dim=2 * SPLIT)], | |
| ) | |
| base = _config.ModelTransformFactory()(model_config) | |
| ins = list(base.inputs) | |
| k = next(i for i, f in enumerate(ins) if type(f).__name__ == "TokenizePrompt") + 1 | |
| model_tf = _t.Group(inputs=[*ins[:k], InterleaveLR(), *ins[k:]], # tokenise the 54-D state first | |
| outputs=[DeinterleaveLR(), *base.outputs]) | |
| return dataclasses.replace( | |
| self.create_base_config(assets_dirs, model_config), | |
| repack_transforms=repack, data_transforms=data_tf, model_transforms=model_tf, | |
| action_sequence_keys=self.action_sequence_keys, | |
| ) | |
| return Pi05Interleaved, DexhandDataConfig | |
| def get_config(repo_id, *, T=50, max_token_len=288, assets_base_dir=None, **overrides): | |
| """TrainConfig 'dexhand_ilv'. repo_id: a LeRobot repo id under $HF_LEROBOT_HOME, 'a+b+...', or 'glob:<pattern>' | |
| (e.g. 'glob:dexhand/cover_v2_7seg_*'), expanded by the dataset patch below.""" | |
| from openpi.training import config as _config | |
| from openpi.training import weight_loaders | |
| Pi05Interleaved, DexhandDataConfig = _classes() | |
| model = Pi05Interleaved(pi05=True, action_dim=PAD, action_horizon=2 * T, max_token_len=max_token_len) | |
| data = DexhandDataConfig( | |
| repo_id=repo_id, | |
| assets=_config.AssetsConfig(assets_dir=assets_base_dir, asset_id=ASSET_ID) if assets_base_dir else | |
| _config.AssetsConfig(asset_id=ASSET_ID), | |
| base_config=_config.DataConfig(prompt_from_task=True, video_backend="pyav"), | |
| ) | |
| kw = dict(seed=0, batch_size=256) | |
| kw.update(overrides) | |
| return _config.TrainConfig( | |
| name="dexhand_ilv", model=model, data=data, | |
| weight_loader=weight_loaders.CheckpointWeightLoader("gs://openpi-assets/checkpoints/pi05_base/params"), **kw) | |
| # ------------------------------------------------------------------ data loading patch | |
| def expand_repo_ids(repo_id): | |
| home = pathlib.Path(os.environ["HF_LEROBOT_HOME"]) | |
| out = [] | |
| for part in repo_id.split("+"): | |
| if part.startswith("glob:"): | |
| hits = sorted(str(pathlib.Path(p).relative_to(home)) for p in glob.glob(str(home / part[5:]))) | |
| assert hits, f"no dataset matches {part}" | |
| out += hits | |
| else: | |
| out.append(part) | |
| return out | |
| def install_dataset_patch(): | |
| """openpi's create_torch_dataset, except: (1) several repos ('a+b', 'glob:...') are concatenated, each keeping its | |
| own task table; (2) the interleaved model's action horizon is 2T, but each sample needs T future actions.""" | |
| import torch.utils.data | |
| from lerobot.datasets import lerobot_dataset | |
| from openpi import transforms as _t | |
| from openpi.training import data_loader as _dl | |
| def create_torch_dataset(data_config, action_horizon, model_config): | |
| repo_id = data_config.repo_id | |
| if repo_id is None: | |
| raise ValueError("Repo ID is not set. Cannot create dataset.") | |
| if repo_id == "fake": | |
| return _dl.FakeDataset(model_config, num_samples=1024) | |
| T = action_horizon // 2 if getattr(model_config, "state_dim", None) else action_horizon | |
| parts = [] | |
| for r in expand_repo_ids(repo_id): | |
| meta = lerobot_dataset.LeRobotDatasetMetadata(r) | |
| ds = lerobot_dataset.LeRobotDataset( | |
| r, delta_timestamps={k: [t / meta.fps for t in range(T)] for k in data_config.action_sequence_keys}, | |
| video_backend=data_config.video_backend) | |
| if data_config.prompt_from_task: | |
| ds = _dl.TransformedDataset(ds, [_t.PromptFromLeRobotTask(meta.tasks)]) | |
| parts.append(ds) | |
| print(f"datasets: {len(parts)}, frames {sum(len(p) for p in parts)}, future actions per sample {T}", flush=True) | |
| if BALANCE_TASKS: | |
| return TaskBalancedDataset(expand_repo_ids(repo_id), parts) | |
| return parts[0] if len(parts) == 1 else torch.utils.data.ConcatDataset(parts) | |
| _dl.create_torch_dataset = create_torch_dataset | |
| BALANCE_TASKS = False # set by run.py --balance-tasks: sample tasks uniformly, frames uniformly within a task | |
| TASK_OF = {"pickup_v3": "pickup_v2", "pickup_m3b": "pickup_m3a"} # batches that are the same task (general_pickup, | |
| # two object sets) | |
| def task_key(repo): | |
| """'dexhand/pens_v2_seg_p0_s003' -> 'pens_v2'; pickup_v3 counts as pickup_v2 (one task).""" | |
| name = repo.split("/")[-1] | |
| for tag in ("_seg_", "_fseg_", "_7seg_"): | |
| if tag in name: | |
| name = name.split(tag)[0] | |
| break | |
| return TASK_OF.get(name, name) | |
| class TaskBalancedDataset: | |
| """Mid-training mixture: index i -> task i % n_tasks, frame (i // n_tasks) spread uniformly over that task's frames. | |
| With the loader's shuffle=True every task gets the same share of each batch whatever its size (cover_m2 700 and | |
| dolls_v2 577 segments vs align_v2 200: a plain concatenation gave them 37 % of the samples, 2026-10-24). Length = | |
| n_tasks x the largest task, so the biggest task is seen once per pass and the others are oversampled.""" | |
| def __init__(self, repos, parts): | |
| import torch.utils.data | |
| groups = {} | |
| for r, ds in zip(repos, parts): | |
| groups.setdefault(task_key(r), []).append(ds) | |
| self.tasks = sorted(groups) | |
| self.sets = [torch.utils.data.ConcatDataset(groups[t]) for t in self.tasks] | |
| self.sizes = [len(d) for d in self.sets] | |
| self.per_task = max(self.sizes) | |
| print("task-balanced mixture:", {t: n for t, n in zip(self.tasks, self.sizes)}, "-> per task", self.per_task, flush=True) | |
| def __len__(self): | |
| return self.per_task * len(self.tasks) | |
| def __getitem__(self, i): | |
| t, j = i % len(self.tasks), i // len(self.tasks) | |
| n = self.sizes[t] | |
| k = (j * n) // self.per_task if n < self.per_task else j # stride-map the slot onto this task's frames | |
| return self.sets[t][min(k, n - 1)] | |