dexhand-train-kit / code /dexhand /train /dexhand_openpi.py
Noctis12138's picture
Add files using upload-large-folder tool
6a08a28 verified
Raw History Blame Contribute Delete
11.4 kB
"""openpi (XPolicyLab's vendored copy, not edited) configuration for the dual Marvin + Wuji robot, sub-task data.
Data: LeRobot v3.0 shards of gen/to_lerobot.py (HF Noctis12138/dexhand-cover-data, lerobot/cover_v2_7seg_*):
observation.state / action 54-D (left arm 7, left hand 20, right arm 7, right hand 20; action = commanded joint
targets), observation.images.cam_high / cam_left_wrist / cam_right_wrist (480x640), task = sub-task instruction.
Model: pi0.5 from pi05_base with the action dim kept at 32 (user decision: every pretrained weight is reused, no
widened head). The 54-D action chunk is interleaved left / right ("Ego-Pi style"):
(T, 54) -> left [arm 7 + hand 20] and right [arm 7 + hand 20], each zero-padded 27 -> 32 -> L0, R0, L1, R1, ...
= (2T, 32); the model's action horizon is 2T, the data loader reads T future actions.
The 54-D state is not interleaved: pi0.5 only uses it as discretised text in the prompt (TokenizePrompt runs before
the interleave), and the policy hands the transformed state to the output transforms (AbsoluteActions needs all 54).
So the model's state spec is 54-D (Pi05Interleaved.inputs_spec); the network never reads obs.state in pi0.5.
Everything else follows XPolicyLab's own pi05_wuji_marvin_54d config (same robot): WujiInputs / WujiOutputs camera
mapping, arms as deltas and hands absolute (make_bool_mask(7, -20, 7, -20)), quantile normalisation. Difference:
action_dim 32 + interleave instead of action_dim 54, the full (not partial) pi05_base weights, cam_high as base camera.
"""
import dataclasses
import glob
import os
import pathlib
import numpy as np
SPLIT, PAD = 27, 32 # per-side dims (arm 7 + hand 20), padded per-side width = model action dim
ASSET_ID = "dual_marvin_wuji"
# ------------------------------------------------------------------ transforms
def _transforms():
from openpi import transforms as _t
return _t
@dataclasses.dataclass(frozen=True)
class InterleaveLR:
"""actions (..., T, 54) -> (..., 2T, 32): L0, R0, L1, R1, ... (each side zero-padded 27 -> 32)."""
def __call__(self, data):
if "actions" not in data:
return data
a = np.asarray(data["actions"])
assert a.shape[-1] == 2 * SPLIT, a.shape
pad = [(0, 0)] * (a.ndim - 1) + [(0, PAD - SPLIT)]
lr = np.stack([np.pad(a[..., :SPLIT], pad), np.pad(a[..., SPLIT:], pad)], axis=-2) # (..., T, 2, 32)
data["actions"] = lr.reshape(*a.shape[:-2], 2 * a.shape[-2], PAD)
return data
@dataclasses.dataclass(frozen=True)
class DeinterleaveLR:
"""actions (..., 2T, 32) -> (..., T, 54); inverse of InterleaveLR (padding dropped)."""
def __call__(self, data):
a = np.asarray(data["actions"])
assert a.shape[-2] % 2 == 0 and a.shape[-1] >= SPLIT, a.shape
lr = a.reshape(*a.shape[:-2], a.shape[-2] // 2, 2, a.shape[-1])
data["actions"] = np.concatenate([lr[..., 0, :SPLIT], lr[..., 1, :SPLIT]], axis=-1)
return data
class RemoveStrings:
"""Drop string fields (prompt) before batching numeric stats (module level: picklable for spawn workers)."""
def __call__(self, x):
return {k: v for k, v in x.items() if not np.issubdtype(np.asarray(v).dtype, np.str_)}
# ------------------------------------------------------------------ model / data configs
def _classes():
"""Built lazily: openpi is only importable inside its venv."""
import jax
import jax.numpy as jnp
from openpi.models import pi0_config
from openpi.policies import wuji_policy
from openpi.training import config as _config
from openpi.shared import array_typing as at
_t = _transforms()
@dataclasses.dataclass(frozen=True)
class Pi05Interleaved(pi0_config.Pi0Config):
"""Pi0Config whose observation state is state_dim wide (54) while actions stay action_dim (32) wide."""
state_dim: int = 54
def inputs_spec(self, *, batch_size: int = 1):
obs, act = super().inputs_spec(batch_size=batch_size)
with at.disable_typechecking():
obs = dataclasses.replace(obs, state=jax.ShapeDtypeStruct([batch_size, self.state_dim], jnp.float32))
return obs, act
@dataclasses.dataclass(frozen=True)
class DexhandDataConfig(_config.DataConfigFactory):
base_image_key: str = "observation.images.cam_high"
action_sequence_keys: tuple = ("action",)
def create(self, assets_dirs: pathlib.Path, model_config) -> _config.DataConfig:
repack = _t.Group(inputs=[_t.RepackTransform({
"observation/image": self.base_image_key,
"observation/left_wrist_image": "observation.images.cam_left_wrist",
"observation/right_wrist_image": "observation.images.cam_right_wrist",
"observation/state": "observation.state",
"actions": "action",
"prompt": "prompt",
})])
mask = _t.make_bool_mask(7, -20, 7, -20) # arms delta, hands absolute (PD past contact)
data_tf = _t.Group(
inputs=[wuji_policy.WujiInputs(model_type=model_config.model_type), _t.DeltaActions(mask)],
outputs=[_t.AbsoluteActions(mask), wuji_policy.WujiOutputs(action_dim=2 * SPLIT)],
)
base = _config.ModelTransformFactory()(model_config)
ins = list(base.inputs)
k = next(i for i, f in enumerate(ins) if type(f).__name__ == "TokenizePrompt") + 1
model_tf = _t.Group(inputs=[*ins[:k], InterleaveLR(), *ins[k:]], # tokenise the 54-D state first
outputs=[DeinterleaveLR(), *base.outputs])
return dataclasses.replace(
self.create_base_config(assets_dirs, model_config),
repack_transforms=repack, data_transforms=data_tf, model_transforms=model_tf,
action_sequence_keys=self.action_sequence_keys,
)
return Pi05Interleaved, DexhandDataConfig
def get_config(repo_id, *, T=50, max_token_len=288, assets_base_dir=None, **overrides):
"""TrainConfig 'dexhand_ilv'. repo_id: a LeRobot repo id under $HF_LEROBOT_HOME, 'a+b+...', or 'glob:<pattern>'
(e.g. 'glob:dexhand/cover_v2_7seg_*'), expanded by the dataset patch below."""
from openpi.training import config as _config
from openpi.training import weight_loaders
Pi05Interleaved, DexhandDataConfig = _classes()
model = Pi05Interleaved(pi05=True, action_dim=PAD, action_horizon=2 * T, max_token_len=max_token_len)
data = DexhandDataConfig(
repo_id=repo_id,
assets=_config.AssetsConfig(assets_dir=assets_base_dir, asset_id=ASSET_ID) if assets_base_dir else
_config.AssetsConfig(asset_id=ASSET_ID),
base_config=_config.DataConfig(prompt_from_task=True, video_backend="pyav"),
)
kw = dict(seed=0, batch_size=256)
kw.update(overrides)
return _config.TrainConfig(
name="dexhand_ilv", model=model, data=data,
weight_loader=weight_loaders.CheckpointWeightLoader("gs://openpi-assets/checkpoints/pi05_base/params"), **kw)
# ------------------------------------------------------------------ data loading patch
def expand_repo_ids(repo_id):
home = pathlib.Path(os.environ["HF_LEROBOT_HOME"])
out = []
for part in repo_id.split("+"):
if part.startswith("glob:"):
hits = sorted(str(pathlib.Path(p).relative_to(home)) for p in glob.glob(str(home / part[5:])))
assert hits, f"no dataset matches {part}"
out += hits
else:
out.append(part)
return out
def install_dataset_patch():
"""openpi's create_torch_dataset, except: (1) several repos ('a+b', 'glob:...') are concatenated, each keeping its
own task table; (2) the interleaved model's action horizon is 2T, but each sample needs T future actions."""
import torch.utils.data
from lerobot.datasets import lerobot_dataset
from openpi import transforms as _t
from openpi.training import data_loader as _dl
def create_torch_dataset(data_config, action_horizon, model_config):
repo_id = data_config.repo_id
if repo_id is None:
raise ValueError("Repo ID is not set. Cannot create dataset.")
if repo_id == "fake":
return _dl.FakeDataset(model_config, num_samples=1024)
T = action_horizon // 2 if getattr(model_config, "state_dim", None) else action_horizon
parts = []
for r in expand_repo_ids(repo_id):
meta = lerobot_dataset.LeRobotDatasetMetadata(r)
ds = lerobot_dataset.LeRobotDataset(
r, delta_timestamps={k: [t / meta.fps for t in range(T)] for k in data_config.action_sequence_keys},
video_backend=data_config.video_backend)
if data_config.prompt_from_task:
ds = _dl.TransformedDataset(ds, [_t.PromptFromLeRobotTask(meta.tasks)])
parts.append(ds)
print(f"datasets: {len(parts)}, frames {sum(len(p) for p in parts)}, future actions per sample {T}", flush=True)
if BALANCE_TASKS:
return TaskBalancedDataset(expand_repo_ids(repo_id), parts)
return parts[0] if len(parts) == 1 else torch.utils.data.ConcatDataset(parts)
_dl.create_torch_dataset = create_torch_dataset
BALANCE_TASKS = False # set by run.py --balance-tasks: sample tasks uniformly, frames uniformly within a task
TASK_OF = {"pickup_v3": "pickup_v2", "pickup_m3b": "pickup_m3a"} # batches that are the same task (general_pickup,
# two object sets)
def task_key(repo):
"""'dexhand/pens_v2_seg_p0_s003' -> 'pens_v2'; pickup_v3 counts as pickup_v2 (one task)."""
name = repo.split("/")[-1]
for tag in ("_seg_", "_fseg_", "_7seg_"):
if tag in name:
name = name.split(tag)[0]
break
return TASK_OF.get(name, name)
class TaskBalancedDataset:
"""Mid-training mixture: index i -> task i % n_tasks, frame (i // n_tasks) spread uniformly over that task's frames.
With the loader's shuffle=True every task gets the same share of each batch whatever its size (cover_m2 700 and
dolls_v2 577 segments vs align_v2 200: a plain concatenation gave them 37 % of the samples, 2026-10-24). Length =
n_tasks x the largest task, so the biggest task is seen once per pass and the others are oversampled."""
def __init__(self, repos, parts):
import torch.utils.data
groups = {}
for r, ds in zip(repos, parts):
groups.setdefault(task_key(r), []).append(ds)
self.tasks = sorted(groups)
self.sets = [torch.utils.data.ConcatDataset(groups[t]) for t in self.tasks]
self.sizes = [len(d) for d in self.sets]
self.per_task = max(self.sizes)
print("task-balanced mixture:", {t: n for t, n in zip(self.tasks, self.sizes)}, "-> per task", self.per_task, flush=True)
def __len__(self):
return self.per_task * len(self.tasks)
def __getitem__(self, i):
t, j = i % len(self.tasks), i // len(self.tasks)
n = self.sizes[t]
k = (j * n) // self.per_task if n < self.per_task else j # stride-map the slot onto this task's frames
return self.sets[t][min(k, n - 1)]