training_code/fruit_pick_config.py

# Training config for LingBot-VA on the real-world Franka "fruit_pick" dataset
# (data/fruit_pick_lerobot_v21 -- 100 episodes, 20149 frames, task "Lift the
# lid, put it aside, and pick the black plum.").
#
# Source: downloaded 2026-08-09 from https://huggingface.co/datasets/aabyaneh/fruit_pick
# as LeRobot codebase_version v3.0 (one shared video file per camera, not
# per-episode) -- LingBot-VA's own pipeline (extract_latents.py,
# LatentLeRobotDataset) has no v3.0 awareness (hardcodes revision="v2.1",
# opens per-episode mp4 paths directly), so this repo carries its own genuine
# v2.1 conversion at data/fruit_pick_lerobot_v21, produced by
# scripts/convert_fruit_pick_v3_to_v21.py (decodes both source videos
# sequentially, re-encodes per-episode via the official
# LeRobotDataset.create()/add_frame()/save_episode() writer API -- verified
# byte-identical state/action values and matching frame counts against the
# source for spot-checked episodes) + scripts/backfill_action_config.py (the
# official writer doesn't produce the action_config field extract_latents.py
# needs; same gap hit for place_cube_in_bowl, same fix).
#
# observation.state is already this repo's standard 8-dim eef_pos(3) +
# axisangle(3) + gripper(2) convention (fastwam_train's
# fix_fruit_pick_state_schema.py rewrote it from HF's native 15-dim state
# using the SAME quat2axisangle formula lift2lerobot's own converter uses --
# so this is consistent with lift_new/place_cube_bowl's state convention,
# not a new one). action is 7-dim delta_xyz + delta_rot(always 0) + grasp,
# same schema/mask as every other dataset here.
#
# Gripper convention DIFFERS from place_cube_bowl: fruit_pick's raw action
# column is natively {-1, +1} (verified: np.unique gives exactly [-1., 1.]),
# NOT place_cube_bowl's remapped {0.0=closed, 1.0=open}. This dataset was
# never passed through the (1-g)/2 HDF5-sourced remap (it came pre-converted
# from HF, not from raw HDF5), so the value is whatever the original
# collection tool logged -- which of {-1, +1} means open vs. closed has NOT
# been verified against a physical signal (no gripper_width channel was kept
# after the 15->8 dim state rewrite to cross-check against, unlike
# place_cube_bowl's verification). This does NOT affect training (norm_stat
# below is computed FROM THIS DATASET'S OWN actual range, not reused from
# place_cube_bowl, so the quantile normalization is internally consistent
# regardless of which polarity is "open") -- but get the real mapping
# confirmed before deploying any policy trained from this config.
#
# q01/q99 below are computed from ALL 100 episodes' actual action data
# (data/fruit_pick_lerobot_v21/data/chunk-000/*.parquet, verified byte-copy
# of the source's action column) -- there is no held-out split to exclude
# from the normalizer here (see below).
#
# NO validation split (user decision 2026-08-09, matching the FastWAM
# fruit_pick jobs: use the entire 100-episode dataset for training). Neither
# `num_train_episodes` nor `val_split` is set below -- wan_va/train.py's
# Trainer.__init__ getattr-defaults both to 0/falsy, landing on the
# `else: train_dataset = full_dataset; val_dataset = None` branch (verified
# by reading train.py directly) rather than risking an empty val Subset from
# num_train_episodes=100 (=total episodes).
import os
from copy import deepcopy

from easydict import EasyDict

from wan_va.configs.va_libero_cfg import va_libero_cfg

REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

WAN_INIT_PATH = os.environ.get("WAM_WAN_INIT", os.path.join(REPO_ROOT, "ckpts/lingbot_va_wan_init"))
LINGBOT_BASE_PATH = os.environ.get("WAM_LINGBOT_BASE", os.path.join(REPO_ROOT, "ckpts/lingbot-va-base"))
DATASET_PATH = os.environ.get(
    "WAM_FRUIT_PICK_DATASET",
    os.path.join(REPO_ROOT, "data/fruit_pick_lerobot_v21"),
)


def _make(init_path: str) -> EasyDict:
    cfg = EasyDict(__name__="Config: VA fruit_pick")
    cfg.update(deepcopy(va_libero_cfg))
    cfg.wan22_pretrained_model_name_or_path = init_path

    cfg.dataset_path = DATASET_PATH
    cfg.empty_emb_path = os.path.join(cfg.dataset_path, "empty_emb.pt")

    cfg.height = 256
    cfg.width = 256
    cfg.obs_cam_keys = ["observation.images.image", "observation.images.wrist_image"]

    cfg.action_dim = 30
    cfg.used_action_channel_ids = list(range(0, 7))
    inverse_ids = [len(cfg.used_action_channel_ids)] * cfg.action_dim
    for i, j in enumerate(cfg.used_action_channel_ids):
        inverse_ids[j] = i
    cfg.inverse_used_action_channel_ids = inverse_ids

    cfg.action_norm_method = "quantiles"
    # Computed 2026-08-09 from ALL 100 episodes of data/fruit_pick_lerobot_v21
    # (np.quantile(actions, 0.01/0.99, axis=0)) -- fruit_pick's own actual
    # range, NOT place_cube_bowl's numbers (different task, different scale).
    cfg.norm_stat = {
        "q01": [
            -0.0054, -0.01794857, -0.01162286, 0.0, 0.0, 0.0, -1.0,
        ] + [0.0] * 23,
        "q99": [
            0.0087696, 0.01789714, 0.0120096, 0.0, 0.0, 0.0, 1.0,
        ] + [0.0] * 23,
    }

    cfg.action_channel_names = ["dx", "dy", "dz", "drx", "dry", "drz", "gripper"]
    cfg.action_channel_groups = {"xyz": [0, 1, 2], "rotation": [3, 4, 5], "gripper": [6]}
    cfg.gripper_channel_index = 6  # raw units are {-1.0, 1.0}; polarity meaning unverified, see header

    # Same phase-2 fine-tune recipe as lift_new_base/place_cube_bowl_base (arXiv:2601.21998
    # "Post-Training Details" for adapting a robot-co-pretrained init to a real-world
    # platform): this is the same Franka platform, just a different task, so the same
    # recipe applies rather than the scratch-init 8000-step floor.
    cfg.enable_wandb = True
    cfg.load_worker = 8
    cfg.save_interval = 200
    cfg.val_interval = cfg.save_interval
    cfg.gc_interval = 50
    cfg.cfg_prob = 0.1
    cfg.learning_rate = 1e-5
    cfg.beta1, cfg.beta2 = 0.9, 0.95
    cfg.weight_decay = 1e-1
    cfg.warmup_steps = 10
    cfg.batch_size = 1
    # Global batch is the recipe-critical quantity (maintainers' guidance is 32-64 episodes;
    # place_cube_bowl/lift_new jobs ran at 80) -- derive grad_accum from the launch's GPU
    # count instead of hardcoding it, same as va_place_cube_bowl_cfg.py.
    _global_batch = int(os.environ.get("WAM_GLOBAL_BATCH", 32))
    _world_size = int(os.environ.get("WORLD_SIZE", 1))
    if _global_batch % (cfg.batch_size * _world_size) != 0:
        raise ValueError(
            f"WAM_GLOBAL_BATCH={_global_batch} is not divisible by "
            f"batch_size({cfg.batch_size}) x WORLD_SIZE({_world_size})"
        )
    cfg.gradient_accumulation_steps = _global_batch // (cfg.batch_size * _world_size)
    cfg.num_steps = int(
        os.environ.get("WAM_NUM_STEPS", 8000 if init_path == WAN_INIT_PATH else 3000)
    )

    # Optional weight warm-start; see va_place_cube_bowl_cfg.py's header for the exact
    # semantics (WEIGHTS ONLY, not a bit-exact resume).
    _resume_from = os.environ.get("WAM_RESUME_FROM")
    if _resume_from:
        cfg.resume_from = _resume_from

    return cfg


FRUIT_PICK_CONFIGS = {
    "fruit_pick_scratch": _make(WAN_INIT_PATH),
    "fruit_pick_base": _make(LINGBOT_BASE_PATH),
}