# Training config for LingBot-VA on the real-world Franka "fruit_pick" dataset # (data/fruit_pick_lerobot_v21 -- 100 episodes, 20149 frames, task "Lift the # lid, put it aside, and pick the black plum."). # # Source: downloaded 2026-08-09 from https://huggingface.co/datasets/aabyaneh/fruit_pick # as LeRobot codebase_version v3.0 (one shared video file per camera, not # per-episode) -- LingBot-VA's own pipeline (extract_latents.py, # LatentLeRobotDataset) has no v3.0 awareness (hardcodes revision="v2.1", # opens per-episode mp4 paths directly), so this repo carries its own genuine # v2.1 conversion at data/fruit_pick_lerobot_v21, produced by # scripts/convert_fruit_pick_v3_to_v21.py (decodes both source videos # sequentially, re-encodes per-episode via the official # LeRobotDataset.create()/add_frame()/save_episode() writer API -- verified # byte-identical state/action values and matching frame counts against the # source for spot-checked episodes) + scripts/backfill_action_config.py (the # official writer doesn't produce the action_config field extract_latents.py # needs; same gap hit for place_cube_in_bowl, same fix). # # observation.state is already this repo's standard 8-dim eef_pos(3) + # axisangle(3) + gripper(2) convention (fastwam_train's # fix_fruit_pick_state_schema.py rewrote it from HF's native 15-dim state # using the SAME quat2axisangle formula lift2lerobot's own converter uses -- # so this is consistent with lift_new/place_cube_bowl's state convention, # not a new one). action is 7-dim delta_xyz + delta_rot(always 0) + grasp, # same schema/mask as every other dataset here. # # Gripper convention DIFFERS from place_cube_bowl: fruit_pick's raw action # column is natively {-1, +1} (verified: np.unique gives exactly [-1., 1.]), # NOT place_cube_bowl's remapped {0.0=closed, 1.0=open}. This dataset was # never passed through the (1-g)/2 HDF5-sourced remap (it came pre-converted # from HF, not from raw HDF5), so the value is whatever the original # collection tool logged -- which of {-1, +1} means open vs. closed has NOT # been verified against a physical signal (no gripper_width channel was kept # after the 15->8 dim state rewrite to cross-check against, unlike # place_cube_bowl's verification). This does NOT affect training (norm_stat # below is computed FROM THIS DATASET'S OWN actual range, not reused from # place_cube_bowl, so the quantile normalization is internally consistent # regardless of which polarity is "open") -- but get the real mapping # confirmed before deploying any policy trained from this config. # # q01/q99 below are computed from ALL 100 episodes' actual action data # (data/fruit_pick_lerobot_v21/data/chunk-000/*.parquet, verified byte-copy # of the source's action column) -- there is no held-out split to exclude # from the normalizer here (see below). # # NO validation split (user decision 2026-08-09, matching the FastWAM # fruit_pick jobs: use the entire 100-episode dataset for training). Neither # `num_train_episodes` nor `val_split` is set below -- wan_va/train.py's # Trainer.__init__ getattr-defaults both to 0/falsy, landing on the # `else: train_dataset = full_dataset; val_dataset = None` branch (verified # by reading train.py directly) rather than risking an empty val Subset from # num_train_episodes=100 (=total episodes). import os from copy import deepcopy from easydict import EasyDict from wan_va.configs.va_libero_cfg import va_libero_cfg REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) WAN_INIT_PATH = os.environ.get("WAM_WAN_INIT", os.path.join(REPO_ROOT, "ckpts/lingbot_va_wan_init")) LINGBOT_BASE_PATH = os.environ.get("WAM_LINGBOT_BASE", os.path.join(REPO_ROOT, "ckpts/lingbot-va-base")) DATASET_PATH = os.environ.get( "WAM_FRUIT_PICK_DATASET", os.path.join(REPO_ROOT, "data/fruit_pick_lerobot_v21"), ) def _make(init_path: str) -> EasyDict: cfg = EasyDict(__name__="Config: VA fruit_pick") cfg.update(deepcopy(va_libero_cfg)) cfg.wan22_pretrained_model_name_or_path = init_path cfg.dataset_path = DATASET_PATH cfg.empty_emb_path = os.path.join(cfg.dataset_path, "empty_emb.pt") cfg.height = 256 cfg.width = 256 cfg.obs_cam_keys = ["observation.images.image", "observation.images.wrist_image"] cfg.action_dim = 30 cfg.used_action_channel_ids = list(range(0, 7)) inverse_ids = [len(cfg.used_action_channel_ids)] * cfg.action_dim for i, j in enumerate(cfg.used_action_channel_ids): inverse_ids[j] = i cfg.inverse_used_action_channel_ids = inverse_ids cfg.action_norm_method = "quantiles" # Computed 2026-08-09 from ALL 100 episodes of data/fruit_pick_lerobot_v21 # (np.quantile(actions, 0.01/0.99, axis=0)) -- fruit_pick's own actual # range, NOT place_cube_bowl's numbers (different task, different scale). cfg.norm_stat = { "q01": [ -0.0054, -0.01794857, -0.01162286, 0.0, 0.0, 0.0, -1.0, ] + [0.0] * 23, "q99": [ 0.0087696, 0.01789714, 0.0120096, 0.0, 0.0, 0.0, 1.0, ] + [0.0] * 23, } cfg.action_channel_names = ["dx", "dy", "dz", "drx", "dry", "drz", "gripper"] cfg.action_channel_groups = {"xyz": [0, 1, 2], "rotation": [3, 4, 5], "gripper": [6]} cfg.gripper_channel_index = 6 # raw units are {-1.0, 1.0}; polarity meaning unverified, see header # Same phase-2 fine-tune recipe as lift_new_base/place_cube_bowl_base (arXiv:2601.21998 # "Post-Training Details" for adapting a robot-co-pretrained init to a real-world # platform): this is the same Franka platform, just a different task, so the same # recipe applies rather than the scratch-init 8000-step floor. cfg.enable_wandb = True cfg.load_worker = 8 cfg.save_interval = 200 cfg.val_interval = cfg.save_interval cfg.gc_interval = 50 cfg.cfg_prob = 0.1 cfg.learning_rate = 1e-5 cfg.beta1, cfg.beta2 = 0.9, 0.95 cfg.weight_decay = 1e-1 cfg.warmup_steps = 10 cfg.batch_size = 1 # Global batch is the recipe-critical quantity (maintainers' guidance is 32-64 episodes; # place_cube_bowl/lift_new jobs ran at 80) -- derive grad_accum from the launch's GPU # count instead of hardcoding it, same as va_place_cube_bowl_cfg.py. _global_batch = int(os.environ.get("WAM_GLOBAL_BATCH", 32)) _world_size = int(os.environ.get("WORLD_SIZE", 1)) if _global_batch % (cfg.batch_size * _world_size) != 0: raise ValueError( f"WAM_GLOBAL_BATCH={_global_batch} is not divisible by " f"batch_size({cfg.batch_size}) x WORLD_SIZE({_world_size})" ) cfg.gradient_accumulation_steps = _global_batch // (cfg.batch_size * _world_size) cfg.num_steps = int( os.environ.get("WAM_NUM_STEPS", 8000 if init_path == WAN_INIT_PATH else 3000) ) # Optional weight warm-start; see va_place_cube_bowl_cfg.py's header for the exact # semantics (WEIGHTS ONLY, not a bit-exact resume). _resume_from = os.environ.get("WAM_RESUME_FROM") if _resume_from: cfg.resume_from = _resume_from return cfg FRUIT_PICK_CONFIGS = { "fruit_pick_scratch": _make(WAN_INIT_PATH), "fruit_pick_base": _make(LINGBOT_BASE_PATH), }