add ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k_ftstep50k

ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k_ftstep50k/action_spec.json

@@ -0,0 +1,160 @@
+{
+  "model": "ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k",
+  "embodiment": "oxe_s0d_droid (DROID Franka Panda + Robotiq 2F-85, 15 Hz), VLA PT v2 left-arm head slots",
+  "weights": "EMA of the training weights (decay 0.99, openpi pi0.5 TrainConfig.ema_decay) — the checkpoint file holds the EMA",
+  "inputs": {
+    "images": {
+      "order": [
+        "exterior (exterior_1_left or exterior_2_left; training sampled one at random per sample, as the PT)",
+        "wrist_left"
+      ],
+      "preprocess": "PIL resize to 224x224 (stretch, no padding), uint8 RGB; no augmentation at inference"
+    },
+    "prompt": {
+      "template": "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format.",
+      "note": "applied inside the model from config.full.yaml CoT_prompt to the state prompt string above; send the raw instruction"
+    },
+    "state": {
+      "layout": [
+        "eef_x",
+        "eef_y",
+        "eef_z",
+        "r6d_0",
+        "r6d_1",
+        "r6d_2",
+        "r6d_3",
+        "r6d_4",
+        "r6d_5",
+        "gripper_norm",
+        "joint_0",
+        "joint_1",
+        "joint_2",
+        "joint_3",
+        "joint_4",
+        "joint_5",
+        "joint_6"
+      ],
+      "units": "eef_x..z: end-effector position in m in the robot base frame (FK of the DROID joint positions, panda_hand); r6d_0..5: end-effector rotation in the robot base frame as the first two columns of the rotation matrix [R[:,0], R[:,1]]; gripper_norm = 1 - DROID gripper_position (1 = open, 0 = closed); joint_0..6: observed joint positions in rad",
+      "use": "goes into the prompt only (pi0.5 style), not to the action head: starVLA/dataloader/ft_state_v2.py prompt(load_stats(<state_prompt>), instruction, raw_state) -> 'Task: {cleaned instruction}, State: {17 ints};\\nAction: ' with adapter droid_s0 (raw 17-D state into the VLA PT v2 left-arm block) and bins = digitize((x - q01) / (q99 - q01 + 1e-6) * 2 - 1, linspace(-1, 1, 257)[:-1]) - 1, no clip",
+      "state_prompt_stats": "droid_s0.261011.json",
+      "q01": [
+        0.26230692863464355,
+        -0.4206789080211912,
+        -0.057528488406558435,
+        -0.8242896797180718,
+        -0.8572412867108428,
+        -0.8219378955921748,
+        -0.8541473544879027,
+        -0.9957590703450361,
+        -0.8298022919839001,
+        0.00440526008605957,
+        -0.8339833669677408,
+        -0.8640233282567292,
+        -0.9020540952750266,
+        -2.771083118287859,
+        -1.6646083886230074,
+        0.533203125,
+        -2.099756536881615
+      ],
+      "q99": [
+        0.7914561699403493,
+        0.4331828368904659,
+        0.7906060505443602,
+        0.9959730506897151,
+        0.9952307314400629,
+        0.7746930340570487,
+        0.9949591160774409,
+        0.849599627866384,
+        0.7944404576319001,
+        1.0,
+        0.9373434964244457,
+        1.464923035814553,
+        0.7110593196715427,
+        -0.4518592655658722,
+        1.561298730659986,
+        3.4644680210148397,
+        2.1449354486613306
+      ],
+      "q01_q99_use": "the bins above use these (= the PT class S2 state statistics, also in the state_prompt stats file), no clip",
+      "note": "send the RAW (unnormalized) 17-D state; config.full.yaml datasets.vla_data.state_prompt names the stats file"
+    },
+    "robot_tag": {
+      "value": "oxe_s0d_droid",
+      "required": true,
+      "note": "action head is padding at action_dim=64/state_dim=66; oxe_s0d_droid sits at slot offset 0 for action (14) and state (17). Every example must carry robot_tag='oxe_s0d_droid' (QwenPI_v3_pad_q35.predict_action slices the 14 dims out of 64)."
+    }
+  },
+  "outputs": {
+    "shape": [
+      16,
+      14
+    ],
+    "horizon": 16,
+    "layout": [
+      "dx",
+      "dy",
+      "dz",
+      "drx",
+      "dry",
+      "drz",
+      "gripper_position",
+      "djoint_0",
+      "djoint_1",
+      "djoint_2",
+      "djoint_3",
+      "djoint_4",
+      "djoint_5",
+      "djoint_6"
+    ],
+    "unnormalize": {
+      "0:6": "x = (y + 1) / 2 * (q99 - q01) + q01 (trained on clip(2 (x - q01)/(q99 - q01) - 1, -1, 1); PT class S2 statistics)",
+      "6:14": "x = (y + 1) / 2 * (q99 - q01 + 1e-6) + q01 (openpi Unnormalize quantile; trained on (x - q01)/(q99 - q01 + 1e-6) * 2 - 1, no clip; openpi pi05_base droid norm_stats actions[7] for the gripper and actions[0:7] for the joints)"
+    },
+    "q01": [
+      -0.011063759418540988,
+      -0.015102064967254523,
+      -0.01545244904467366,
+      -0.03504985095760447,
+      -0.03904482649111776,
+      -0.04988467483135882,
+      0.0,
+      -0.45159998536109924,
+      -0.7979999780654907,
+      -0.438400000333786,
+      -0.9088000059127808,
+      -0.6340000033378601,
+      -0.6327999830245972,
+      -0.7516000270843506
+    ],
+    "q99": [
+      0.012471440217949912,
+      0.015453049356457752,
+      0.017237048596143723,
+      0.03658537187648141,
+      0.03663981106746467,
+      0.04898791981183934,
+      0.998199999332428,
+      0.43880000710487366,
+      0.7644000053405762,
+      0.4431999921798706,
+      0.7860000133514404,
+      0.6380000114440918,
+      0.6567999720573425,
+      0.724399983882904
+    ],
+    "to_command": {
+      "joint": "openpi AbsoluteActions: joint target for step k = x[k, 7:14] + the observed joint position at query time (the same observation for all 16 steps); this is the openpi pi05 DROID joint-position command",
+      "gripper": "x[k, 6] = DROID commanded gripper_position, absolute, 1 = closed (as openpi pi05 DROID; RoboLab binarizes > 0.5)",
+      "pose": "per-frame EEF deltas, cumulative from the observed pose at query time (p_0, R_0): p_{k+1} = p_k + x[k, 0:3]; R_{k+1} = Rotation.from_rotvec(x[k, 3:6]).as_matrix() @ R_k (base frame). Use either the joint or the pose output."
+    },
+    "training_target": "x[k, 7:14] = action.joint_position[t+k] - observation.state.joint_position[t];  x[k, 6] = action.gripper_position[t+k]  (commanded, openpi DeltaActions(make_bool_mask(7, -1)); past the episode end the last command repeats);  x[k, 0:3] = p[t+k+1] - p[t+k], x[k, 3:6] = rotvec(R[t+k+1] R[t+k]^T) (action.s0.eef_delta, 0 past the episode end)"
+  },
+  "notes": [
+    "joint / gripper representation, normalization and statistics = openpi pi05_full_droid_finetune (pi05_base assets/droid/norm_stats.json)",
+    "EEF delta slots = the VLA PT v2's (vla_v2_layout left-arm block, class S2 statistics), kept by user decision 261011",
+    "training images had the pi0.5 augmentation (exterior: crop 95% + rotate +-5 deg; all: color jitter); none at inference",
+    "exterior camera slot: training read exterior_1_left or exterior_2_left at random (the PT's vla_v2.view_alternates); send either"
+  ],
+  "framework": "QwenPI_v3_pad_q35_v2 (Qwen3.5-4B-thinkoff VLM, LayerwiseFM head, action 64 / state 66); code tree rfm-lam/exaone-vla-q35; action head = pi0.5 action expert (framework.action_model.pi05_expert true, flow_matching_head/pi05_expert.py), read from config.full.yaml",
+  "pretrained_from": "VLA PT v2 vla_pt_v2_q35pilam_Lfull5e-5_stateprompt_pi05_hum30_int159 steps_50000 (job 93775)"
+}

ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k_ftstep50k/config.full.yaml

@@ -0,0 +1,156 @@
+run_id: ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k
+run_root_dir: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/runs
+seed: 42
+wandb_entity: null
+wandb_project: starVLA_ft_droid_openpi
+is_debug: false
+version_id: '0.21'
+framework:
+  name: QwenPI_v3_pad_q35_v2
+  qwenvl:
+    base_vlm: /mnt/lgair-dataset/lgair/home/jinhyung.kim/project/rfm/lbwork/q35diag/weights/Qwen3.5-4B-thinkoff
+    attn_implementation: sdpa
+    vl_hidden_dim: 2560
+    num_vl_layers: 32
+    freeze_vlm_all: false
+    freeze_vision_encoder: false
+  action_model:
+    action_model_type: LayerwiseFM
+    action_dim: 64
+    state_dim: 66
+    action_horizon: 16
+    repeated_diffusion_steps: 8
+    num_inference_timesteps: 4
+    add_pos_embed: true
+    max_seq_len: 1024
+    num_target_vision_tokens: 32
+    noise_beta_alpha: 1.5
+    noise_beta_beta: 1.0
+    noise_s: 0.999
+    num_timestep_buckets: 1000
+    multi_embodiment_mode: padding
+    max_num_embodiments: 96
+    state_injection: none
+    diffusion_model_cfg:
+      action_dit_hidden_dim: 1024
+      dropout: 0.0
+      final_dropout: true
+      interleave_self_attention: true
+      norm_type: ada_norm
+      positional_embeddings: null
+      attention_head_dim: 64
+      output_dim: 1024
+      cross_attention_dim: 1024
+      num_layers: 32
+      input_embedding_dim: 1024
+      num_attention_heads: 16
+    dit_encoder_mask: true
+    future_action_window_size: 15
+    past_action_window_size: 0
+    pi05_expert: true
+datasets:
+  vlm_data:
+    dataset_py: vlm_datasets
+    dataformat: llava_json
+    dataset_use: sharegpt4v_coco
+    eval_dataset: sharegpt4v_coco
+    data_flatten: false
+    base_interval: 2
+    max_pixels: 307200
+    min_pixels: 784
+    model_max_length: 2048
+    model_type: qwen2.5vl
+    per_device_batch_size: 4
+  vla_data:
+    dataset_py: lerobot_datasets
+    data_root_dir: /mnt/lgair-dataset/lgair/home/jinhyung.kim/project/rfm/data/vla_root_s0max
+    data_mix: openpi_droid_v2o_sp_alt
+    action_type: delta_ee
+    sequential_step_sampling: false
+    CoT_prompt: Your task is {instruction}. To identify the key objects for your task.
+      Locate their bounding boxes in [x1,y1,x2,y2] format.
+    per_device_batch_size: 8
+    load_all_data_for_training: true
+    video_backend: pyav
+    include_state: false
+    num_workers: 8
+    preprocess_in_worker: false
+    state_prompt: /mnt/lgair-dataset/lgair/home/jmleeluck/dataroots/vla_pt_v2/ft_state_stats/droid_s0.261011.json
+trainer:
+  max_train_steps: 100000
+  num_warmup_steps: 1000
+  save_interval: 5000
+  eval_interval: 1000000
+  learning_rate:
+    base: 2.5e-05
+    qwen_vl_interface: 1.0e-05
+    action_model: 0.0001
+  lr_scheduler_type: cosine_with_min_lr
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+  freeze_modules: null
+  loss_scale:
+    vla: 1.0
+    vlm: 0.1
+  max_grad_norm: 1.0
+  ema_decay: 0.99
+  weight_decay: 0.0
+  logging_frequency: 100
+  gradient_clipping: 1.0
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: true
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  pretrained_checkpoint: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/runs/vla_pt_v2_q35pilam_Lfull5e-5_stateprompt_pi05_hum30_int159/checkpoints/steps_50000_model.safetensors
+  save_full_state: true
+  is_resume: true
+  reload_modules: null
+launch:
+  runtime: container
+  image: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/enroot/exaonevla-q35-20260911.sqsh
+  venv: /opt/venv
+  workdir: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/exaone-vla-q35
+  entry: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/exaone-vla-q35/starVLA/training/train_starvla.py
+  accelerate_config: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/exaone-vla-q35/starVLA/config/deepseeds/deepspeed_zero2.yaml
+  nodes: 4
+  gpus_per_node: 8
+  keep_last_states: 2
+  env:
+    TOKENIZERS_PARALLELISM: 'false'
+    HF_HOME: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/exaone-vla/playground/.hf_cache
+    WANDB_MODE: offline
+    STARVLA_CLUSTER: b200
+    STARVLA_PG_TIMEOUT: '7200'
+    STARVLA_STACKDUMP_DIR: '0'
+    STARVLA_NAN_GUARD: '1'
+    STARVLA_NOTIFY: '1'
+    STARVLA_NAN_GUARD_NOTIFY: '1'
+    STARVLA_GRAD_GUARD: '1'
+    STARVLA_DROP_INCOMPLETE_EPISODES: '0'
+    STARVLA_STATS_DIR: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/stats_cache
+    STARVLA_FORCE_GC: '0'
+    STARVLA_TRAJ_CACHE_MAX: '32'
+    STARVLA_ALLOW_PARTIAL_LOAD: ''
+    STARVLA_DISABLE_WORKER_PREPROCESS: ''
+    STARVLA_ALLOW_UNFROZEN_VLM: ''
+    TORCH_HOME: /mnt/lgair-dataset/lgair/home/jmleeluck/.cache/torch
+    OPENPI_DROID_FILTER: /mnt/lgair-dataset/lgair/home/jmleeluck/jobs/droid_idle.260924/full/openpi_droid.json
+    NCCL_IB_HCA: mlx5_00,mlx5_01,mlx5_02,mlx5_03,mlx5_04,mlx5_05,mlx5_06,mlx5_07
+    NCCL_SOCKET_IFNAME: bond0
+    NCCL_BLOCKING_WAIT: '1'
+    NCCL_ASYNC_ERROR_HANDLING: '1'
+    VLA_V2_PT_STATS: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/runs/ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k/pt_vla_v2_stats.json
+  slurm:
+    qos: tier2_5
+    cpus_per_task: 112
+    mem: 1900000M
+    exclusive: false
+    requeue: true
+    exclude: il02-b200-node076,il02-b200-node080,il02-b200-node086,il02-b200-node089,il02-b200-node090,il02-b200-node113,il02-b200-node127
+config_yaml: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/runs/ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k/submitted_config.261014_181453.yaml
+output_dir: /mnt/lgair-dataset/lgair/home/jmleeluck/rfm-lam/runs/ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k

ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k_ftstep50k/dataset_statistics.json

@@ -0,0 +1,236 @@
+{
+  "oxe_s0d_droid": {
+    "action": {
+      "mean": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        0.43498775362968445,
+        -0.005160212982445955,
+        0.013146234676241875,
+        0.002338092541322112,
+        0.028497571125626564,
+        0.0015346353175118566,
+        -0.0015720161609351635,
+        0.0025643655098974705
+      ],
+      "std": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        0.44128793478012085,
+        0.1532931625843048,
+        0.30146104097366333,
+        0.14914393424987793,
+        0.294704794883728,
+        0.22254551947116852,
+        0.24063511192798615,
+        0.2638004422187805
+      ],
+      "max": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN
+      ],
+      "min": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN
+      ],
+      "q01": [
+        -0.011063759418540988,
+        -0.015102064967254523,
+        -0.01545244904467366,
+        -0.03504985095760447,
+        -0.03904482649111776,
+        -0.04988467483135882,
+        0.0,
+        -0.45159998536109924,
+        -0.7979999780654907,
+        -0.438400000333786,
+        -0.9088000059127808,
+        -0.6340000033378601,
+        -0.6327999830245972,
+        -0.7516000270843506
+      ],
+      "q99": [
+        0.012471440217949912,
+        0.015453049356457752,
+        0.017237048596143723,
+        0.03658537187648141,
+        0.03663981106746467,
+        0.04898791981183934,
+        0.998199999332428,
+        0.43880000710487366,
+        0.7644000053405762,
+        0.4431999921798706,
+        0.7860000133514404,
+        0.6380000114440918,
+        0.6567999720573425,
+        0.724399983882904
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true
+      ]
+    },
+    "state": {
+      "mean": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN
+      ],
+      "std": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN
+      ],
+      "max": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN
+      ],
+      "min": [
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN,
+        NaN
+      ],
+      "q01": [
+        0.26230692863464355,
+        -0.4206789080211912,
+        -0.057528488406558435,
+        -0.8242896797180718,
+        -0.8572412867108428,
+        -0.8219378955921748,
+        -0.8541473544879027,
+        -0.9957590703450361,
+        -0.8298022919839001,
+        0.00440526008605957,
+        -0.8339833669677408,
+        -0.8640233282567292,
+        -0.9020540952750266,
+        -2.771083118287859,
+        -1.6646083886230074,
+        0.533203125,
+        -2.099756536881615
+      ],
+      "q99": [
+        0.7914561699403493,
+        0.4331828368904659,
+        0.7906060505443602,
+        0.9959730506897151,
+        0.9952307314400629,
+        0.7746930340570487,
+        0.9949591160774409,
+        0.849599627866384,
+        0.7944404576319001,
+        1.0,
+        0.9373434964244457,
+        1.464923035814553,
+        0.7110593196715427,
+        -0.4518592655658722,
+        1.561298730659986,
+        3.4644680210148397,
+        2.1449354486613306
+      ]
+    },
+    "num_transitions": 21059892,
+    "num_trajectories": 78724
+  }
+}

ft_droid_v2o_q35piv2_Lfull5e-5_stateprompt_pi05_hum30_int159_50k_ftstep50k/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e495fd25fe5b3fa14a2c7c5f96523e0a366733431eeed562ca868a4cc499c7b
+size 12162397705