checkpoint bundle
Browse files- README.md +11 -0
- bridgequant_vla.yaml +218 -0
- config.json +233 -0
- metadata.json +22 -0
- model.safetensors +3 -0
- norm_stats.json +66 -0
- train_config.yaml +244 -0
README.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: other
|
| 3 |
+
tags: [robotics, vision-language-action, bridgequant]
|
| 4 |
+
---
|
| 5 |
+
# bridgequant-robotwin-76m
|
| 6 |
+
|
| 7 |
+
BridgeQuant 76M trained jointly on three RoboTwin 2.0 tasks (beat_block_hammer, stack_blocks_two, place_empty_cup; 50 clean demos each; relative arm joints). Multi-task joint training reached 0% in the simulator with this budget; the single-task 7M (`bridgequant-robotwin-7m-hammer`) is the working RoboTwin checkpoint.
|
| 8 |
+
|
| 9 |
+
Files: `model.safetensors`, `norm_stats.json`, `bridgequant_vla.yaml` / `config.json` / `metadata.json` (the bundle `load_trainable_weights` expects), `train_config.yaml` (the experiment YAML; evaluation and deployment use the same YAML).
|
| 10 |
+
|
| 11 |
+
Code: https://github.com/VLAOpt/bridgequant-async
|
bridgequant_vla.yaml
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
policy:
|
| 2 |
+
checkpoint: runs/robotwin_tinyvl_augS_wd3e-3_rel/checkpoint_last
|
| 3 |
+
device: cuda:0
|
| 4 |
+
init_mode: scratch
|
| 5 |
+
base_checkpoint: null
|
| 6 |
+
state_dim: 14
|
| 7 |
+
action_dim: 14
|
| 8 |
+
n_cameras: 3
|
| 9 |
+
norm_stats_checkpoint: null
|
| 10 |
+
n_embodiments: 1
|
| 11 |
+
max_state_dim: 32
|
| 12 |
+
max_action_dim: 32
|
| 13 |
+
prefix_format: raw
|
| 14 |
+
model:
|
| 15 |
+
type: bridgequant_async_vla
|
| 16 |
+
backbone:
|
| 17 |
+
type: tiny_vl
|
| 18 |
+
model_name: prajjwal1/bert-mini
|
| 19 |
+
num_layers: 4
|
| 20 |
+
train_mode: finetune
|
| 21 |
+
language_first: false
|
| 22 |
+
kwargs:
|
| 23 |
+
load_weights: true
|
| 24 |
+
vision_model: facebook/dinov3-vits16-pretrain-lvd1689m
|
| 25 |
+
hidden_size: 512
|
| 26 |
+
image_size: 256
|
| 27 |
+
n_cameras: 3
|
| 28 |
+
num_heads: 8
|
| 29 |
+
mlp_ratio: 4.0
|
| 30 |
+
mask_mode: m1
|
| 31 |
+
freeze_text: true
|
| 32 |
+
expert:
|
| 33 |
+
type: bridgequantasyncvla_expert
|
| 34 |
+
hidden_size: 512
|
| 35 |
+
depth: 6
|
| 36 |
+
num_heads: 8
|
| 37 |
+
mlp_ratio: 4.0
|
| 38 |
+
moh:
|
| 39 |
+
enabled: false
|
| 40 |
+
horizons: []
|
| 41 |
+
stride: 0
|
| 42 |
+
gate_temperature: 1.0
|
| 43 |
+
ind_weight: 1.0
|
| 44 |
+
bal_weight: 0.001
|
| 45 |
+
ind_reduction: sum
|
| 46 |
+
kwargs:
|
| 47 |
+
chunk_index_features: true
|
| 48 |
+
action_head:
|
| 49 |
+
type: bridge_flow_matching
|
| 50 |
+
action_horizon: 16
|
| 51 |
+
n_action_steps: 8
|
| 52 |
+
num_steps: 1
|
| 53 |
+
gripper_bottleneck: false
|
| 54 |
+
bottleneck_residual_lambda: 0.01
|
| 55 |
+
bottleneck_residual_init_alpha: 0.01
|
| 56 |
+
kwargs: {}
|
| 57 |
+
proprio:
|
| 58 |
+
proprio_to_vlm: false
|
| 59 |
+
proprio_to_policy: true
|
| 60 |
+
state_dropout: 0.0
|
| 61 |
+
cycle_count_in_state: false
|
| 62 |
+
kwargs: {}
|
| 63 |
+
temporal:
|
| 64 |
+
n_obs_steps: 1
|
| 65 |
+
enabled: false
|
| 66 |
+
type: retnet
|
| 67 |
+
window_size: 1
|
| 68 |
+
context_refresh_every: 8
|
| 69 |
+
context_validity: true
|
| 70 |
+
memory_age_aug: 0.0
|
| 71 |
+
retention_chunk_size: 0
|
| 72 |
+
decay_min: 0.03
|
| 73 |
+
decay_max: 1.0
|
| 74 |
+
burnin_steps: 0
|
| 75 |
+
history_noise_std: 0.0
|
| 76 |
+
history_noise_rho: 0.0
|
| 77 |
+
history_source: executed_action
|
| 78 |
+
context_refresh_choices: null
|
| 79 |
+
drift_loss_weight: 0.0
|
| 80 |
+
reinjection_mode: additive
|
| 81 |
+
refresh_proprio_delta_threshold: null
|
| 82 |
+
refresh_action_magnitude_threshold: null
|
| 83 |
+
refresh_frame_diff_threshold: null
|
| 84 |
+
refresh_instruction_change: true
|
| 85 |
+
d_model: 256
|
| 86 |
+
num_heads: 4
|
| 87 |
+
control_dt: 0.0667
|
| 88 |
+
kwargs: {}
|
| 89 |
+
stream_units: 1
|
| 90 |
+
stream_stride: 5
|
| 91 |
+
stream_stride_jitter: 2
|
| 92 |
+
bridge:
|
| 93 |
+
enabled: true
|
| 94 |
+
num_queries: 96
|
| 95 |
+
dim: 512
|
| 96 |
+
tap_layers:
|
| 97 |
+
- 0
|
| 98 |
+
- 1
|
| 99 |
+
- 2
|
| 100 |
+
- 3
|
| 101 |
+
rank: 256
|
| 102 |
+
min_grounding_confidence: null
|
| 103 |
+
gate_init: 0.3
|
| 104 |
+
z_bridge_pos_embed: true
|
| 105 |
+
zero_z_bridge: false
|
| 106 |
+
kwargs: {}
|
| 107 |
+
roles:
|
| 108 |
+
entity: 24
|
| 109 |
+
goal: 24
|
| 110 |
+
embodiment: 24
|
| 111 |
+
progress: 12
|
| 112 |
+
global_context: 12
|
| 113 |
+
alignment: 0
|
| 114 |
+
alignment_asymmetric_injection: false
|
| 115 |
+
alignment_asymmetric_scale: 1.0
|
| 116 |
+
plan_film: false
|
| 117 |
+
plan_film_scale: 1.0
|
| 118 |
+
plan_film_state: false
|
| 119 |
+
lang_kv_to_expert: false
|
| 120 |
+
mem_kv_to_expert: false
|
| 121 |
+
masked_groups: []
|
| 122 |
+
routing: null
|
| 123 |
+
routing_rows: null
|
| 124 |
+
spatial_binding_weight: 0.0
|
| 125 |
+
spatial_binding_hidden: 256
|
| 126 |
+
spatial_binding_huber_delta: null
|
| 127 |
+
alignment_offset_weight: 0.0
|
| 128 |
+
alignment_source_group: null
|
| 129 |
+
alignment_margin_weight: 0.0
|
| 130 |
+
alignment_margin_epsilon_cm: 2.5
|
| 131 |
+
alignment_huber_delta: 0.1
|
| 132 |
+
alignment_hidden: 256
|
| 133 |
+
alignment_phase_beta: 2.0
|
| 134 |
+
alignment_phase_sigma: 2.0
|
| 135 |
+
alignment_obj_window_frames: null
|
| 136 |
+
alignment_stop_gradient: false
|
| 137 |
+
competitive_read_groups: []
|
| 138 |
+
attn_diversity_weight: 0.0
|
| 139 |
+
attn_entropy_weight: 0.0
|
| 140 |
+
attn_shaping_groups: []
|
| 141 |
+
aux_losses:
|
| 142 |
+
- type: smoothness
|
| 143 |
+
weight: 0.05
|
| 144 |
+
kwargs: {}
|
| 145 |
+
loss:
|
| 146 |
+
zero_padded_noise: true
|
| 147 |
+
mask_padded_dims: true
|
| 148 |
+
first_action_weight: 1.0
|
| 149 |
+
direction_commit_weight: 0.0
|
| 150 |
+
direction_commit_tau: 0.15
|
| 151 |
+
direction_commit_mode: hard
|
| 152 |
+
gripper_bce_weight: 0.0
|
| 153 |
+
gripper_bce_logit_scale: 4.0
|
| 154 |
+
task_balance_alpha: 0.25
|
| 155 |
+
task_balance_mode: excess
|
| 156 |
+
task_balance_ema: 0.99
|
| 157 |
+
task_balance_clip: 4.0
|
| 158 |
+
phase_transition_weight: 3.0
|
| 159 |
+
phase_transition_radius: 2
|
| 160 |
+
aux_exclude_contact_window: false
|
| 161 |
+
anchor:
|
| 162 |
+
enabled: false
|
| 163 |
+
layers:
|
| 164 |
+
- 7
|
| 165 |
+
- 8
|
| 166 |
+
- 9
|
| 167 |
+
- 10
|
| 168 |
+
- 11
|
| 169 |
+
span: lang
|
| 170 |
+
weight: 0.05
|
| 171 |
+
embodiment:
|
| 172 |
+
name: aloha_agilex
|
| 173 |
+
embodiment_id: 0
|
| 174 |
+
action_names:
|
| 175 |
+
- left_joint_0
|
| 176 |
+
- left_joint_1
|
| 177 |
+
- left_joint_2
|
| 178 |
+
- left_joint_3
|
| 179 |
+
- left_joint_4
|
| 180 |
+
- left_joint_5
|
| 181 |
+
- left_gripper
|
| 182 |
+
- right_joint_0
|
| 183 |
+
- right_joint_1
|
| 184 |
+
- right_joint_2
|
| 185 |
+
- right_joint_3
|
| 186 |
+
- right_joint_4
|
| 187 |
+
- right_joint_5
|
| 188 |
+
- right_gripper
|
| 189 |
+
xyz_dims: []
|
| 190 |
+
rot_dims: []
|
| 191 |
+
gripper_dim: null
|
| 192 |
+
relative_action_dims:
|
| 193 |
+
- 0
|
| 194 |
+
- 1
|
| 195 |
+
- 2
|
| 196 |
+
- 3
|
| 197 |
+
- 4
|
| 198 |
+
- 5
|
| 199 |
+
- 7
|
| 200 |
+
- 8
|
| 201 |
+
- 9
|
| 202 |
+
- 10
|
| 203 |
+
- 11
|
| 204 |
+
- 12
|
| 205 |
+
gripper_policy_open_value: 1.0
|
| 206 |
+
gripper_policy_close_value: 0.0
|
| 207 |
+
gripper_env_open_value: -1.0
|
| 208 |
+
gripper_env_close_value: 1.0
|
| 209 |
+
eval:
|
| 210 |
+
benchmark: libero
|
| 211 |
+
suite: libero_spatial
|
| 212 |
+
n_episodes: 10
|
| 213 |
+
seed: 42
|
| 214 |
+
max_steps: 280
|
| 215 |
+
render_mode: rgb_array
|
| 216 |
+
train: null
|
| 217 |
+
run_dir: runs/robotwin_tinyvl_augS_wd3e-3_rel/checkpoint_last/eval
|
| 218 |
+
experiment_name: BQ_robotwin_tinyvl_augS_wd3e-3_rel_step0080000
|
config.json
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint": "",
|
| 3 |
+
"device": "cuda:0",
|
| 4 |
+
"init_mode": "scratch",
|
| 5 |
+
"base_checkpoint": null,
|
| 6 |
+
"state_dim": 14,
|
| 7 |
+
"action_dim": 14,
|
| 8 |
+
"n_cameras": 3,
|
| 9 |
+
"norm_stats_checkpoint": null,
|
| 10 |
+
"n_embodiments": 1,
|
| 11 |
+
"max_state_dim": 32,
|
| 12 |
+
"max_action_dim": 32,
|
| 13 |
+
"prefix_format": "raw",
|
| 14 |
+
"model": {
|
| 15 |
+
"type": "bridgequant_async_vla"
|
| 16 |
+
},
|
| 17 |
+
"backbone": {
|
| 18 |
+
"type": "tiny_vl",
|
| 19 |
+
"model_name": "prajjwal1/bert-mini",
|
| 20 |
+
"num_layers": 4,
|
| 21 |
+
"train_mode": "finetune",
|
| 22 |
+
"language_first": false,
|
| 23 |
+
"kwargs": {
|
| 24 |
+
"load_weights": true,
|
| 25 |
+
"vision_model": "facebook/dinov3-vits16-pretrain-lvd1689m",
|
| 26 |
+
"hidden_size": 512,
|
| 27 |
+
"image_size": 256,
|
| 28 |
+
"n_cameras": 3,
|
| 29 |
+
"num_heads": 8,
|
| 30 |
+
"mlp_ratio": 4.0,
|
| 31 |
+
"mask_mode": "m1",
|
| 32 |
+
"freeze_text": true
|
| 33 |
+
}
|
| 34 |
+
},
|
| 35 |
+
"expert": {
|
| 36 |
+
"type": "bridgequantasyncvla_expert",
|
| 37 |
+
"hidden_size": 512,
|
| 38 |
+
"depth": 6,
|
| 39 |
+
"num_heads": 8,
|
| 40 |
+
"mlp_ratio": 4.0,
|
| 41 |
+
"moh": {
|
| 42 |
+
"enabled": false,
|
| 43 |
+
"horizons": [],
|
| 44 |
+
"stride": 0,
|
| 45 |
+
"gate_temperature": 1.0,
|
| 46 |
+
"ind_weight": 1.0,
|
| 47 |
+
"bal_weight": 0.001,
|
| 48 |
+
"ind_reduction": "sum"
|
| 49 |
+
},
|
| 50 |
+
"kwargs": {
|
| 51 |
+
"chunk_index_features": true
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
"action_head": {
|
| 55 |
+
"type": "bridge_flow_matching",
|
| 56 |
+
"action_horizon": 16,
|
| 57 |
+
"n_action_steps": 8,
|
| 58 |
+
"num_steps": 1,
|
| 59 |
+
"gripper_bottleneck": false,
|
| 60 |
+
"bottleneck_residual_lambda": 0.01,
|
| 61 |
+
"bottleneck_residual_init_alpha": 0.01,
|
| 62 |
+
"kwargs": {}
|
| 63 |
+
},
|
| 64 |
+
"proprio": {
|
| 65 |
+
"proprio_to_vlm": false,
|
| 66 |
+
"proprio_to_policy": true,
|
| 67 |
+
"state_dropout": 0.0,
|
| 68 |
+
"cycle_count_in_state": false,
|
| 69 |
+
"kwargs": {}
|
| 70 |
+
},
|
| 71 |
+
"temporal": {
|
| 72 |
+
"n_obs_steps": 1,
|
| 73 |
+
"enabled": false,
|
| 74 |
+
"type": "retnet",
|
| 75 |
+
"window_size": 1,
|
| 76 |
+
"context_refresh_every": 8,
|
| 77 |
+
"context_validity": true,
|
| 78 |
+
"memory_age_aug": 0.0,
|
| 79 |
+
"retention_chunk_size": 0,
|
| 80 |
+
"decay_min": 0.03,
|
| 81 |
+
"decay_max": 1.0,
|
| 82 |
+
"burnin_steps": 0,
|
| 83 |
+
"history_noise_std": 0.0,
|
| 84 |
+
"history_noise_rho": 0.0,
|
| 85 |
+
"history_source": "executed_action",
|
| 86 |
+
"context_refresh_choices": null,
|
| 87 |
+
"drift_loss_weight": 0.0,
|
| 88 |
+
"reinjection_mode": "additive",
|
| 89 |
+
"refresh_proprio_delta_threshold": null,
|
| 90 |
+
"refresh_action_magnitude_threshold": null,
|
| 91 |
+
"refresh_frame_diff_threshold": null,
|
| 92 |
+
"refresh_instruction_change": true,
|
| 93 |
+
"d_model": 256,
|
| 94 |
+
"num_heads": 4,
|
| 95 |
+
"control_dt": 0.0667,
|
| 96 |
+
"kwargs": {},
|
| 97 |
+
"stream_units": 1,
|
| 98 |
+
"stream_stride": 5,
|
| 99 |
+
"stream_stride_jitter": 2
|
| 100 |
+
},
|
| 101 |
+
"bridge": {
|
| 102 |
+
"enabled": true,
|
| 103 |
+
"num_queries": 96,
|
| 104 |
+
"dim": 512,
|
| 105 |
+
"tap_layers": [
|
| 106 |
+
0,
|
| 107 |
+
1,
|
| 108 |
+
2,
|
| 109 |
+
3
|
| 110 |
+
],
|
| 111 |
+
"rank": 256,
|
| 112 |
+
"min_grounding_confidence": null,
|
| 113 |
+
"gate_init": 0.3,
|
| 114 |
+
"z_bridge_pos_embed": true,
|
| 115 |
+
"zero_z_bridge": false,
|
| 116 |
+
"kwargs": {}
|
| 117 |
+
},
|
| 118 |
+
"roles": {
|
| 119 |
+
"entity": 24,
|
| 120 |
+
"goal": 24,
|
| 121 |
+
"embodiment": 24,
|
| 122 |
+
"progress": 12,
|
| 123 |
+
"global_context": 12,
|
| 124 |
+
"alignment": 0,
|
| 125 |
+
"alignment_asymmetric_injection": false,
|
| 126 |
+
"alignment_asymmetric_scale": 1.0,
|
| 127 |
+
"plan_film": false,
|
| 128 |
+
"plan_film_scale": 1.0,
|
| 129 |
+
"plan_film_state": false,
|
| 130 |
+
"lang_kv_to_expert": false,
|
| 131 |
+
"mem_kv_to_expert": false,
|
| 132 |
+
"masked_groups": [],
|
| 133 |
+
"routing": null,
|
| 134 |
+
"routing_rows": null,
|
| 135 |
+
"spatial_binding_weight": 0.0,
|
| 136 |
+
"spatial_binding_hidden": 256,
|
| 137 |
+
"spatial_binding_huber_delta": null,
|
| 138 |
+
"alignment_offset_weight": 0.0,
|
| 139 |
+
"alignment_source_group": null,
|
| 140 |
+
"alignment_margin_weight": 0.0,
|
| 141 |
+
"alignment_margin_epsilon_cm": 2.5,
|
| 142 |
+
"alignment_huber_delta": 0.1,
|
| 143 |
+
"alignment_hidden": 256,
|
| 144 |
+
"alignment_phase_beta": 2.0,
|
| 145 |
+
"alignment_phase_sigma": 2.0,
|
| 146 |
+
"alignment_obj_window_frames": null,
|
| 147 |
+
"alignment_stop_gradient": false,
|
| 148 |
+
"competitive_read_groups": [],
|
| 149 |
+
"attn_diversity_weight": 0.0,
|
| 150 |
+
"attn_entropy_weight": 0.0,
|
| 151 |
+
"attn_shaping_groups": []
|
| 152 |
+
},
|
| 153 |
+
"aux_losses": [
|
| 154 |
+
{
|
| 155 |
+
"type": "smoothness",
|
| 156 |
+
"weight": 0.05,
|
| 157 |
+
"kwargs": {}
|
| 158 |
+
}
|
| 159 |
+
],
|
| 160 |
+
"loss": {
|
| 161 |
+
"zero_padded_noise": true,
|
| 162 |
+
"mask_padded_dims": true,
|
| 163 |
+
"first_action_weight": 1.0,
|
| 164 |
+
"direction_commit_weight": 0.0,
|
| 165 |
+
"direction_commit_tau": 0.15,
|
| 166 |
+
"direction_commit_mode": "hard",
|
| 167 |
+
"gripper_bce_weight": 0.0,
|
| 168 |
+
"gripper_bce_logit_scale": 4.0,
|
| 169 |
+
"task_balance_alpha": 0.25,
|
| 170 |
+
"task_balance_mode": "excess",
|
| 171 |
+
"task_balance_ema": 0.99,
|
| 172 |
+
"task_balance_clip": 4.0,
|
| 173 |
+
"phase_transition_weight": 3.0,
|
| 174 |
+
"phase_transition_radius": 2,
|
| 175 |
+
"aux_exclude_contact_window": false
|
| 176 |
+
},
|
| 177 |
+
"anchor": {
|
| 178 |
+
"enabled": false,
|
| 179 |
+
"layers": [
|
| 180 |
+
7,
|
| 181 |
+
8,
|
| 182 |
+
9,
|
| 183 |
+
10,
|
| 184 |
+
11
|
| 185 |
+
],
|
| 186 |
+
"span": "lang",
|
| 187 |
+
"weight": 0.05
|
| 188 |
+
},
|
| 189 |
+
"embodiment": {
|
| 190 |
+
"name": "aloha_agilex",
|
| 191 |
+
"embodiment_id": 0,
|
| 192 |
+
"action_names": [
|
| 193 |
+
"left_joint_0",
|
| 194 |
+
"left_joint_1",
|
| 195 |
+
"left_joint_2",
|
| 196 |
+
"left_joint_3",
|
| 197 |
+
"left_joint_4",
|
| 198 |
+
"left_joint_5",
|
| 199 |
+
"left_gripper",
|
| 200 |
+
"right_joint_0",
|
| 201 |
+
"right_joint_1",
|
| 202 |
+
"right_joint_2",
|
| 203 |
+
"right_joint_3",
|
| 204 |
+
"right_joint_4",
|
| 205 |
+
"right_joint_5",
|
| 206 |
+
"right_gripper"
|
| 207 |
+
],
|
| 208 |
+
"xyz_dims": [],
|
| 209 |
+
"rot_dims": [],
|
| 210 |
+
"gripper_dim": null,
|
| 211 |
+
"relative_action_dims": [
|
| 212 |
+
0,
|
| 213 |
+
1,
|
| 214 |
+
2,
|
| 215 |
+
3,
|
| 216 |
+
4,
|
| 217 |
+
5,
|
| 218 |
+
7,
|
| 219 |
+
8,
|
| 220 |
+
9,
|
| 221 |
+
10,
|
| 222 |
+
11,
|
| 223 |
+
12
|
| 224 |
+
],
|
| 225 |
+
"gripper_policy_open_value": 1.0,
|
| 226 |
+
"gripper_policy_close_value": 0.0,
|
| 227 |
+
"gripper_env_open_value": -1.0,
|
| 228 |
+
"gripper_env_close_value": 1.0
|
| 229 |
+
},
|
| 230 |
+
"model_type": "bridgequant_async_vla",
|
| 231 |
+
"bridgequant_vla_format_version": 1,
|
| 232 |
+
"step": 80000
|
| 233 |
+
}
|
metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata_version": 3,
|
| 3 |
+
"step": 80000,
|
| 4 |
+
"architecture": "BridgeQuantAsyncVLA",
|
| 5 |
+
"format": "safetensors",
|
| 6 |
+
"parameters": {
|
| 7 |
+
"total": 76095659,
|
| 8 |
+
"trainable": 64990891,
|
| 9 |
+
"frozen": 11104768,
|
| 10 |
+
"backbone": 46050688,
|
| 11 |
+
"connector": 4226309,
|
| 12 |
+
"action_head": 25818662,
|
| 13 |
+
"action_head/expert": 25802246,
|
| 14 |
+
"action_head/decoder": 16416
|
| 15 |
+
},
|
| 16 |
+
"backbone_type": "tiny_vl",
|
| 17 |
+
"expert_type": "bridgequantasyncvla_expert",
|
| 18 |
+
"action_head_type": "bridge_flow_matching",
|
| 19 |
+
"action_dim": 14,
|
| 20 |
+
"state_dim": 14,
|
| 21 |
+
"n_cameras": 3
|
| 22 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f14ea4bf8b1619d280efd6f80d1f9f7187b7dab373435526435a06e59cf77443
|
| 3 |
+
size 260017996
|
norm_stats.json
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"observation.state.mean": [
|
| 3 |
+
-0.21669170260429382,
|
| 4 |
+
1.089146614074707,
|
| 5 |
+
0.7939556241035461,
|
| 6 |
+
-0.3320826590061188,
|
| 7 |
+
0.05963357537984848,
|
| 8 |
+
-0.04071592167019844,
|
| 9 |
+
0.6729960441589355,
|
| 10 |
+
0.2336694598197937,
|
| 11 |
+
1.105362892150879,
|
| 12 |
+
0.8215522170066833,
|
| 13 |
+
-0.339979887008667,
|
| 14 |
+
-0.015037603676319122,
|
| 15 |
+
0.01934969052672386,
|
| 16 |
+
0.6736600995063782
|
| 17 |
+
],
|
| 18 |
+
"observation.state.std": [
|
| 19 |
+
0.2924947738647461,
|
| 20 |
+
0.9922713041305542,
|
| 21 |
+
0.7804890871047974,
|
| 22 |
+
0.6718838214874268,
|
| 23 |
+
0.24959948658943176,
|
| 24 |
+
0.5415620803833008,
|
| 25 |
+
0.4522083103656769,
|
| 26 |
+
0.31203868985176086,
|
| 27 |
+
1.0176671743392944,
|
| 28 |
+
0.805407702922821,
|
| 29 |
+
0.7186060547828674,
|
| 30 |
+
0.26038840413093567,
|
| 31 |
+
0.5929632186889648,
|
| 32 |
+
0.4514675438404083
|
| 33 |
+
],
|
| 34 |
+
"action.mean": [
|
| 35 |
+
-0.2176886647939682,
|
| 36 |
+
1.0937432050704956,
|
| 37 |
+
0.7973873019218445,
|
| 38 |
+
-0.3334546387195587,
|
| 39 |
+
0.05996466800570488,
|
| 40 |
+
-0.041341494768857956,
|
| 41 |
+
0.6717881560325623,
|
| 42 |
+
0.23512062430381775,
|
| 43 |
+
1.1110566854476929,
|
| 44 |
+
0.8257458209991455,
|
| 45 |
+
-0.34155911207199097,
|
| 46 |
+
-0.015165342949330807,
|
| 47 |
+
0.019972549751400948,
|
| 48 |
+
0.6724995970726013
|
| 49 |
+
],
|
| 50 |
+
"action.std": [
|
| 51 |
+
0.29288938641548157,
|
| 52 |
+
0.9918171763420105,
|
| 53 |
+
0.7806346416473389,
|
| 54 |
+
0.6733620762825012,
|
| 55 |
+
0.25023597478866577,
|
| 56 |
+
0.5435318350791931,
|
| 57 |
+
0.45266854763031006,
|
| 58 |
+
0.312542200088501,
|
| 59 |
+
1.0173609256744385,
|
| 60 |
+
0.8055886626243591,
|
| 61 |
+
0.7205750942230225,
|
| 62 |
+
0.26136842370033264,
|
| 63 |
+
0.5955495238304138,
|
| 64 |
+
0.4519122540950775
|
| 65 |
+
]
|
| 66 |
+
}
|
train_config.yaml
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# aug_strong + weight_decay 3e-3: the midpoint of the wd pair. wd1e-3 gave the best mean
|
| 2 |
+
# (0.963: goal 0.980, spatial 0.972 with t07 42/50) and wd1e-2 the best long (0.950) but
|
| 3 |
+
# collapsed spatial t07 to 6/50. Spatial t07 is the dose-sensitive task; this arm locates
|
| 4 |
+
# where between the two doses it turns.
|
| 5 |
+
|
| 6 |
+
policy:
|
| 7 |
+
checkpoint: "" # path to eval checkpoint bundle ("" = fresh)
|
| 8 |
+
device: "cuda:0"
|
| 9 |
+
init_mode: "scratch" # scratch | finetune (finetune needs base_checkpoint)
|
| 10 |
+
base_checkpoint: null # source weights when init_mode=finetune
|
| 11 |
+
state_dim: 14 # proprio dim (EE pos+ori+gripper for LIBERO)
|
| 12 |
+
action_dim: 14 # x y z roll pitch yaw gripper
|
| 13 |
+
n_cameras: 3 # agentview + wrist; role slots split across cameras
|
| 14 |
+
n_embodiments: 1 # embodiment-conditioned projector bank size
|
| 15 |
+
max_state_dim: 32 # max-dimension + mask interface (padded state width)
|
| 16 |
+
max_action_dim: 32 # padded action width shared by all embodiments
|
| 17 |
+
norm_stats_checkpoint: null # borrow norm stats from another checkpoint (null = own)
|
| 18 |
+
|
| 19 |
+
model:
|
| 20 |
+
type: "bridgequant_async_vla" # registry key (models/registry.py)
|
| 21 |
+
|
| 22 |
+
backbone:
|
| 23 |
+
type: "tiny_vl" # TinyBridge trunk: ViT-S/16 + BERT-mini (frozen) + 4-layer fusion, M1 mask
|
| 24 |
+
model_name: "prajjwal1/bert-mini" # TEXT encoder id = tokenizer source (English only, <=32 tokens)
|
| 25 |
+
num_layers: 4 # fusion-trunk depth (tap_layers index into these)
|
| 26 |
+
train_mode: "finetune" # ViT trainable; text frozen regardless; fusion/proj always train
|
| 27 |
+
kwargs:
|
| 28 |
+
load_weights: true # DINOv3-S + BERT-mini pretrained weights (false = random init, smoke only)
|
| 29 |
+
vision_model: "facebook/dinov3-vits16-pretrain-lvd1689m"
|
| 30 |
+
hidden_size: 512 # = bridge.dim, teacher-compatible width (warm-start expert/head possible)
|
| 31 |
+
image_size: 256 # 16x16 = 256 tokens per camera
|
| 32 |
+
n_cameras: 3
|
| 33 |
+
num_heads: 8
|
| 34 |
+
mlp_ratio: 4.0
|
| 35 |
+
mask_mode: "m1" # "bidirectional" = TurboVLA-style ablation
|
| 36 |
+
freeze_text: true
|
| 37 |
+
|
| 38 |
+
expert:
|
| 39 |
+
type: "bridgequantasyncvla_expert" # ACTION_EXPERT_REGISTRY key
|
| 40 |
+
hidden_size: 512 # 0 = derive from backbone hidden
|
| 41 |
+
depth: 6
|
| 42 |
+
num_heads: 8 # 0 = auto (hidden // 64)
|
| 43 |
+
mlp_ratio: 4.0
|
| 44 |
+
kwargs:
|
| 45 |
+
chunk_index_features: true # Fourier(h) + progress h/chunk on the injection queries
|
| 46 |
+
|
| 47 |
+
action_head:
|
| 48 |
+
type: "bridge_flow_matching" # bridge_flow_matching | regression | classification
|
| 49 |
+
action_horizon: 16 # TRAINED prediction horizon
|
| 50 |
+
n_action_steps: 8 # eval/runtime reuse default ONLY — not a training objective;
|
| 51 |
+
# tune at eval (1/4/8/16) via --n-action-steps
|
| 52 |
+
num_steps: 1 # one-step Euler deployment (validated operating point)
|
| 53 |
+
kwargs: {}
|
| 54 |
+
|
| 55 |
+
proprio:
|
| 56 |
+
# Vision-forced: no state content in the expert path (the B2 lesson — the
|
| 57 |
+
# proprio shortcut starves visual grounding of gradient pressure). The role
|
| 58 |
+
# reader keeps its state channel: it only re-weights attention keys.
|
| 59 |
+
proprio_to_vlm: false # matches the v5 winner — this rung IS the restored baseline
|
| 60 |
+
proprio_to_policy: true
|
| 61 |
+
state_dropout: 0.0 # no training-time regularization on the state path
|
| 62 |
+
kwargs: {}
|
| 63 |
+
|
| 64 |
+
temporal:
|
| 65 |
+
n_obs_steps: 1 # observation history stays single-frame
|
| 66 |
+
enabled: false # LADDER RUNG: RetNet temporal bridge OFF (no_retnet variant)
|
| 67 |
+
type: retnet
|
| 68 |
+
window_size: 1 # no window sampling (temporal bridge disabled)
|
| 69 |
+
context_refresh_every: 8 # simulated deployment refresh pattern (age cycles 0..7)
|
| 70 |
+
context_validity: true # feed context validity as a per-tick temporal input
|
| 71 |
+
retention_chunk_size: 0 # 0 = full parallel form; raise for long windows
|
| 72 |
+
d_model: 256 # RetNet group-token width
|
| 73 |
+
num_heads: 4 # retention heads (multi-scale decay)
|
| 74 |
+
control_dt: 0.0667 # LIBERO is 10 fps — physical tick duration
|
| 75 |
+
kwargs: {}
|
| 76 |
+
|
| 77 |
+
bridge:
|
| 78 |
+
enabled: true
|
| 79 |
+
num_queries: 96 # AQ tokens (split across n_cameras)
|
| 80 |
+
dim: 512 # d_bridge (0 = backbone hidden)
|
| 81 |
+
tap_layers: [0, 1, 2, 3] # softmax fusion over all 4 fusion layers (same shape as the 4-tap teacher)
|
| 82 |
+
rank: 256 # compressed semantic memory width r (raised from 64: rank caps z_bridge content)
|
| 83 |
+
gate_init: 0.3 # tanh(0.3)≈0.29 — bridge wins over zero-output early
|
| 84 |
+
z_bridge_pos_embed: true # per-camera + per-slot codes for spatial routing
|
| 85 |
+
kwargs: {}
|
| 86 |
+
|
| 87 |
+
roles: # semantic role-query split (sums to bridge.num_queries)
|
| 88 |
+
entity: 24
|
| 89 |
+
goal: 24
|
| 90 |
+
embodiment: 24
|
| 91 |
+
progress: 12
|
| 92 |
+
global_context: 12
|
| 93 |
+
masked_groups: [] # zero a group's outputs (static shapes preserved)
|
| 94 |
+
|
| 95 |
+
aux_losses: [{type: smoothness, weight: 0.05}] # COMBO: from smooth rung (92.0%)
|
| 96 |
+
|
| 97 |
+
loss:
|
| 98 |
+
task_balance_alpha: 0.25 # exponent on (loss_k - floor_k); the delta that closed 4/4 on LFM2.5-VL
|
| 99 |
+
task_balance_mode: "excess"
|
| 100 |
+
task_balance_ema: 0.99
|
| 101 |
+
task_balance_clip: 4.0
|
| 102 |
+
zero_padded_noise: true # keep on: no Gaussian garbage in padded action dims
|
| 103 |
+
mask_padded_dims: true # keep on: padded dims never enter the loss
|
| 104 |
+
first_action_weight: 1.0 # do NOT enable unless evaluating a strict K=1 policy
|
| 105 |
+
# SINGLE-DELTA vs lfm25vl_base_40k: 3x loss weight on chunk steps within
|
| 106 |
+
# +-radius of a ground-truth gripper transition. Motivation (200-ep evidence,
|
| 107 |
+
# 2026-07-16): the failure mode of BOTH ladder checkpoints is a precision
|
| 108 |
+
# grasp-miss (lift15 ~ 0, close-on-air -> timeout) whose victim objects
|
| 109 |
+
# ROTATE with training stage (20k: bbq_sauce/cream_cheese; 40k: orange_juice/
|
| 110 |
+
# salad_dressing) - uniform chunk loss lets grasp-moment accuracy trade off
|
| 111 |
+
# across objects. K-independent (weights GT-transition steps, not step 0).
|
| 112 |
+
phase_transition_weight: 3.0
|
| 113 |
+
phase_transition_radius: 2
|
| 114 |
+
gripper_bce_weight: 0.0 # baseline 0.0; ablate 0.1 / 0.2 in follow-up runs
|
| 115 |
+
gripper_bce_logit_scale: 4.0
|
| 116 |
+
|
| 117 |
+
embodiment: # RoboTwin 2.0 Aloha-AgileX: 14 joint positions, two grippers (preset, configs/embodiment.py)
|
| 118 |
+
name: "aloha_agilex"
|
| 119 |
+
relative_action_dims: [0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12] # arm joints as offsets from the current state; grippers absolute
|
| 120 |
+
|
| 121 |
+
eval:
|
| 122 |
+
benchmark: "libero" # libero | libero_plus
|
| 123 |
+
suite: "libero_spatial" # MUST match train suite or SR=0
|
| 124 |
+
n_episodes: 20
|
| 125 |
+
seed: 42
|
| 126 |
+
max_steps: 220 # env steps per episode
|
| 127 |
+
execution_mode: "async" # mid-training eval mirrors deployment (four-clock async rollout); needs n_action_steps>1
|
| 128 |
+
render_mode: "rgb_array"
|
| 129 |
+
image_size: null # null = native env resolution
|
| 130 |
+
batch_size: 1 # envs stepped together per task
|
| 131 |
+
max_parallel_tasks: 1
|
| 132 |
+
policy_batch_size: null # null = batch_size
|
| 133 |
+
save_video: true # mid-training evals write per-episode rollout videos (with attention overlay panels) under run_dir/libero_object/eval_XXXXXXX/videos
|
| 134 |
+
render_segmentation: false # diagnostic: sim instance masks → grounding region-correctness (UNTESTED map; validate on your sim)
|
| 135 |
+
video_fps: 30
|
| 136 |
+
filter_categories: null # libero_plus only: perturbation categories
|
| 137 |
+
|
| 138 |
+
train:
|
| 139 |
+
# Data — exactly one of data_dir / dataset_repo_id is used (data_dir wins if set).
|
| 140 |
+
data_dirs: # joint over all four suites (40 tasks); one cache per dir
|
| 141 |
+
- "examples/RoboTwin/RoboTwin_lerobot_v30"
|
| 142 |
+
suite: "libero_spatial" # HDF5 only; ignored for LeRobot v3
|
| 143 |
+
max_demos_per_task: null # HDF5 only; null = all demos
|
| 144 |
+
camera_key: "agentview_rgb" # HDF5 only
|
| 145 |
+
fps: 15 # dataset FPS for delta_timestamps
|
| 146 |
+
extra_cameras: null # extra LeRobot camera keys to load
|
| 147 |
+
video_backend: "pyav" # RoboTwin AV1 videos: torchcodec rounds the last frame of a file out of range
|
| 148 |
+
video_tolerance_s: 0.07 # one frame at 15 fps
|
| 149 |
+
cache_dirs:
|
| 150 |
+
- "examples/RoboTwin/RoboTwin_lerobot_v30_cache"
|
| 151 |
+
|
| 152 |
+
# Optimisation
|
| 153 |
+
batch_size: 16
|
| 154 |
+
lr: 1.0e-4 # connector/head/fusion from scratch
|
| 155 |
+
vision_lr_mult: 0.5 # pretrained ViT-S at 5e-5
|
| 156 |
+
# risks catastrophic drift; 2e-5 starves the from-scratch
|
| 157 |
+
# connector/head. Compound with the unfreeze by necessity.
|
| 158 |
+
weight_decay: 3.0e-3
|
| 159 |
+
n_steps: 80000
|
| 160 |
+
warmup_steps: 2000
|
| 161 |
+
# AUG CONTROL @30k: GR00T-style photometric jitter (N1 defaults) + 95% random
|
| 162 |
+
# crop, same 30k schedule as the baseline family — isolates the augmentation
|
| 163 |
+
# effect at fixed length before betting on longer schedules.
|
| 164 |
+
image_aug: true
|
| 165 |
+
# LIGHT PHOTOMETRIC AUG (single delta vs aug30k). The N1-default jitter
|
| 166 |
+
# (saturation 0.5, hue 0.08) lifted the easy tasks to 133/140 — the campaign's
|
| 167 |
+
# best — but killed task 07 (2/20 vs a 7-15 band) in ALL FOUR aug runs, i.e.
|
| 168 |
+
# systematically, not by seed. Task 07 ("bowl on the ramekin") is the finest
|
| 169 |
+
# colour/texture discrimination in the suite, so heavy chroma distortion is the
|
| 170 |
+
# prime suspect. Halve saturation, cut hue ~3x, keep brightness/contrast and
|
| 171 |
+
# the crop: aiming to retain the easy-task gain without the chroma damage.
|
| 172 |
+
aug_color_jitter: {brightness: 0.5, contrast: 0.6, saturation: 0.4, hue: 0.08}
|
| 173 |
+
aug_random_crop_scale: 0.70
|
| 174 |
+
seed: 42 # GR00T-convention default; new-run standard from here on
|
| 175 |
+
grad_clip_norm: 10.0
|
| 176 |
+
gradient_accumulation_steps: 1
|
| 177 |
+
|
| 178 |
+
# Schedule
|
| 179 |
+
save_every: 5000 # 20 checkpoints over the run
|
| 180 |
+
log_every: 100
|
| 181 |
+
eval_every: 0 # screening standalone tren checkpoint (EGL hazard)
|
| 182 |
+
|
| 183 |
+
# Data loading
|
| 184 |
+
num_workers: 2
|
| 185 |
+
prefetch_factor: 2
|
| 186 |
+
|
| 187 |
+
# Normalization stats
|
| 188 |
+
compute_norm_stats: true
|
| 189 |
+
norm_stats_max_samples: null # null = full dataset / metadata stats
|
| 190 |
+
|
| 191 |
+
# Mixed precision / kernels
|
| 192 |
+
use_amp: true
|
| 193 |
+
amp_dtype: "bfloat16" # bfloat16 | float16
|
| 194 |
+
cudnn_benchmark: true
|
| 195 |
+
allow_tf32: true
|
| 196 |
+
compile_model: false # true ≈ +15% throughput once the run is stable
|
| 197 |
+
compile_mode: "reduce-overhead" # default | reduce-overhead | max-autotune
|
| 198 |
+
profile_steps: 0 # >0 = capture torch profiler trace for first N steps
|
| 199 |
+
|
| 200 |
+
# Weights & Biases
|
| 201 |
+
wandb_mode: "online" # online | offline | disabled
|
| 202 |
+
wandb_watch_model: false
|
| 203 |
+
wandb_log_attention_maps: true # attention panels + 3x3 forward-order grids to W&B
|
| 204 |
+
wandb_attention_log_every: 2000 # ~50 panels over 100k + milestones (5k/10k/20k/50k); lower for short runs
|
| 205 |
+
wandb_log_vlm_attention: true # frozen-trunk internal role-query routing panels
|
| 206 |
+
wandb_log_vlm_text_grounding: true # text->visual grounding proxy panels
|
| 207 |
+
wandb_project: "bridgequant-async"
|
| 208 |
+
wandb_entity: hothinhhung77
|
| 209 |
+
wandb_tags: ["bridgequant_vla", "lfm25vl", "libero_object", "scratch",
|
| 210 |
+
"experiments", "chunk16", "sub1b",
|
| 211 |
+
"dualrate", "role_queries", "mla_reader", "retnet", "maxdim_mask"]
|
| 212 |
+
wandb_notes: >
|
| 213 |
+
Dual-rate architecture run — role queries, grounding adapter, compressed
|
| 214 |
+
semantic memory (rank 256), MLA-inspired role reader, RetNet temporal bridge
|
| 215 |
+
(window 8, time-aware decay), one-step flow expert with
|
| 216 |
+
embodiment-conditioned projectors. Frozen LFM2.5-VL-450M backbone.
|
| 217 |
+
|
| 218 |
+
# Reviewer/ablation diagnostics — numerically inert (no change to the training path).
|
| 219 |
+
# All groups are wired → they produce W&B metrics (heavier ones run only every
|
| 220 |
+
# scalar_interval steps, under no_grad, so the hot path stays clean):
|
| 221 |
+
# params/* param+optimizer audit grad/* per-module grad norms
|
| 222 |
+
# roles/* role-slot specialization bridge/* direct-vs-refinement + gate
|
| 223 |
+
# grounding/* region-correctness (no-op until a target mask exists)
|
| 224 |
+
# geometry/* query-side causal deltas (state/age/geo re-aim the read)
|
| 225 |
+
# temporal/* γ spectrum + history/reset/parallel≡recurrent causal probes (window>1)
|
| 226 |
+
# flow/* velocity MSE + predicted-action scale
|
| 227 |
+
# async_trace per-replan provenance in eval_summary.json (eval only)
|
| 228 |
+
diagnostics:
|
| 229 |
+
enabled: true
|
| 230 |
+
log_to_wandb: true
|
| 231 |
+
scalar_interval: 100
|
| 232 |
+
grad_interval: 1000
|
| 233 |
+
param_audit_at_start: true
|
| 234 |
+
log_role_diagnostics: true # roles/*
|
| 235 |
+
log_bridge_diagnostics: true # bridge/*
|
| 236 |
+
log_grounding_visuals: true # grounding/* (needs a target mask)
|
| 237 |
+
log_geometry_diagnostics: true # geometry/* (causal re-forward, interval-gated)
|
| 238 |
+
log_temporal_diagnostics: true # temporal/* (γ always; heavy probes need window>1)
|
| 239 |
+
log_flow_diagnostics: true # flow/*
|
| 240 |
+
log_async_trace: true # async_trace (eval only)
|
| 241 |
+
|
| 242 |
+
run_dir: "runs/robotwin_tinyvl_augS_wd3e-3_rel"
|
| 243 |
+
# run_dir basename.
|
| 244 |
+
experiment_name: "spatial_lfm25vl_fullft_aug_strong_aug_strong_wd1e-3"
|