twanghcmut commited on
Commit
b05fc58
·
verified ·
1 Parent(s): 1f32b78

checkpoint bundle

Browse files
Files changed (7) hide show
  1. README.md +11 -0
  2. bridgequant_vla.yaml +218 -0
  3. config.json +233 -0
  4. metadata.json +22 -0
  5. model.safetensors +3 -0
  6. norm_stats.json +66 -0
  7. train_config.yaml +244 -0
README.md ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: other
3
+ tags: [robotics, vision-language-action, bridgequant]
4
+ ---
5
+ # bridgequant-robotwin-76m
6
+
7
+ BridgeQuant 76M trained jointly on three RoboTwin 2.0 tasks (beat_block_hammer, stack_blocks_two, place_empty_cup; 50 clean demos each; relative arm joints). Multi-task joint training reached 0% in the simulator with this budget; the single-task 7M (`bridgequant-robotwin-7m-hammer`) is the working RoboTwin checkpoint.
8
+
9
+ Files: `model.safetensors`, `norm_stats.json`, `bridgequant_vla.yaml` / `config.json` / `metadata.json` (the bundle `load_trainable_weights` expects), `train_config.yaml` (the experiment YAML; evaluation and deployment use the same YAML).
10
+
11
+ Code: https://github.com/VLAOpt/bridgequant-async
bridgequant_vla.yaml ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ policy:
2
+ checkpoint: runs/robotwin_tinyvl_augS_wd3e-3_rel/checkpoint_last
3
+ device: cuda:0
4
+ init_mode: scratch
5
+ base_checkpoint: null
6
+ state_dim: 14
7
+ action_dim: 14
8
+ n_cameras: 3
9
+ norm_stats_checkpoint: null
10
+ n_embodiments: 1
11
+ max_state_dim: 32
12
+ max_action_dim: 32
13
+ prefix_format: raw
14
+ model:
15
+ type: bridgequant_async_vla
16
+ backbone:
17
+ type: tiny_vl
18
+ model_name: prajjwal1/bert-mini
19
+ num_layers: 4
20
+ train_mode: finetune
21
+ language_first: false
22
+ kwargs:
23
+ load_weights: true
24
+ vision_model: facebook/dinov3-vits16-pretrain-lvd1689m
25
+ hidden_size: 512
26
+ image_size: 256
27
+ n_cameras: 3
28
+ num_heads: 8
29
+ mlp_ratio: 4.0
30
+ mask_mode: m1
31
+ freeze_text: true
32
+ expert:
33
+ type: bridgequantasyncvla_expert
34
+ hidden_size: 512
35
+ depth: 6
36
+ num_heads: 8
37
+ mlp_ratio: 4.0
38
+ moh:
39
+ enabled: false
40
+ horizons: []
41
+ stride: 0
42
+ gate_temperature: 1.0
43
+ ind_weight: 1.0
44
+ bal_weight: 0.001
45
+ ind_reduction: sum
46
+ kwargs:
47
+ chunk_index_features: true
48
+ action_head:
49
+ type: bridge_flow_matching
50
+ action_horizon: 16
51
+ n_action_steps: 8
52
+ num_steps: 1
53
+ gripper_bottleneck: false
54
+ bottleneck_residual_lambda: 0.01
55
+ bottleneck_residual_init_alpha: 0.01
56
+ kwargs: {}
57
+ proprio:
58
+ proprio_to_vlm: false
59
+ proprio_to_policy: true
60
+ state_dropout: 0.0
61
+ cycle_count_in_state: false
62
+ kwargs: {}
63
+ temporal:
64
+ n_obs_steps: 1
65
+ enabled: false
66
+ type: retnet
67
+ window_size: 1
68
+ context_refresh_every: 8
69
+ context_validity: true
70
+ memory_age_aug: 0.0
71
+ retention_chunk_size: 0
72
+ decay_min: 0.03
73
+ decay_max: 1.0
74
+ burnin_steps: 0
75
+ history_noise_std: 0.0
76
+ history_noise_rho: 0.0
77
+ history_source: executed_action
78
+ context_refresh_choices: null
79
+ drift_loss_weight: 0.0
80
+ reinjection_mode: additive
81
+ refresh_proprio_delta_threshold: null
82
+ refresh_action_magnitude_threshold: null
83
+ refresh_frame_diff_threshold: null
84
+ refresh_instruction_change: true
85
+ d_model: 256
86
+ num_heads: 4
87
+ control_dt: 0.0667
88
+ kwargs: {}
89
+ stream_units: 1
90
+ stream_stride: 5
91
+ stream_stride_jitter: 2
92
+ bridge:
93
+ enabled: true
94
+ num_queries: 96
95
+ dim: 512
96
+ tap_layers:
97
+ - 0
98
+ - 1
99
+ - 2
100
+ - 3
101
+ rank: 256
102
+ min_grounding_confidence: null
103
+ gate_init: 0.3
104
+ z_bridge_pos_embed: true
105
+ zero_z_bridge: false
106
+ kwargs: {}
107
+ roles:
108
+ entity: 24
109
+ goal: 24
110
+ embodiment: 24
111
+ progress: 12
112
+ global_context: 12
113
+ alignment: 0
114
+ alignment_asymmetric_injection: false
115
+ alignment_asymmetric_scale: 1.0
116
+ plan_film: false
117
+ plan_film_scale: 1.0
118
+ plan_film_state: false
119
+ lang_kv_to_expert: false
120
+ mem_kv_to_expert: false
121
+ masked_groups: []
122
+ routing: null
123
+ routing_rows: null
124
+ spatial_binding_weight: 0.0
125
+ spatial_binding_hidden: 256
126
+ spatial_binding_huber_delta: null
127
+ alignment_offset_weight: 0.0
128
+ alignment_source_group: null
129
+ alignment_margin_weight: 0.0
130
+ alignment_margin_epsilon_cm: 2.5
131
+ alignment_huber_delta: 0.1
132
+ alignment_hidden: 256
133
+ alignment_phase_beta: 2.0
134
+ alignment_phase_sigma: 2.0
135
+ alignment_obj_window_frames: null
136
+ alignment_stop_gradient: false
137
+ competitive_read_groups: []
138
+ attn_diversity_weight: 0.0
139
+ attn_entropy_weight: 0.0
140
+ attn_shaping_groups: []
141
+ aux_losses:
142
+ - type: smoothness
143
+ weight: 0.05
144
+ kwargs: {}
145
+ loss:
146
+ zero_padded_noise: true
147
+ mask_padded_dims: true
148
+ first_action_weight: 1.0
149
+ direction_commit_weight: 0.0
150
+ direction_commit_tau: 0.15
151
+ direction_commit_mode: hard
152
+ gripper_bce_weight: 0.0
153
+ gripper_bce_logit_scale: 4.0
154
+ task_balance_alpha: 0.25
155
+ task_balance_mode: excess
156
+ task_balance_ema: 0.99
157
+ task_balance_clip: 4.0
158
+ phase_transition_weight: 3.0
159
+ phase_transition_radius: 2
160
+ aux_exclude_contact_window: false
161
+ anchor:
162
+ enabled: false
163
+ layers:
164
+ - 7
165
+ - 8
166
+ - 9
167
+ - 10
168
+ - 11
169
+ span: lang
170
+ weight: 0.05
171
+ embodiment:
172
+ name: aloha_agilex
173
+ embodiment_id: 0
174
+ action_names:
175
+ - left_joint_0
176
+ - left_joint_1
177
+ - left_joint_2
178
+ - left_joint_3
179
+ - left_joint_4
180
+ - left_joint_5
181
+ - left_gripper
182
+ - right_joint_0
183
+ - right_joint_1
184
+ - right_joint_2
185
+ - right_joint_3
186
+ - right_joint_4
187
+ - right_joint_5
188
+ - right_gripper
189
+ xyz_dims: []
190
+ rot_dims: []
191
+ gripper_dim: null
192
+ relative_action_dims:
193
+ - 0
194
+ - 1
195
+ - 2
196
+ - 3
197
+ - 4
198
+ - 5
199
+ - 7
200
+ - 8
201
+ - 9
202
+ - 10
203
+ - 11
204
+ - 12
205
+ gripper_policy_open_value: 1.0
206
+ gripper_policy_close_value: 0.0
207
+ gripper_env_open_value: -1.0
208
+ gripper_env_close_value: 1.0
209
+ eval:
210
+ benchmark: libero
211
+ suite: libero_spatial
212
+ n_episodes: 10
213
+ seed: 42
214
+ max_steps: 280
215
+ render_mode: rgb_array
216
+ train: null
217
+ run_dir: runs/robotwin_tinyvl_augS_wd3e-3_rel/checkpoint_last/eval
218
+ experiment_name: BQ_robotwin_tinyvl_augS_wd3e-3_rel_step0080000
config.json ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "",
3
+ "device": "cuda:0",
4
+ "init_mode": "scratch",
5
+ "base_checkpoint": null,
6
+ "state_dim": 14,
7
+ "action_dim": 14,
8
+ "n_cameras": 3,
9
+ "norm_stats_checkpoint": null,
10
+ "n_embodiments": 1,
11
+ "max_state_dim": 32,
12
+ "max_action_dim": 32,
13
+ "prefix_format": "raw",
14
+ "model": {
15
+ "type": "bridgequant_async_vla"
16
+ },
17
+ "backbone": {
18
+ "type": "tiny_vl",
19
+ "model_name": "prajjwal1/bert-mini",
20
+ "num_layers": 4,
21
+ "train_mode": "finetune",
22
+ "language_first": false,
23
+ "kwargs": {
24
+ "load_weights": true,
25
+ "vision_model": "facebook/dinov3-vits16-pretrain-lvd1689m",
26
+ "hidden_size": 512,
27
+ "image_size": 256,
28
+ "n_cameras": 3,
29
+ "num_heads": 8,
30
+ "mlp_ratio": 4.0,
31
+ "mask_mode": "m1",
32
+ "freeze_text": true
33
+ }
34
+ },
35
+ "expert": {
36
+ "type": "bridgequantasyncvla_expert",
37
+ "hidden_size": 512,
38
+ "depth": 6,
39
+ "num_heads": 8,
40
+ "mlp_ratio": 4.0,
41
+ "moh": {
42
+ "enabled": false,
43
+ "horizons": [],
44
+ "stride": 0,
45
+ "gate_temperature": 1.0,
46
+ "ind_weight": 1.0,
47
+ "bal_weight": 0.001,
48
+ "ind_reduction": "sum"
49
+ },
50
+ "kwargs": {
51
+ "chunk_index_features": true
52
+ }
53
+ },
54
+ "action_head": {
55
+ "type": "bridge_flow_matching",
56
+ "action_horizon": 16,
57
+ "n_action_steps": 8,
58
+ "num_steps": 1,
59
+ "gripper_bottleneck": false,
60
+ "bottleneck_residual_lambda": 0.01,
61
+ "bottleneck_residual_init_alpha": 0.01,
62
+ "kwargs": {}
63
+ },
64
+ "proprio": {
65
+ "proprio_to_vlm": false,
66
+ "proprio_to_policy": true,
67
+ "state_dropout": 0.0,
68
+ "cycle_count_in_state": false,
69
+ "kwargs": {}
70
+ },
71
+ "temporal": {
72
+ "n_obs_steps": 1,
73
+ "enabled": false,
74
+ "type": "retnet",
75
+ "window_size": 1,
76
+ "context_refresh_every": 8,
77
+ "context_validity": true,
78
+ "memory_age_aug": 0.0,
79
+ "retention_chunk_size": 0,
80
+ "decay_min": 0.03,
81
+ "decay_max": 1.0,
82
+ "burnin_steps": 0,
83
+ "history_noise_std": 0.0,
84
+ "history_noise_rho": 0.0,
85
+ "history_source": "executed_action",
86
+ "context_refresh_choices": null,
87
+ "drift_loss_weight": 0.0,
88
+ "reinjection_mode": "additive",
89
+ "refresh_proprio_delta_threshold": null,
90
+ "refresh_action_magnitude_threshold": null,
91
+ "refresh_frame_diff_threshold": null,
92
+ "refresh_instruction_change": true,
93
+ "d_model": 256,
94
+ "num_heads": 4,
95
+ "control_dt": 0.0667,
96
+ "kwargs": {},
97
+ "stream_units": 1,
98
+ "stream_stride": 5,
99
+ "stream_stride_jitter": 2
100
+ },
101
+ "bridge": {
102
+ "enabled": true,
103
+ "num_queries": 96,
104
+ "dim": 512,
105
+ "tap_layers": [
106
+ 0,
107
+ 1,
108
+ 2,
109
+ 3
110
+ ],
111
+ "rank": 256,
112
+ "min_grounding_confidence": null,
113
+ "gate_init": 0.3,
114
+ "z_bridge_pos_embed": true,
115
+ "zero_z_bridge": false,
116
+ "kwargs": {}
117
+ },
118
+ "roles": {
119
+ "entity": 24,
120
+ "goal": 24,
121
+ "embodiment": 24,
122
+ "progress": 12,
123
+ "global_context": 12,
124
+ "alignment": 0,
125
+ "alignment_asymmetric_injection": false,
126
+ "alignment_asymmetric_scale": 1.0,
127
+ "plan_film": false,
128
+ "plan_film_scale": 1.0,
129
+ "plan_film_state": false,
130
+ "lang_kv_to_expert": false,
131
+ "mem_kv_to_expert": false,
132
+ "masked_groups": [],
133
+ "routing": null,
134
+ "routing_rows": null,
135
+ "spatial_binding_weight": 0.0,
136
+ "spatial_binding_hidden": 256,
137
+ "spatial_binding_huber_delta": null,
138
+ "alignment_offset_weight": 0.0,
139
+ "alignment_source_group": null,
140
+ "alignment_margin_weight": 0.0,
141
+ "alignment_margin_epsilon_cm": 2.5,
142
+ "alignment_huber_delta": 0.1,
143
+ "alignment_hidden": 256,
144
+ "alignment_phase_beta": 2.0,
145
+ "alignment_phase_sigma": 2.0,
146
+ "alignment_obj_window_frames": null,
147
+ "alignment_stop_gradient": false,
148
+ "competitive_read_groups": [],
149
+ "attn_diversity_weight": 0.0,
150
+ "attn_entropy_weight": 0.0,
151
+ "attn_shaping_groups": []
152
+ },
153
+ "aux_losses": [
154
+ {
155
+ "type": "smoothness",
156
+ "weight": 0.05,
157
+ "kwargs": {}
158
+ }
159
+ ],
160
+ "loss": {
161
+ "zero_padded_noise": true,
162
+ "mask_padded_dims": true,
163
+ "first_action_weight": 1.0,
164
+ "direction_commit_weight": 0.0,
165
+ "direction_commit_tau": 0.15,
166
+ "direction_commit_mode": "hard",
167
+ "gripper_bce_weight": 0.0,
168
+ "gripper_bce_logit_scale": 4.0,
169
+ "task_balance_alpha": 0.25,
170
+ "task_balance_mode": "excess",
171
+ "task_balance_ema": 0.99,
172
+ "task_balance_clip": 4.0,
173
+ "phase_transition_weight": 3.0,
174
+ "phase_transition_radius": 2,
175
+ "aux_exclude_contact_window": false
176
+ },
177
+ "anchor": {
178
+ "enabled": false,
179
+ "layers": [
180
+ 7,
181
+ 8,
182
+ 9,
183
+ 10,
184
+ 11
185
+ ],
186
+ "span": "lang",
187
+ "weight": 0.05
188
+ },
189
+ "embodiment": {
190
+ "name": "aloha_agilex",
191
+ "embodiment_id": 0,
192
+ "action_names": [
193
+ "left_joint_0",
194
+ "left_joint_1",
195
+ "left_joint_2",
196
+ "left_joint_3",
197
+ "left_joint_4",
198
+ "left_joint_5",
199
+ "left_gripper",
200
+ "right_joint_0",
201
+ "right_joint_1",
202
+ "right_joint_2",
203
+ "right_joint_3",
204
+ "right_joint_4",
205
+ "right_joint_5",
206
+ "right_gripper"
207
+ ],
208
+ "xyz_dims": [],
209
+ "rot_dims": [],
210
+ "gripper_dim": null,
211
+ "relative_action_dims": [
212
+ 0,
213
+ 1,
214
+ 2,
215
+ 3,
216
+ 4,
217
+ 5,
218
+ 7,
219
+ 8,
220
+ 9,
221
+ 10,
222
+ 11,
223
+ 12
224
+ ],
225
+ "gripper_policy_open_value": 1.0,
226
+ "gripper_policy_close_value": 0.0,
227
+ "gripper_env_open_value": -1.0,
228
+ "gripper_env_close_value": 1.0
229
+ },
230
+ "model_type": "bridgequant_async_vla",
231
+ "bridgequant_vla_format_version": 1,
232
+ "step": 80000
233
+ }
metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata_version": 3,
3
+ "step": 80000,
4
+ "architecture": "BridgeQuantAsyncVLA",
5
+ "format": "safetensors",
6
+ "parameters": {
7
+ "total": 76095659,
8
+ "trainable": 64990891,
9
+ "frozen": 11104768,
10
+ "backbone": 46050688,
11
+ "connector": 4226309,
12
+ "action_head": 25818662,
13
+ "action_head/expert": 25802246,
14
+ "action_head/decoder": 16416
15
+ },
16
+ "backbone_type": "tiny_vl",
17
+ "expert_type": "bridgequantasyncvla_expert",
18
+ "action_head_type": "bridge_flow_matching",
19
+ "action_dim": 14,
20
+ "state_dim": 14,
21
+ "n_cameras": 3
22
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f14ea4bf8b1619d280efd6f80d1f9f7187b7dab373435526435a06e59cf77443
3
+ size 260017996
norm_stats.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "observation.state.mean": [
3
+ -0.21669170260429382,
4
+ 1.089146614074707,
5
+ 0.7939556241035461,
6
+ -0.3320826590061188,
7
+ 0.05963357537984848,
8
+ -0.04071592167019844,
9
+ 0.6729960441589355,
10
+ 0.2336694598197937,
11
+ 1.105362892150879,
12
+ 0.8215522170066833,
13
+ -0.339979887008667,
14
+ -0.015037603676319122,
15
+ 0.01934969052672386,
16
+ 0.6736600995063782
17
+ ],
18
+ "observation.state.std": [
19
+ 0.2924947738647461,
20
+ 0.9922713041305542,
21
+ 0.7804890871047974,
22
+ 0.6718838214874268,
23
+ 0.24959948658943176,
24
+ 0.5415620803833008,
25
+ 0.4522083103656769,
26
+ 0.31203868985176086,
27
+ 1.0176671743392944,
28
+ 0.805407702922821,
29
+ 0.7186060547828674,
30
+ 0.26038840413093567,
31
+ 0.5929632186889648,
32
+ 0.4514675438404083
33
+ ],
34
+ "action.mean": [
35
+ -0.2176886647939682,
36
+ 1.0937432050704956,
37
+ 0.7973873019218445,
38
+ -0.3334546387195587,
39
+ 0.05996466800570488,
40
+ -0.041341494768857956,
41
+ 0.6717881560325623,
42
+ 0.23512062430381775,
43
+ 1.1110566854476929,
44
+ 0.8257458209991455,
45
+ -0.34155911207199097,
46
+ -0.015165342949330807,
47
+ 0.019972549751400948,
48
+ 0.6724995970726013
49
+ ],
50
+ "action.std": [
51
+ 0.29288938641548157,
52
+ 0.9918171763420105,
53
+ 0.7806346416473389,
54
+ 0.6733620762825012,
55
+ 0.25023597478866577,
56
+ 0.5435318350791931,
57
+ 0.45266854763031006,
58
+ 0.312542200088501,
59
+ 1.0173609256744385,
60
+ 0.8055886626243591,
61
+ 0.7205750942230225,
62
+ 0.26136842370033264,
63
+ 0.5955495238304138,
64
+ 0.4519122540950775
65
+ ]
66
+ }
train_config.yaml ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # aug_strong + weight_decay 3e-3: the midpoint of the wd pair. wd1e-3 gave the best mean
2
+ # (0.963: goal 0.980, spatial 0.972 with t07 42/50) and wd1e-2 the best long (0.950) but
3
+ # collapsed spatial t07 to 6/50. Spatial t07 is the dose-sensitive task; this arm locates
4
+ # where between the two doses it turns.
5
+
6
+ policy:
7
+ checkpoint: "" # path to eval checkpoint bundle ("" = fresh)
8
+ device: "cuda:0"
9
+ init_mode: "scratch" # scratch | finetune (finetune needs base_checkpoint)
10
+ base_checkpoint: null # source weights when init_mode=finetune
11
+ state_dim: 14 # proprio dim (EE pos+ori+gripper for LIBERO)
12
+ action_dim: 14 # x y z roll pitch yaw gripper
13
+ n_cameras: 3 # agentview + wrist; role slots split across cameras
14
+ n_embodiments: 1 # embodiment-conditioned projector bank size
15
+ max_state_dim: 32 # max-dimension + mask interface (padded state width)
16
+ max_action_dim: 32 # padded action width shared by all embodiments
17
+ norm_stats_checkpoint: null # borrow norm stats from another checkpoint (null = own)
18
+
19
+ model:
20
+ type: "bridgequant_async_vla" # registry key (models/registry.py)
21
+
22
+ backbone:
23
+ type: "tiny_vl" # TinyBridge trunk: ViT-S/16 + BERT-mini (frozen) + 4-layer fusion, M1 mask
24
+ model_name: "prajjwal1/bert-mini" # TEXT encoder id = tokenizer source (English only, <=32 tokens)
25
+ num_layers: 4 # fusion-trunk depth (tap_layers index into these)
26
+ train_mode: "finetune" # ViT trainable; text frozen regardless; fusion/proj always train
27
+ kwargs:
28
+ load_weights: true # DINOv3-S + BERT-mini pretrained weights (false = random init, smoke only)
29
+ vision_model: "facebook/dinov3-vits16-pretrain-lvd1689m"
30
+ hidden_size: 512 # = bridge.dim, teacher-compatible width (warm-start expert/head possible)
31
+ image_size: 256 # 16x16 = 256 tokens per camera
32
+ n_cameras: 3
33
+ num_heads: 8
34
+ mlp_ratio: 4.0
35
+ mask_mode: "m1" # "bidirectional" = TurboVLA-style ablation
36
+ freeze_text: true
37
+
38
+ expert:
39
+ type: "bridgequantasyncvla_expert" # ACTION_EXPERT_REGISTRY key
40
+ hidden_size: 512 # 0 = derive from backbone hidden
41
+ depth: 6
42
+ num_heads: 8 # 0 = auto (hidden // 64)
43
+ mlp_ratio: 4.0
44
+ kwargs:
45
+ chunk_index_features: true # Fourier(h) + progress h/chunk on the injection queries
46
+
47
+ action_head:
48
+ type: "bridge_flow_matching" # bridge_flow_matching | regression | classification
49
+ action_horizon: 16 # TRAINED prediction horizon
50
+ n_action_steps: 8 # eval/runtime reuse default ONLY — not a training objective;
51
+ # tune at eval (1/4/8/16) via --n-action-steps
52
+ num_steps: 1 # one-step Euler deployment (validated operating point)
53
+ kwargs: {}
54
+
55
+ proprio:
56
+ # Vision-forced: no state content in the expert path (the B2 lesson — the
57
+ # proprio shortcut starves visual grounding of gradient pressure). The role
58
+ # reader keeps its state channel: it only re-weights attention keys.
59
+ proprio_to_vlm: false # matches the v5 winner — this rung IS the restored baseline
60
+ proprio_to_policy: true
61
+ state_dropout: 0.0 # no training-time regularization on the state path
62
+ kwargs: {}
63
+
64
+ temporal:
65
+ n_obs_steps: 1 # observation history stays single-frame
66
+ enabled: false # LADDER RUNG: RetNet temporal bridge OFF (no_retnet variant)
67
+ type: retnet
68
+ window_size: 1 # no window sampling (temporal bridge disabled)
69
+ context_refresh_every: 8 # simulated deployment refresh pattern (age cycles 0..7)
70
+ context_validity: true # feed context validity as a per-tick temporal input
71
+ retention_chunk_size: 0 # 0 = full parallel form; raise for long windows
72
+ d_model: 256 # RetNet group-token width
73
+ num_heads: 4 # retention heads (multi-scale decay)
74
+ control_dt: 0.0667 # LIBERO is 10 fps — physical tick duration
75
+ kwargs: {}
76
+
77
+ bridge:
78
+ enabled: true
79
+ num_queries: 96 # AQ tokens (split across n_cameras)
80
+ dim: 512 # d_bridge (0 = backbone hidden)
81
+ tap_layers: [0, 1, 2, 3] # softmax fusion over all 4 fusion layers (same shape as the 4-tap teacher)
82
+ rank: 256 # compressed semantic memory width r (raised from 64: rank caps z_bridge content)
83
+ gate_init: 0.3 # tanh(0.3)≈0.29 — bridge wins over zero-output early
84
+ z_bridge_pos_embed: true # per-camera + per-slot codes for spatial routing
85
+ kwargs: {}
86
+
87
+ roles: # semantic role-query split (sums to bridge.num_queries)
88
+ entity: 24
89
+ goal: 24
90
+ embodiment: 24
91
+ progress: 12
92
+ global_context: 12
93
+ masked_groups: [] # zero a group's outputs (static shapes preserved)
94
+
95
+ aux_losses: [{type: smoothness, weight: 0.05}] # COMBO: from smooth rung (92.0%)
96
+
97
+ loss:
98
+ task_balance_alpha: 0.25 # exponent on (loss_k - floor_k); the delta that closed 4/4 on LFM2.5-VL
99
+ task_balance_mode: "excess"
100
+ task_balance_ema: 0.99
101
+ task_balance_clip: 4.0
102
+ zero_padded_noise: true # keep on: no Gaussian garbage in padded action dims
103
+ mask_padded_dims: true # keep on: padded dims never enter the loss
104
+ first_action_weight: 1.0 # do NOT enable unless evaluating a strict K=1 policy
105
+ # SINGLE-DELTA vs lfm25vl_base_40k: 3x loss weight on chunk steps within
106
+ # +-radius of a ground-truth gripper transition. Motivation (200-ep evidence,
107
+ # 2026-07-16): the failure mode of BOTH ladder checkpoints is a precision
108
+ # grasp-miss (lift15 ~ 0, close-on-air -> timeout) whose victim objects
109
+ # ROTATE with training stage (20k: bbq_sauce/cream_cheese; 40k: orange_juice/
110
+ # salad_dressing) - uniform chunk loss lets grasp-moment accuracy trade off
111
+ # across objects. K-independent (weights GT-transition steps, not step 0).
112
+ phase_transition_weight: 3.0
113
+ phase_transition_radius: 2
114
+ gripper_bce_weight: 0.0 # baseline 0.0; ablate 0.1 / 0.2 in follow-up runs
115
+ gripper_bce_logit_scale: 4.0
116
+
117
+ embodiment: # RoboTwin 2.0 Aloha-AgileX: 14 joint positions, two grippers (preset, configs/embodiment.py)
118
+ name: "aloha_agilex"
119
+ relative_action_dims: [0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12] # arm joints as offsets from the current state; grippers absolute
120
+
121
+ eval:
122
+ benchmark: "libero" # libero | libero_plus
123
+ suite: "libero_spatial" # MUST match train suite or SR=0
124
+ n_episodes: 20
125
+ seed: 42
126
+ max_steps: 220 # env steps per episode
127
+ execution_mode: "async" # mid-training eval mirrors deployment (four-clock async rollout); needs n_action_steps>1
128
+ render_mode: "rgb_array"
129
+ image_size: null # null = native env resolution
130
+ batch_size: 1 # envs stepped together per task
131
+ max_parallel_tasks: 1
132
+ policy_batch_size: null # null = batch_size
133
+ save_video: true # mid-training evals write per-episode rollout videos (with attention overlay panels) under run_dir/libero_object/eval_XXXXXXX/videos
134
+ render_segmentation: false # diagnostic: sim instance masks → grounding region-correctness (UNTESTED map; validate on your sim)
135
+ video_fps: 30
136
+ filter_categories: null # libero_plus only: perturbation categories
137
+
138
+ train:
139
+ # Data — exactly one of data_dir / dataset_repo_id is used (data_dir wins if set).
140
+ data_dirs: # joint over all four suites (40 tasks); one cache per dir
141
+ - "examples/RoboTwin/RoboTwin_lerobot_v30"
142
+ suite: "libero_spatial" # HDF5 only; ignored for LeRobot v3
143
+ max_demos_per_task: null # HDF5 only; null = all demos
144
+ camera_key: "agentview_rgb" # HDF5 only
145
+ fps: 15 # dataset FPS for delta_timestamps
146
+ extra_cameras: null # extra LeRobot camera keys to load
147
+ video_backend: "pyav" # RoboTwin AV1 videos: torchcodec rounds the last frame of a file out of range
148
+ video_tolerance_s: 0.07 # one frame at 15 fps
149
+ cache_dirs:
150
+ - "examples/RoboTwin/RoboTwin_lerobot_v30_cache"
151
+
152
+ # Optimisation
153
+ batch_size: 16
154
+ lr: 1.0e-4 # connector/head/fusion from scratch
155
+ vision_lr_mult: 0.5 # pretrained ViT-S at 5e-5
156
+ # risks catastrophic drift; 2e-5 starves the from-scratch
157
+ # connector/head. Compound with the unfreeze by necessity.
158
+ weight_decay: 3.0e-3
159
+ n_steps: 80000
160
+ warmup_steps: 2000
161
+ # AUG CONTROL @30k: GR00T-style photometric jitter (N1 defaults) + 95% random
162
+ # crop, same 30k schedule as the baseline family — isolates the augmentation
163
+ # effect at fixed length before betting on longer schedules.
164
+ image_aug: true
165
+ # LIGHT PHOTOMETRIC AUG (single delta vs aug30k). The N1-default jitter
166
+ # (saturation 0.5, hue 0.08) lifted the easy tasks to 133/140 — the campaign's
167
+ # best — but killed task 07 (2/20 vs a 7-15 band) in ALL FOUR aug runs, i.e.
168
+ # systematically, not by seed. Task 07 ("bowl on the ramekin") is the finest
169
+ # colour/texture discrimination in the suite, so heavy chroma distortion is the
170
+ # prime suspect. Halve saturation, cut hue ~3x, keep brightness/contrast and
171
+ # the crop: aiming to retain the easy-task gain without the chroma damage.
172
+ aug_color_jitter: {brightness: 0.5, contrast: 0.6, saturation: 0.4, hue: 0.08}
173
+ aug_random_crop_scale: 0.70
174
+ seed: 42 # GR00T-convention default; new-run standard from here on
175
+ grad_clip_norm: 10.0
176
+ gradient_accumulation_steps: 1
177
+
178
+ # Schedule
179
+ save_every: 5000 # 20 checkpoints over the run
180
+ log_every: 100
181
+ eval_every: 0 # screening standalone tren checkpoint (EGL hazard)
182
+
183
+ # Data loading
184
+ num_workers: 2
185
+ prefetch_factor: 2
186
+
187
+ # Normalization stats
188
+ compute_norm_stats: true
189
+ norm_stats_max_samples: null # null = full dataset / metadata stats
190
+
191
+ # Mixed precision / kernels
192
+ use_amp: true
193
+ amp_dtype: "bfloat16" # bfloat16 | float16
194
+ cudnn_benchmark: true
195
+ allow_tf32: true
196
+ compile_model: false # true ≈ +15% throughput once the run is stable
197
+ compile_mode: "reduce-overhead" # default | reduce-overhead | max-autotune
198
+ profile_steps: 0 # >0 = capture torch profiler trace for first N steps
199
+
200
+ # Weights & Biases
201
+ wandb_mode: "online" # online | offline | disabled
202
+ wandb_watch_model: false
203
+ wandb_log_attention_maps: true # attention panels + 3x3 forward-order grids to W&B
204
+ wandb_attention_log_every: 2000 # ~50 panels over 100k + milestones (5k/10k/20k/50k); lower for short runs
205
+ wandb_log_vlm_attention: true # frozen-trunk internal role-query routing panels
206
+ wandb_log_vlm_text_grounding: true # text->visual grounding proxy panels
207
+ wandb_project: "bridgequant-async"
208
+ wandb_entity: hothinhhung77
209
+ wandb_tags: ["bridgequant_vla", "lfm25vl", "libero_object", "scratch",
210
+ "experiments", "chunk16", "sub1b",
211
+ "dualrate", "role_queries", "mla_reader", "retnet", "maxdim_mask"]
212
+ wandb_notes: >
213
+ Dual-rate architecture run — role queries, grounding adapter, compressed
214
+ semantic memory (rank 256), MLA-inspired role reader, RetNet temporal bridge
215
+ (window 8, time-aware decay), one-step flow expert with
216
+ embodiment-conditioned projectors. Frozen LFM2.5-VL-450M backbone.
217
+
218
+ # Reviewer/ablation diagnostics — numerically inert (no change to the training path).
219
+ # All groups are wired → they produce W&B metrics (heavier ones run only every
220
+ # scalar_interval steps, under no_grad, so the hot path stays clean):
221
+ # params/* param+optimizer audit grad/* per-module grad norms
222
+ # roles/* role-slot specialization bridge/* direct-vs-refinement + gate
223
+ # grounding/* region-correctness (no-op until a target mask exists)
224
+ # geometry/* query-side causal deltas (state/age/geo re-aim the read)
225
+ # temporal/* γ spectrum + history/reset/parallel≡recurrent causal probes (window>1)
226
+ # flow/* velocity MSE + predicted-action scale
227
+ # async_trace per-replan provenance in eval_summary.json (eval only)
228
+ diagnostics:
229
+ enabled: true
230
+ log_to_wandb: true
231
+ scalar_interval: 100
232
+ grad_interval: 1000
233
+ param_audit_at_start: true
234
+ log_role_diagnostics: true # roles/*
235
+ log_bridge_diagnostics: true # bridge/*
236
+ log_grounding_visuals: true # grounding/* (needs a target mask)
237
+ log_geometry_diagnostics: true # geometry/* (causal re-forward, interval-gated)
238
+ log_temporal_diagnostics: true # temporal/* (γ always; heavy probes need window>1)
239
+ log_flow_diagnostics: true # flow/*
240
+ log_async_trace: true # async_trace (eval only)
241
+
242
+ run_dir: "runs/robotwin_tinyvl_augS_wd3e-3_rel"
243
+ # run_dir basename.
244
+ experiment_name: "spatial_lfm25vl_fullft_aug_strong_aug_strong_wd1e-3"