Hugging Face's logo

DynamicDatasets
/
BeastXVLARefinerV2_lemon_bowl_euler_single_scene

Safetensors
Model card Files
xet
 Community
jannick-st commited on
Commit
c9ed052
·
verified ·
1 Parent(s): 81bbde3

beast_xvla_refiner_v2, step 40000, joint train_mode=both

Browse files
Files changed (7) hide show
  1. config.json +144 -0
  2. model.safetensors +3 -0
  3. policy_postprocessor.json +32 -0
  4. policy_postprocessor_step_0_unnormalizer_processor.safetensors +3 -0
  5. policy_preprocessor.json +133 -0
  6. policy_preprocessor_step_9_normalizer_processor.safetensors +3 -0
  7. train_config.json +297 -0
config.json ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "beast_xvla_refiner_v2",
3
+ "n_obs_steps": 1,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 6
9
+ ]
10
+ },
11
+ "observation.environment_state": {
12
+ "type": "ENV",
13
+ "shape": [
14
+ 9
15
+ ]
16
+ },
17
+ "observation.images.image3": {
18
+ "type": "VISUAL",
19
+ "shape": [
20
+ 3,
21
+ 224,
22
+ 224
23
+ ]
24
+ },
25
+ "observation.images.image": {
26
+ "type": "VISUAL",
27
+ "shape": [
28
+ 3,
29
+ 224,
30
+ 224
31
+ ]
32
+ },
33
+ "observation.images.image2": {
34
+ "type": "VISUAL",
35
+ "shape": [
36
+ 3,
37
+ 224,
38
+ 224
39
+ ]
40
+ },
41
+ "observation.action_velocity": {
42
+ "type": "STATE",
43
+ "shape": [
44
+ 7
45
+ ]
46
+ }
47
+ },
48
+ "output_features": {
49
+ "action": {
50
+ "type": "ACTION",
51
+ "shape": [
52
+ 7
53
+ ]
54
+ }
55
+ },
56
+ "device": "cuda",
57
+ "use_amp": false,
58
+ "use_peft": false,
59
+ "push_to_hub": false,
60
+ "repo_id": null,
61
+ "private": null,
62
+ "tags": null,
63
+ "license": null,
64
+ "pretrained_path": null,
65
+ "fps": 25,
66
+ "normalization_mapping": {
67
+ "VISUAL": "IDENTITY",
68
+ "STATE": "IDENTITY",
69
+ "ACTION": "MEAN_STD"
70
+ },
71
+ "xvla_checkpoint": "/vol/coro/jstranghoener/lerobot_volume/huggingface/repos/xvla-base",
72
+ "num_flow_steps": 10,
73
+ "tokenizer_name": "facebook/bart-large",
74
+ "tokenizer_max_length": 50,
75
+ "tokenizer_padding_side": "right",
76
+ "pad_language_to": "max_length",
77
+ "domain_id": 0,
78
+ "xvla_max_action_dim": 20,
79
+ "resize_imgs_with_padding": [
80
+ 224,
81
+ 224
82
+ ],
83
+ "num_image_views": 3,
84
+ "dtype": "bfloat16",
85
+ "freeze_vision_encoder": true,
86
+ "freeze_language_encoder": false,
87
+ "train_policy_transformer": true,
88
+ "train_soft_prompts": true,
89
+ "n_action_steps": 12,
90
+ "chunk_size": 24,
91
+ "num_system2_basis": 12,
92
+ "num_system1_basis": 9,
93
+ "degree_p": 3,
94
+ "gripper_zero_order": true,
95
+ "gripper_dof": 1,
96
+ "sys2_w_min": -5.0,
97
+ "sys2_w_max": 5.0,
98
+ "sys1_w_min": -5.0,
99
+ "sys1_w_max": 5.0,
100
+ "spline_fit_regularization": 1e-05,
101
+ "refiner_hidden_size": 384,
102
+ "refiner_depth": 6,
103
+ "refiner_num_heads": 6,
104
+ "refiner_max_cond_len": 512,
105
+ "action_loss_weight": 1.0,
106
+ "pin_position": true,
107
+ "pin_velocity": true,
108
+ "condition_on_anchor": false,
109
+ "condition_on_anchor_velocity": true,
110
+ "history_len": 5,
111
+ "history_feature_dim": 128,
112
+ "history_time_frequencies_hz": [
113
+ 0.5,
114
+ 1.0,
115
+ 2.0,
116
+ 4.0,
117
+ 8.0
118
+ ],
119
+ "history_freeze_backbone": true,
120
+ "history_pretrained_backbone": true,
121
+ "history_dropout": 0.0,
122
+ "history_diff_weight": 0.001,
123
+ "blind_to_plan": false,
124
+ "predict_action_directly": false,
125
+ "vlm_grad_sys1": true,
126
+ "vlm_grad_sys2": true,
127
+ "train_mode": "both",
128
+ "inference_mode": "both",
129
+ "return_act_chunk": false,
130
+ "action_velocity_key": "observation.action_velocity",
131
+ "sys2_optimizer_lr": 2e-05,
132
+ "sys1_optimizer_lr": 0.0003,
133
+ "optimizer_betas": [
134
+ 0.9,
135
+ 0.95
136
+ ],
137
+ "optimizer_eps": 1e-08,
138
+ "optimizer_weight_decay": 0.0001,
139
+ "optimizer_grad_clip_norm": 1.0,
140
+ "scheduler_warmup_steps": 1000,
141
+ "scheduler_decay_steps": 40000,
142
+ "sys2_scheduler_decay_lr": 2.5e-06,
143
+ "sys1_scheduler_decay_lr": 1e-05
144
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b874b779c23da72323b13eac8cb4ba813a25c972397e754c09da85bfe94bd0df
3
+ size 1850525222
policy_postprocessor.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "unnormalizer_processor",
6
+ "config": {
7
+ "eps": 1e-08,
8
+ "features": {
9
+ "action": {
10
+ "type": "ACTION",
11
+ "shape": [
12
+ 7
13
+ ]
14
+ }
15
+ },
16
+ "norm_map": {
17
+ "VISUAL": "IDENTITY",
18
+ "STATE": "IDENTITY",
19
+ "ACTION": "MEAN_STD"
20
+ }
21
+ },
22
+ "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
23
+ },
24
+ {
25
+ "registry_name": "device_processor",
26
+ "config": {
27
+ "device": "cpu",
28
+ "float_dtype": null
29
+ }
30
+ }
31
+ ]
32
+ }
policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7a9e8469294193b3bea881f6ba32d29cafd5cbe88923c35a1cead8064fe8e51
3
+ size 8604
policy_preprocessor.json ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {
8
+ "observation.images.opst_cam": "observation.images.image",
9
+ "observation.images.side_cam": "observation.images.image2",
10
+ "observation.images.wrist_cam": "observation.images.image3"
11
+ }
12
+ }
13
+ },
14
+ {
15
+ "registry_name": "beast_refiner_add_batch_dimension",
16
+ "config": {}
17
+ },
18
+ {
19
+ "registry_name": "tokenizer_processor",
20
+ "config": {
21
+ "max_length": 50,
22
+ "task_key": "task",
23
+ "padding_side": "right",
24
+ "padding": "max_length",
25
+ "truncation": true,
26
+ "tokenizer_name": "facebook/bart-large"
27
+ }
28
+ },
29
+ {
30
+ "registry_name": "beast_xvla_refiner_v2_image_to_float",
31
+ "config": {}
32
+ },
33
+ {
34
+ "registry_name": "beast_xvla_refiner_v2_imagenet_normalize",
35
+ "config": {}
36
+ },
37
+ {
38
+ "registry_name": "beast_xvla_refiner_v2_add_domain_id",
39
+ "config": {}
40
+ },
41
+ {
42
+ "registry_name": "beast_xvla_refiner_v2_velocity_fallback",
43
+ "config": {
44
+ "velocity_key": "observation.action_velocity",
45
+ "action_dim": 7
46
+ }
47
+ },
48
+ {
49
+ "registry_name": "beast_xvla_refiner_v2_normalize_velocity",
50
+ "config": {
51
+ "velocity_key": "observation.action_velocity",
52
+ "action_scale": [
53
+ 0.1521461308002472,
54
+ 0.2062179297208786,
55
+ 0.12297115474939346,
56
+ 0.1427786946296692,
57
+ 0.2901972532272339,
58
+ 0.47654685378074646,
59
+ 0.9008533954620361
60
+ ]
61
+ }
62
+ },
63
+ {
64
+ "registry_name": "device_processor",
65
+ "config": {
66
+ "device": "cuda",
67
+ "float_dtype": null
68
+ }
69
+ },
70
+ {
71
+ "registry_name": "normalizer_processor",
72
+ "config": {
73
+ "eps": 1e-08,
74
+ "features": {
75
+ "observation.state": {
76
+ "type": "STATE",
77
+ "shape": [
78
+ 6
79
+ ]
80
+ },
81
+ "observation.environment_state": {
82
+ "type": "ENV",
83
+ "shape": [
84
+ 9
85
+ ]
86
+ },
87
+ "observation.images.image3": {
88
+ "type": "VISUAL",
89
+ "shape": [
90
+ 3,
91
+ 224,
92
+ 224
93
+ ]
94
+ },
95
+ "observation.images.image": {
96
+ "type": "VISUAL",
97
+ "shape": [
98
+ 3,
99
+ 224,
100
+ 224
101
+ ]
102
+ },
103
+ "observation.images.image2": {
104
+ "type": "VISUAL",
105
+ "shape": [
106
+ 3,
107
+ 224,
108
+ 224
109
+ ]
110
+ },
111
+ "observation.action_velocity": {
112
+ "type": "STATE",
113
+ "shape": [
114
+ 7
115
+ ]
116
+ },
117
+ "action": {
118
+ "type": "ACTION",
119
+ "shape": [
120
+ 7
121
+ ]
122
+ }
123
+ },
124
+ "norm_map": {
125
+ "VISUAL": "IDENTITY",
126
+ "STATE": "IDENTITY",
127
+ "ACTION": "MEAN_STD"
128
+ }
129
+ },
130
+ "state_file": "policy_preprocessor_step_9_normalizer_processor.safetensors"
131
+ }
132
+ ]
133
+ }
policy_preprocessor_step_9_normalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7a9e8469294193b3bea881f6ba32d29cafd5cbe88923c35a1cead8064fe8e51
3
+ size 8604
train_config.json ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "lemon_bowl_euler_single_scene",
4
+ "root": "/vol/coro/jstranghoener/lerobot_volume/data/dom/lemon_bowl_euler_single_scene",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "tfs": {
11
+ "brightness": {
12
+ "weight": 1.0,
13
+ "type": "ColorJitter",
14
+ "kwargs": {
15
+ "brightness": [
16
+ 0.8,
17
+ 1.2
18
+ ]
19
+ }
20
+ },
21
+ "contrast": {
22
+ "weight": 1.0,
23
+ "type": "ColorJitter",
24
+ "kwargs": {
25
+ "contrast": [
26
+ 0.8,
27
+ 1.2
28
+ ]
29
+ }
30
+ },
31
+ "saturation": {
32
+ "weight": 1.0,
33
+ "type": "ColorJitter",
34
+ "kwargs": {
35
+ "saturation": [
36
+ 0.5,
37
+ 1.5
38
+ ]
39
+ }
40
+ },
41
+ "hue": {
42
+ "weight": 1.0,
43
+ "type": "ColorJitter",
44
+ "kwargs": {
45
+ "hue": [
46
+ -0.05,
47
+ 0.05
48
+ ]
49
+ }
50
+ },
51
+ "sharpness": {
52
+ "weight": 1.0,
53
+ "type": "SharpnessJitter",
54
+ "kwargs": {
55
+ "sharpness": [
56
+ 0.5,
57
+ 1.5
58
+ ]
59
+ }
60
+ },
61
+ "affine": {
62
+ "weight": 1.0,
63
+ "type": "RandomAffine",
64
+ "kwargs": {
65
+ "degrees": [
66
+ -5.0,
67
+ 5.0
68
+ ],
69
+ "translate": [
70
+ 0.05,
71
+ 0.05
72
+ ]
73
+ }
74
+ }
75
+ }
76
+ },
77
+ "revision": null,
78
+ "use_imagenet_stats": true,
79
+ "video_backend": "pyav",
80
+ "streaming": false
81
+ },
82
+ "env": null,
83
+ "policy": {
84
+ "type": "beast_xvla_refiner_v2",
85
+ "n_obs_steps": 1,
86
+ "input_features": {
87
+ "observation.state": {
88
+ "type": "STATE",
89
+ "shape": [
90
+ 6
91
+ ]
92
+ },
93
+ "observation.environment_state": {
94
+ "type": "ENV",
95
+ "shape": [
96
+ 9
97
+ ]
98
+ },
99
+ "observation.images.image3": {
100
+ "type": "VISUAL",
101
+ "shape": [
102
+ 3,
103
+ 224,
104
+ 224
105
+ ]
106
+ },
107
+ "observation.images.image": {
108
+ "type": "VISUAL",
109
+ "shape": [
110
+ 3,
111
+ 224,
112
+ 224
113
+ ]
114
+ },
115
+ "observation.images.image2": {
116
+ "type": "VISUAL",
117
+ "shape": [
118
+ 3,
119
+ 224,
120
+ 224
121
+ ]
122
+ },
123
+ "observation.action_velocity": {
124
+ "type": "STATE",
125
+ "shape": [
126
+ 7
127
+ ]
128
+ }
129
+ },
130
+ "output_features": {
131
+ "action": {
132
+ "type": "ACTION",
133
+ "shape": [
134
+ 7
135
+ ]
136
+ }
137
+ },
138
+ "device": "cuda",
139
+ "use_amp": false,
140
+ "use_peft": false,
141
+ "push_to_hub": false,
142
+ "repo_id": null,
143
+ "private": null,
144
+ "tags": null,
145
+ "license": null,
146
+ "pretrained_path": null,
147
+ "fps": 25,
148
+ "normalization_mapping": {
149
+ "VISUAL": "IDENTITY",
150
+ "STATE": "IDENTITY",
151
+ "ACTION": "MEAN_STD"
152
+ },
153
+ "xvla_checkpoint": "/vol/coro/jstranghoener/lerobot_volume/huggingface/repos/xvla-base",
154
+ "num_flow_steps": 10,
155
+ "tokenizer_name": "facebook/bart-large",
156
+ "tokenizer_max_length": 50,
157
+ "tokenizer_padding_side": "right",
158
+ "pad_language_to": "max_length",
159
+ "domain_id": 0,
160
+ "xvla_max_action_dim": 20,
161
+ "resize_imgs_with_padding": [
162
+ 224,
163
+ 224
164
+ ],
165
+ "num_image_views": 3,
166
+ "dtype": "bfloat16",
167
+ "freeze_vision_encoder": true,
168
+ "freeze_language_encoder": false,
169
+ "train_policy_transformer": true,
170
+ "train_soft_prompts": true,
171
+ "n_action_steps": 12,
172
+ "chunk_size": 24,
173
+ "num_system2_basis": 12,
174
+ "num_system1_basis": 9,
175
+ "degree_p": 3,
176
+ "gripper_zero_order": true,
177
+ "gripper_dof": 1,
178
+ "sys2_w_min": -5.0,
179
+ "sys2_w_max": 5.0,
180
+ "sys1_w_min": -5.0,
181
+ "sys1_w_max": 5.0,
182
+ "spline_fit_regularization": 1e-05,
183
+ "refiner_hidden_size": 384,
184
+ "refiner_depth": 6,
185
+ "refiner_num_heads": 6,
186
+ "refiner_max_cond_len": 512,
187
+ "action_loss_weight": 1.0,
188
+ "pin_position": true,
189
+ "pin_velocity": true,
190
+ "condition_on_anchor": false,
191
+ "condition_on_anchor_velocity": true,
192
+ "history_len": 5,
193
+ "history_feature_dim": 128,
194
+ "history_time_frequencies_hz": [
195
+ 0.5,
196
+ 1.0,
197
+ 2.0,
198
+ 4.0,
199
+ 8.0
200
+ ],
201
+ "history_freeze_backbone": true,
202
+ "history_pretrained_backbone": true,
203
+ "history_dropout": 0.0,
204
+ "history_diff_weight": 0.001,
205
+ "blind_to_plan": false,
206
+ "predict_action_directly": false,
207
+ "vlm_grad_sys1": true,
208
+ "vlm_grad_sys2": true,
209
+ "train_mode": "both",
210
+ "inference_mode": "both",
211
+ "return_act_chunk": false,
212
+ "action_velocity_key": "observation.action_velocity",
213
+ "sys2_optimizer_lr": 2e-05,
214
+ "sys1_optimizer_lr": 0.0003,
215
+ "optimizer_betas": [
216
+ 0.9,
217
+ 0.95
218
+ ],
219
+ "optimizer_eps": 1e-08,
220
+ "optimizer_weight_decay": 0.0001,
221
+ "optimizer_grad_clip_norm": 1.0,
222
+ "scheduler_warmup_steps": 1000,
223
+ "scheduler_decay_steps": 40000,
224
+ "sys2_scheduler_decay_lr": 2.5e-06,
225
+ "sys1_scheduler_decay_lr": 1e-05
226
+ },
227
+ "output_dir": "/vol/coro/jstranghoener/lerobot_volume/models/dom/lemon_bowl_euler_single_scene/jointly/xvla_refiner_v2_lambda_high_match",
228
+ "job_name": "jointly_xvla_refiner_v2_lambda_high_match_lemon_bowl_euler_single_scene",
229
+ "resume": false,
230
+ "seed": 42,
231
+ "num_workers": 6,
232
+ "batch_size": 8,
233
+ "steps": 40000,
234
+ "eval_freq": 10000,
235
+ "log_freq": 3,
236
+ "tolerance_s": 0.0001,
237
+ "save_checkpoint": true,
238
+ "save_freq": 10000,
239
+ "use_policy_training_preset": true,
240
+ "optimizer": {
241
+ "type": "adamw-beast-xvla-refiner-v2",
242
+ "lr": 2e-05,
243
+ "weight_decay": 0.0001,
244
+ "grad_clip_norm": 1.0,
245
+ "sys2_lr": 2e-05,
246
+ "sys1_lr": 0.0003,
247
+ "betas": [
248
+ 0.9,
249
+ 0.95
250
+ ],
251
+ "eps": 1e-08
252
+ },
253
+ "scheduler": {
254
+ "type": "cosine-beast-xvla-refiner-v2",
255
+ "num_warmup_steps": 1000,
256
+ "num_decay_steps": 40000,
257
+ "sys2_peak_lr": 2e-05,
258
+ "sys2_decay_lr": 2.5e-06,
259
+ "sys1_peak_lr": 0.0003,
260
+ "sys1_decay_lr": 1e-05
261
+ },
262
+ "eval": {
263
+ "n_episodes": 50,
264
+ "batch_size": 50,
265
+ "use_async_envs": false
266
+ },
267
+ "wandb": {
268
+ "enable": true,
269
+ "disable_artifact": true,
270
+ "project": "bielefeld_dom",
271
+ "entity": "beast_refiner",
272
+ "notes": null,
273
+ "run_id": "l1d6bkxx",
274
+ "mode": null
275
+ },
276
+ "peft": null,
277
+ "use_rabc": false,
278
+ "rabc_progress_path": null,
279
+ "rabc_kappa": 0.01,
280
+ "rabc_epsilon": 1e-06,
281
+ "rabc_head_mode": "sparse",
282
+ "rename_map": {
283
+ "observation.images.opst_cam": "observation.images.image",
284
+ "observation.images.side_cam": "observation.images.image2",
285
+ "observation.images.wrist_cam": "observation.images.image3"
286
+ },
287
+ "checkpoint_path": null,
288
+ "offline_validation": {
289
+ "enabled": true,
290
+ "ratio": 0.05,
291
+ "split_seed": 42,
292
+ "max_visualized_episodes": 5,
293
+ "plot_dof": 0
294
+ },
295
+ "resume_optimizer_scheduler_state": true,
296
+ "allow_changed_trainable_params_on_resume": true
297
+ }