nebu1a commited on
Commit
c7db393
·
verified ·
1 Parent(s): e2bdbb9

Upload folder using huggingface_hub

Browse files
config.json ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "smolvla",
3
+ "n_obs_steps": 1,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 6
9
+ ]
10
+ },
11
+ "observation.images.camera1": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 256,
16
+ 256
17
+ ]
18
+ },
19
+ "observation.images.camera2": {
20
+ "type": "VISUAL",
21
+ "shape": [
22
+ 3,
23
+ 256,
24
+ 256
25
+ ]
26
+ },
27
+ "observation.images.camera3": {
28
+ "type": "VISUAL",
29
+ "shape": [
30
+ 3,
31
+ 256,
32
+ 256
33
+ ]
34
+ }
35
+ },
36
+ "output_features": {
37
+ "action": {
38
+ "type": "ACTION",
39
+ "shape": [
40
+ 6
41
+ ]
42
+ }
43
+ },
44
+ "device": "cuda",
45
+ "dtype": null,
46
+ "use_amp": false,
47
+ "use_peft": false,
48
+ "push_to_hub": false,
49
+ "repo_id": "nebu1a/smolvla_sim_stack",
50
+ "private": null,
51
+ "tags": null,
52
+ "license": null,
53
+ "pretrained_path": "lerobot/smolvla_base",
54
+ "pretrained_revision": null,
55
+ "chunk_size": 50,
56
+ "n_action_steps": 50,
57
+ "normalization_mapping": {
58
+ "VISUAL": "IDENTITY",
59
+ "STATE": "MEAN_STD",
60
+ "ACTION": "MEAN_STD"
61
+ },
62
+ "max_state_dim": 32,
63
+ "max_action_dim": 32,
64
+ "resize_imgs_with_padding": [
65
+ 512,
66
+ 512
67
+ ],
68
+ "empty_cameras": 0,
69
+ "adapt_to_pi_aloha": false,
70
+ "use_delta_joint_actions_aloha": false,
71
+ "tokenizer_max_length": 48,
72
+ "num_steps": 10,
73
+ "use_cache": true,
74
+ "freeze_vision_encoder": true,
75
+ "train_expert_only": true,
76
+ "train_state_proj": true,
77
+ "optimizer_lr": 0.0001,
78
+ "optimizer_betas": [
79
+ 0.9,
80
+ 0.95
81
+ ],
82
+ "optimizer_eps": 1e-08,
83
+ "optimizer_weight_decay": 1e-10,
84
+ "optimizer_grad_clip_norm": 10.0,
85
+ "optimizer_fused": null,
86
+ "scheduler_warmup_steps": 300,
87
+ "scheduler_decay_steps": 4000,
88
+ "scheduler_decay_lr": 2.5e-06,
89
+ "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
90
+ "load_vlm_weights": true,
91
+ "add_image_special_tokens": false,
92
+ "attention_mode": "cross_attn",
93
+ "prefix_length": 0,
94
+ "pad_language_to": "max_length",
95
+ "num_expert_layers": 0,
96
+ "num_vlm_layers": 16,
97
+ "self_attn_every_n_layers": 2,
98
+ "expert_width_multiplier": 0.75,
99
+ "min_period": 0.004,
100
+ "max_period": 4.0,
101
+ "rtc_config": null,
102
+ "compile_model": false,
103
+ "compile_mode": "max-autotune"
104
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b655963553ae1410e0f2459379268bfa3b1f34f5082228823a566c1b28210d94
3
+ size 906712552
policy_postprocessor.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "unnormalizer_processor",
6
+ "config": {
7
+ "eps": 1e-08,
8
+ "features": {
9
+ "action": {
10
+ "type": "ACTION",
11
+ "shape": [
12
+ 6
13
+ ]
14
+ }
15
+ },
16
+ "norm_map": {
17
+ "VISUAL": "IDENTITY",
18
+ "STATE": "MEAN_STD",
19
+ "ACTION": "MEAN_STD"
20
+ }
21
+ },
22
+ "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
23
+ },
24
+ {
25
+ "registry_name": "device_processor",
26
+ "config": {
27
+ "device": "cpu",
28
+ "float_dtype": null
29
+ }
30
+ }
31
+ ]
32
+ }
policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d40416feed585c99468b6822f5f8ffc78db49f90f15176e84e77ac32d919dd91
3
+ size 7592
policy_preprocessor.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {
8
+ "observation.images.front": "observation.images.camera1",
9
+ "observation.images.wrist": "observation.images.camera2"
10
+ }
11
+ }
12
+ },
13
+ {
14
+ "registry_name": "to_batch_processor",
15
+ "config": {}
16
+ },
17
+ {
18
+ "registry_name": "smolvla_new_line_processor",
19
+ "config": {}
20
+ },
21
+ {
22
+ "registry_name": "tokenizer_processor",
23
+ "config": {
24
+ "max_length": 48,
25
+ "task_key": "task",
26
+ "padding_side": "right",
27
+ "padding": "max_length",
28
+ "truncation": true,
29
+ "tokenizer_name": "tokenizer"
30
+ },
31
+ "artifacts": {
32
+ "tokenizer_name": "tokenizer"
33
+ }
34
+ },
35
+ {
36
+ "registry_name": "device_processor",
37
+ "config": {
38
+ "device": "cuda",
39
+ "float_dtype": null
40
+ }
41
+ },
42
+ {
43
+ "registry_name": "normalizer_processor",
44
+ "config": {
45
+ "eps": 1e-08,
46
+ "features": {
47
+ "observation.state": {
48
+ "type": "STATE",
49
+ "shape": [
50
+ 6
51
+ ]
52
+ },
53
+ "observation.images.camera1": {
54
+ "type": "VISUAL",
55
+ "shape": [
56
+ 3,
57
+ 256,
58
+ 256
59
+ ]
60
+ },
61
+ "observation.images.camera2": {
62
+ "type": "VISUAL",
63
+ "shape": [
64
+ 3,
65
+ 256,
66
+ 256
67
+ ]
68
+ },
69
+ "observation.images.camera3": {
70
+ "type": "VISUAL",
71
+ "shape": [
72
+ 3,
73
+ 256,
74
+ 256
75
+ ]
76
+ },
77
+ "action": {
78
+ "type": "ACTION",
79
+ "shape": [
80
+ 6
81
+ ]
82
+ }
83
+ },
84
+ "norm_map": {
85
+ "VISUAL": "IDENTITY",
86
+ "STATE": "MEAN_STD",
87
+ "ACTION": "MEAN_STD"
88
+ }
89
+ },
90
+ "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors"
91
+ }
92
+ ]
93
+ }
policy_preprocessor_step_5_normalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f450a1f5a1c9d0d3c1f3a364b109f6460909c42c57ecdf09fcd9bcd099fb146
3
+ size 7600
tokenizer/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ <|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>
2
+ {% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}
tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|im_start|>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "end_of_utterance_token": "<end_of_utterance>",
7
+ "eos_token": "<end_of_utterance>",
8
+ "extra_special_tokens": [
9
+ "<fake_token_around_image>",
10
+ "<image>",
11
+ "<end_of_utterance>"
12
+ ],
13
+ "fake_image_token": "<fake_token_around_image>",
14
+ "global_image_token": "<global-img>",
15
+ "image_token": "<image>",
16
+ "is_local": false,
17
+ "legacy": false,
18
+ "local_files_only": false,
19
+ "model_max_length": 8192,
20
+ "model_specific_special_tokens": {
21
+ "end_of_utterance_token": "<end_of_utterance>",
22
+ "fake_image_token": "<fake_token_around_image>",
23
+ "global_image_token": "<global-img>",
24
+ "image_token": "<image>"
25
+ },
26
+ "pad_token": "<|im_end|>",
27
+ "processor_class": "SmolVLMProcessor",
28
+ "tokenizer_class": "TokenizersBackend",
29
+ "truncation_side": "left",
30
+ "unk_token": "<|endoftext|>",
31
+ "vocab_size": 49152
32
+ }
train_config.json ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "nebu1a/so101_sim_stack",
4
+ "repo_type": "dataset",
5
+ "root": null,
6
+ "episodes": null,
7
+ "exclude_episodes": null,
8
+ "image_transforms": {
9
+ "enable": false,
10
+ "backend": "dataloader",
11
+ "gpu_compile": true,
12
+ "gpu_chunk_size": 32,
13
+ "max_num_transforms": 3,
14
+ "random_order": false,
15
+ "tfs": {
16
+ "brightness": {
17
+ "weight": 1.0,
18
+ "type": "ColorJitter",
19
+ "kwargs": {
20
+ "brightness": [
21
+ 0.8,
22
+ 1.2
23
+ ]
24
+ }
25
+ },
26
+ "contrast": {
27
+ "weight": 1.0,
28
+ "type": "ColorJitter",
29
+ "kwargs": {
30
+ "contrast": [
31
+ 0.8,
32
+ 1.2
33
+ ]
34
+ }
35
+ },
36
+ "saturation": {
37
+ "weight": 1.0,
38
+ "type": "ColorJitter",
39
+ "kwargs": {
40
+ "saturation": [
41
+ 0.5,
42
+ 1.5
43
+ ]
44
+ }
45
+ },
46
+ "hue": {
47
+ "weight": 1.0,
48
+ "type": "ColorJitter",
49
+ "kwargs": {
50
+ "hue": [
51
+ -0.05,
52
+ 0.05
53
+ ]
54
+ }
55
+ },
56
+ "sharpness": {
57
+ "weight": 1.0,
58
+ "type": "SharpnessJitter",
59
+ "kwargs": {
60
+ "sharpness": [
61
+ 0.5,
62
+ 1.5
63
+ ]
64
+ }
65
+ },
66
+ "affine": {
67
+ "weight": 1.0,
68
+ "type": "RandomAffine",
69
+ "kwargs": {
70
+ "degrees": [
71
+ -5.0,
72
+ 5.0
73
+ ],
74
+ "translate": [
75
+ 0.05,
76
+ 0.05
77
+ ]
78
+ }
79
+ }
80
+ }
81
+ },
82
+ "revision": null,
83
+ "use_imagenet_stats": true,
84
+ "video_backend": "torchcodec",
85
+ "return_uint8": false,
86
+ "depth_output_unit": "mm",
87
+ "streaming": false,
88
+ "streaming_episode_pool_size": null,
89
+ "streaming_sampling_strategy": "remaining",
90
+ "streaming_prefetch_episodes": 8,
91
+ "streaming_byte_budget_gb": 8.0,
92
+ "streaming_decode_threads": 2,
93
+ "streaming_decoded_queue_size": 8,
94
+ "streaming_native_http_connections": null,
95
+ "streaming_native_http_subranges": 1,
96
+ "streaming_sidecar_lock_timeout_s": 1800,
97
+ "video_decoder_cache_size": null,
98
+ "eval_split": 0.0
99
+ },
100
+ "env": null,
101
+ "policy": {
102
+ "type": "smolvla",
103
+ "n_obs_steps": 1,
104
+ "input_features": {
105
+ "observation.state": {
106
+ "type": "STATE",
107
+ "shape": [
108
+ 6
109
+ ]
110
+ },
111
+ "observation.images.camera1": {
112
+ "type": "VISUAL",
113
+ "shape": [
114
+ 3,
115
+ 256,
116
+ 256
117
+ ]
118
+ },
119
+ "observation.images.camera2": {
120
+ "type": "VISUAL",
121
+ "shape": [
122
+ 3,
123
+ 256,
124
+ 256
125
+ ]
126
+ },
127
+ "observation.images.camera3": {
128
+ "type": "VISUAL",
129
+ "shape": [
130
+ 3,
131
+ 256,
132
+ 256
133
+ ]
134
+ }
135
+ },
136
+ "output_features": {
137
+ "action": {
138
+ "type": "ACTION",
139
+ "shape": [
140
+ 6
141
+ ]
142
+ }
143
+ },
144
+ "device": "cuda",
145
+ "dtype": null,
146
+ "use_amp": false,
147
+ "use_peft": false,
148
+ "push_to_hub": false,
149
+ "repo_id": "nebu1a/smolvla_sim_stack",
150
+ "private": null,
151
+ "tags": null,
152
+ "license": null,
153
+ "pretrained_path": "lerobot/smolvla_base",
154
+ "pretrained_revision": null,
155
+ "chunk_size": 50,
156
+ "n_action_steps": 50,
157
+ "normalization_mapping": {
158
+ "VISUAL": "IDENTITY",
159
+ "STATE": "MEAN_STD",
160
+ "ACTION": "MEAN_STD"
161
+ },
162
+ "max_state_dim": 32,
163
+ "max_action_dim": 32,
164
+ "resize_imgs_with_padding": [
165
+ 512,
166
+ 512
167
+ ],
168
+ "empty_cameras": 0,
169
+ "adapt_to_pi_aloha": false,
170
+ "use_delta_joint_actions_aloha": false,
171
+ "tokenizer_max_length": 48,
172
+ "num_steps": 10,
173
+ "use_cache": true,
174
+ "freeze_vision_encoder": true,
175
+ "train_expert_only": true,
176
+ "train_state_proj": true,
177
+ "optimizer_lr": 0.0001,
178
+ "optimizer_betas": [
179
+ 0.9,
180
+ 0.95
181
+ ],
182
+ "optimizer_eps": 1e-08,
183
+ "optimizer_weight_decay": 1e-10,
184
+ "optimizer_grad_clip_norm": 10.0,
185
+ "optimizer_fused": null,
186
+ "scheduler_warmup_steps": 300,
187
+ "scheduler_decay_steps": 4000,
188
+ "scheduler_decay_lr": 2.5e-06,
189
+ "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
190
+ "load_vlm_weights": true,
191
+ "add_image_special_tokens": false,
192
+ "attention_mode": "cross_attn",
193
+ "prefix_length": 0,
194
+ "pad_language_to": "max_length",
195
+ "num_expert_layers": 0,
196
+ "num_vlm_layers": 16,
197
+ "self_attn_every_n_layers": 2,
198
+ "expert_width_multiplier": 0.75,
199
+ "min_period": 0.004,
200
+ "max_period": 4.0,
201
+ "rtc_config": null,
202
+ "compile_model": false,
203
+ "compile_mode": "max-autotune"
204
+ },
205
+ "reward_model": null,
206
+ "output_dir": "/kaggle/working/stack_run",
207
+ "job_name": "smolvla_stack",
208
+ "resume": false,
209
+ "seed": 1000,
210
+ "cudnn_deterministic": false,
211
+ "tf32": true,
212
+ "num_workers": 2,
213
+ "batch_size": 8,
214
+ "prefetch_factor": 4,
215
+ "persistent_workers": true,
216
+ "dataloader_multiprocessing_context": "spawn",
217
+ "steps": 4000,
218
+ "env_eval_freq": 20000,
219
+ "log_freq": 50,
220
+ "eval_steps": 0,
221
+ "max_eval_samples": 0,
222
+ "tolerance_s": 0.0001,
223
+ "save_checkpoint": true,
224
+ "save_freq": 1000,
225
+ "checkpoint_format": "safetensors",
226
+ "use_policy_training_preset": true,
227
+ "optimizer": {
228
+ "type": "adamw",
229
+ "lr": 0.0001,
230
+ "weight_decay": 1e-10,
231
+ "grad_clip_norm": 10.0,
232
+ "betas": [
233
+ 0.9,
234
+ 0.95
235
+ ],
236
+ "eps": 1e-08,
237
+ "fused": null
238
+ },
239
+ "scheduler": {
240
+ "type": "cosine_decay_with_warmup",
241
+ "num_warmup_steps": 300,
242
+ "num_decay_steps": 4000,
243
+ "peak_lr": 0.0001,
244
+ "decay_lr": 2.5e-06
245
+ },
246
+ "parallelism": {
247
+ "dp_replicate": 1,
248
+ "dp_shard": 1,
249
+ "context_parallel": {
250
+ "ring_degree": 1,
251
+ "ulysses_degree": 1
252
+ },
253
+ "cfg_parallel": 1
254
+ },
255
+ "accelerator": {
256
+ "mixed_precision": "no",
257
+ "gradient_accumulation": {
258
+ "steps": 1,
259
+ "sync_each_batch": false
260
+ },
261
+ "grad_scaler": {
262
+ "init_scale": 65536.0,
263
+ "growth_factor": 2.0,
264
+ "backoff_factor": 0.5,
265
+ "growth_interval": 2000
266
+ },
267
+ "fsdp": {
268
+ "reshard_after_forward": true,
269
+ "wrap_modules": null,
270
+ "min_num_params": null,
271
+ "cpu_offload": false,
272
+ "ignored_modules": null,
273
+ "mixed_precision_policy": {
274
+ "param_dtype": null,
275
+ "reduce_dtype": null,
276
+ "output_dtype": null,
277
+ "cast_forward_inputs": true
278
+ }
279
+ },
280
+ "ddp": {
281
+ "find_unused_parameters": true,
282
+ "gradient_as_bucket_view": false,
283
+ "static_graph": false
284
+ },
285
+ "init_process_group": {
286
+ "backend": "nccl",
287
+ "init_method": null,
288
+ "timeout": null
289
+ },
290
+ "compile": {
291
+ "enabled": false,
292
+ "backend": "inductor",
293
+ "mode": null,
294
+ "regional": true
295
+ },
296
+ "activation_checkpointing": {
297
+ "mode": "none"
298
+ }
299
+ },
300
+ "eval": {
301
+ "n_episodes": 50,
302
+ "batch_size": 2,
303
+ "use_async_envs": true,
304
+ "recording": false,
305
+ "recording_repo_id": null,
306
+ "recording_private": false
307
+ },
308
+ "ema": null,
309
+ "trackio": {
310
+ "enable": false,
311
+ "disable_artifact": true,
312
+ "project": "lerobot",
313
+ "space_id": null,
314
+ "private": null,
315
+ "run_name": null,
316
+ "resume": null,
317
+ "capture_console": true
318
+ },
319
+ "peft": null,
320
+ "job": {
321
+ "target": null,
322
+ "image": "huggingface/lerobot-gpu:latest",
323
+ "timeout": "2d",
324
+ "detach": false,
325
+ "tags": []
326
+ },
327
+ "save_checkpoint_to_hub": false,
328
+ "sample_weighting": null,
329
+ "rename_map": {
330
+ "observation.images.front": "observation.images.camera1",
331
+ "observation.images.wrist": "observation.images.camera2"
332
+ },
333
+ "checkpoint_path": null
334
+ }