Hugging Face's logo

lerobot
/
g05_base

Robotics
LeRobot
Safetensors
g05
non-commercial
Model card Files
xet
 Community
1

Update render step for lerobot #4183

#1
by maximellerbach HF Staff - opened
base: refs/heads/main
←
from: refs/pr/1
Discussion Files changed
+234
-998
Files changed (1) hide show
  1. policy_preprocessor.json +234 -998
policy_preprocessor.json CHANGED
@@ -1,1022 +1,258 @@
1
  {
2
- "name": "policy_preprocessor",
3
- "steps": [
4
- {
5
- "registry_name": "render_messages_processor",
6
- "config": {
7
- "recipe": {
8
- "messages": [
9
- {
10
- "role": "user",
11
- "content": "${task}",
12
- "stream": "low_level",
13
- "target": false,
14
- "if_present": null,
15
- "tool_calls_from": null
16
- },
17
- {
18
- "role": "assistant",
19
- "content": "BBoxJSON: ${bbox}",
20
- "stream": "low_level",
21
- "target": true,
22
- "if_present": "bbox",
23
- "tool_calls_from": null
24
- },
25
- {
26
- "role": "assistant",
27
- "content": "Subtask: ${subtask}",
28
- "stream": "low_level",
29
- "target": true,
30
- "if_present": "subtask",
31
- "tool_calls_from": null
 
 
 
 
 
 
 
 
 
32
  }
33
- ],
34
- "bindings": {
35
- "bbox": "emitted_at(t, style=vqa, role=assistant, camera=observation.images.exterior)"
36
- },
37
- "blend": null,
38
- "weight": null,
39
- "route": null
40
  },
41
- "render_training": false
42
- }
43
- },
44
- {
45
- "registry_name": "rename_observations_processor",
46
- "config": {
47
- "rename_map": {}
48
- }
49
- },
50
- {
51
- "registry_name": "to_batch_processor",
52
- "config": {}
53
- },
54
- {
55
- "registry_name": "g05_image_transform",
56
- "config": {
57
- "camera_order": [
58
- "observation.images.head_rgb",
59
- "observation.images.left_wrist_rgb",
60
- "observation.images.right_wrist_rgb"
61
- ],
62
- "camera_sizes": {
63
- "observation.images.head_rgb": [
64
- 256,
65
- 256
66
- ],
67
- "observation.images.left_wrist_rgb": [
68
- 256,
69
- 256
70
- ],
71
- "observation.images.right_wrist_rgb": [
72
- 256,
73
- 256
74
- ]
75
  },
76
- "mean": [
77
- 0.5,
78
- 0.5,
79
- 0.5
80
- ],
81
- "std": [
82
- 0.5,
83
- 0.5,
84
- 0.5
85
- ],
86
- "optional_camera_keys": []
87
- }
88
- },
89
- {
90
- "registry_name": "g05_action_operation_mask",
91
- "config": {
92
- "action_parts": [
93
- [
94
- "left_arm",
95
- 6
96
- ],
97
- [
98
- "left_gripper",
99
- 1
100
- ],
101
- [
102
- "right_arm",
103
- 6
104
- ],
105
- [
106
- "right_gripper",
107
- 1
108
- ]
109
- ],
110
- "joint_threshold": null,
111
- "gripper_threshold": null,
112
- "velocity_threshold": 0.1,
113
- "eef_threshold": 0.001,
114
- "dim_thresholds": {
115
- "left_arm": [
116
- 0.00756,
117
- 0.00662,
118
- 0.0432,
119
- 0.00732,
120
- 0.00493,
121
- 0.00612
122
- ],
123
- "right_arm": [
124
- 0.0115,
125
- 0.00717,
126
- 0.0619,
127
- 0.012,
128
- 0.00737,
129
- 0.00918
130
- ]
131
- }
132
- }
133
- },
134
- {
135
- "registry_name": "g05_relative_joint_actions",
136
- "config": {
137
- "enabled": true,
138
- "exclude_joints": [
139
- "gripper"
140
- ],
141
- "action_names": [
142
- "left_arm.0",
143
- "left_arm.1",
144
- "left_arm.2",
145
- "left_arm.3",
146
- "left_arm.4",
147
- "left_arm.5",
148
- "left_gripper.0",
149
- "right_arm.0",
150
- "right_arm.1",
151
- "right_arm.2",
152
- "right_arm.3",
153
- "right_arm.4",
154
- "right_arm.5",
155
- "right_gripper.0"
156
- ],
157
- "num_obs_steps": 6
158
- }
159
- },
160
- {
161
- "registry_name": "g05_embodiment_projection",
162
- "config": {
163
- "embodiment": "galaxea_r1lite",
164
- "policy_state_dim": 27,
165
- "policy_action_dim": 27,
166
- "camera_order": [
167
- "observation.images.head_rgb",
168
- "observation.images.left_wrist_rgb",
169
- "observation.images.right_wrist_rgb"
170
- ]
171
- }
172
- },
173
- {
174
- "registry_name": "g05_tail_normalization",
175
- "config": {
176
- "inverse": false,
177
- "tail_scale": 0.075
178
- },
179
- "state_file": "policy_preprocessor_step_6_g05_tail_normalization.safetensors"
180
- },
181
- {
182
- "registry_name": "normalizer_processor",
183
- "config": {
184
- "eps": 1e-08,
185
- "features": {
186
- "observation.state": {
187
- "type": "STATE",
188
- "shape": [
189
- 27
190
- ]
191
- },
192
- "observation.images.head_rgb": {
193
- "type": "VISUAL",
194
- "shape": [
195
- 3,
196
- 256,
197
- 256
198
- ]
199
- },
200
- "observation.images.left_wrist_rgb": {
201
- "type": "VISUAL",
202
- "shape": [
203
- 3,
204
- 256,
205
- 256
206
- ]
207
- },
208
- "observation.images.right_wrist_rgb": {
209
- "type": "VISUAL",
210
- "shape": [
211
- 3,
212
- 256,
213
- 256
214
- ]
215
- },
216
- "action": {
217
- "type": "ACTION",
218
- "shape": [
219
- 27
220
- ]
221
- }
222
  },
223
- "norm_map": {
224
- "STATE": "MEAN_STD",
225
- "ACTION": "MEAN_STD",
226
- "VISUAL": "IDENTITY"
227
- }
228
- },
229
- "state_file": "policy_preprocessor_step_7_normalizer_processor.safetensors"
230
- },
231
- {
232
- "registry_name": "g05_normalization_clamp",
233
- "config": {
234
- "minimum": -5.0,
235
- "maximum": 5.0
236
- }
237
- },
238
- {
239
- "registry_name": "device_processor",
240
- "config": {
241
- "device": "cuda",
242
- "float_dtype": null
243
- }
244
- },
245
- {
246
- "registry_name": "g05_tokenizer",
247
- "config": {
248
- "checkpoint_path": "lerobot/g05_base",
249
- "policy_config": {
250
- "author_model_config": {
251
- "pretrained_model_path": null,
252
- "hf_processor_class": "g05.models.g05.qwen35.processing.Qwen35ProcessorWrapper",
253
- "base_vocab_size": 248044,
254
- "padded_vocab_size": 248320,
255
- "add_loc_tokens": false,
256
- "image_token_index": 248056,
257
- "pad_token_id": 0,
258
- "vocab_size": 248320,
259
- "eos_token_id": 248044,
260
- "action_dim": 27,
261
- "proprio_dim": 27,
262
- "proprio_encoder": "mlp",
263
- "horizon_steps": 32,
264
- "cond_steps": 1,
265
- "num_obs_steps": 1,
266
- "attn_implementation": "eager",
267
- "position_ids_type": "pi0fast",
268
- "action_position_offset": null,
269
- "checkpoint_vision": true,
270
- "checkpoint_vlm": false,
271
- "checkpoint_action_expert": true,
272
- "vlm": {
273
- "name": "g05.models.g05.qwen35.mixture_qwen35.MixtureQwen35",
274
- "hidden_size": 2048,
275
- "intermediate_size": 6144,
276
- "num_hidden_layers": 24,
277
- "num_attention_heads": 8,
278
- "num_key_value_heads": 2,
279
- "head_dim": 256,
280
- "rms_norm_eps": 1e-06,
281
- "max_position_embeddings": 262144,
282
- "attention_bias": false,
283
- "input_type": "embedding",
284
- "vocab_size": 248320,
285
- "pad_token_id": 0,
286
- "output_type": "lm_head",
287
- "adaptive_mode": null,
288
- "time_hidden_size": 0,
289
- "use_final_norm": true,
290
- "hidden_act": "silu",
291
- "rope_parameters": {
292
- "rope_theta": 10000000.0,
293
- "rope_type": "default",
294
- "mrope_section": [
295
- 11,
296
- 11,
297
- 10
298
- ],
299
- "mrope_interleaved": true,
300
- "partial_rotary_factor": 0.25
301
- },
302
- "linear_attn_backend": "fla",
303
- "linear_conv_kernel_dim": 4,
304
- "linear_key_head_dim": 128,
305
- "linear_value_head_dim": 128,
306
- "linear_num_key_heads": 16,
307
- "linear_num_value_heads": 16,
308
- "layer_types": [
309
- "linear_attention",
310
- "linear_attention",
311
- "linear_attention",
312
- "full_attention",
313
- "linear_attention",
314
- "linear_attention",
315
- "linear_attention",
316
- "full_attention",
317
- "linear_attention",
318
- "linear_attention",
319
- "linear_attention",
320
- "full_attention",
321
- "linear_attention",
322
- "linear_attention",
323
- "linear_attention",
324
- "full_attention",
325
- "linear_attention",
326
- "linear_attention",
327
- "linear_attention",
328
- "full_attention",
329
- "linear_attention",
330
- "linear_attention",
331
- "linear_attention",
332
- "full_attention"
333
- ]
334
- },
335
- "action_expert": {
336
- "name": "g05.models.g05.qwen35.mixture_qwen35.MixtureQwen35",
337
- "hidden_size": 1024,
338
- "intermediate_size": 4096,
339
- "num_hidden_layers": 24,
340
- "num_attention_heads": 8,
341
- "num_key_value_heads": 2,
342
- "head_dim": 256,
343
- "rms_norm_eps": 1e-06,
344
- "max_position_embeddings": 262144,
345
- "attention_bias": false,
346
- "adaptive_mode": "adaLN",
347
- "time_hidden_size": 1024,
348
- "input_type": "linear",
349
- "input_dim": 27,
350
- "output_type": "linear",
351
- "output_dim": 27,
352
- "use_final_norm": true,
353
- "hidden_act": "silu",
354
- "rope_parameters": {
355
- "rope_theta": 10000000.0,
356
- "rope_type": "default",
357
- "mrope_section": [
358
- 11,
359
- 11,
360
- 10
361
- ],
362
- "mrope_interleaved": true,
363
- "partial_rotary_factor": 0.25
364
- },
365
- "linear_attn_backend": "fla",
366
- "linear_conv_kernel_dim": 4,
367
- "linear_key_head_dim": 128,
368
- "linear_value_head_dim": 128,
369
- "linear_num_key_heads": 16,
370
- "linear_num_value_heads": 16,
371
- "layer_types": [
372
- "full_attention",
373
- "full_attention",
374
- "full_attention",
375
- "full_attention",
376
- "full_attention",
377
- "full_attention",
378
- "full_attention",
379
- "full_attention",
380
- "full_attention",
381
- "full_attention",
382
- "full_attention",
383
- "full_attention",
384
- "full_attention",
385
- "full_attention",
386
- "full_attention",
387
- "full_attention",
388
- "full_attention",
389
- "full_attention",
390
- "full_attention",
391
- "full_attention",
392
- "full_attention",
393
- "full_attention",
394
- "full_attention",
395
- "full_attention"
396
- ]
397
- },
398
- "fm": {
399
- "time_convention": "pi_convention",
400
- "flow_sampling": "beta",
401
- "flow_sig_min": 0.001,
402
- "num_inference_steps": 10,
403
- "num_flow_samples": 4,
404
- "fm_weight": 1.0,
405
- "joint_training": false,
406
- "padding_action_weight": 0.0,
407
- "zero_pad_action_target": false,
408
- "final_action_clip_value": null,
409
- "action_causal": false,
410
- "horizon_steps": 32,
411
- "action_dim": 27,
412
- "use_correlated_noise": false,
413
- "correlation_beta": 0.5,
414
- "action_correlation_cholesky_path": "data/stats/action_correlation_cholesky.npy",
415
- "correlated_noise_embodiments": []
416
- },
417
- "ar": {
418
- "ce_weight": 1.0,
419
- "vocab_size": 248320,
420
- "text_token_end": 248044,
421
- "loc_token_end": 248077,
422
- "use_fused_ce": true,
423
- "ce_z_loss_scale": 0.0,
424
- "block_wise_autoregressive": false,
425
- "bos_blk_id": null,
426
- "eos_blk_id": null,
427
- "block_size": null,
428
- "do_sample": false,
429
- "max_new_tokens": 300,
430
- "temperature": 0.7,
431
- "top_k": 128,
432
- "top_p": 0.95,
433
- "repetition_penalty": 1.2,
434
- "no_repeat_ngram_size": 3
435
- },
436
- "vision": {
437
- "name": "g05.models.g05.qwen35.vision.Qwen3_5VisionModel",
438
- "depth": 24,
439
- "hidden_size": 1024,
440
- "num_heads": 16,
441
- "patch_size": 16,
442
- "temporal_patch_size": 2,
443
- "spatial_merge_size": 2,
444
- "in_channels": 3,
445
- "intermediate_size": 4096,
446
- "out_hidden_size": 2048,
447
- "num_position_embeddings": 2304,
448
- "hidden_act": "gelu_pytorch_tanh",
449
- "num_image_tokens": null,
450
- "num_channels": 3,
451
- "temporal_freq": 4,
452
- "spacetime_mode": "factorized",
453
- "token_drop_layer": null,
454
- "temporal_pe_pretrain_frames": null,
455
- "use_temporal_conv3d": false,
456
- "batch_all_cameras": false
457
- },
458
- "vision_projector": {
459
- "name": null
460
- },
461
- "action_tokenizer": "g05.tokenizer.interface.vq_base.VQActionTokenizer",
462
- "AT_CONFIG": {
463
- "vqvae_type": "g05.tokenizer.models.actioncodec2_v2.wrapper.ActionCodecV2Wrapper",
464
- "block_wise_autoregressive": false,
465
- "use_extra_tokens": true,
466
- "block_size": 8,
467
- "parts_meta": {
468
- "left_control": 9,
469
- "left_gripper": 1,
470
- "right_control": 9,
471
- "right_gripper": 1,
472
- "lower_body": 7
473
- },
474
- "rule_based_key_patterns": [
475
- "gripper"
476
- ],
477
- "rule_based_min_block_len": 1,
478
- "rule_based_binarize_threshold": 0.0,
479
- "ckpt_dir": "checkpoints/action_tokenizer.pt",
480
- "model_arch": {
481
- "max_component_dim": 9,
482
- "horizon": 32,
483
- "horizon_patch_size": 8,
484
- "conv_in_action_kernel": 2,
485
- "encoder_channels": 256,
486
- "latent_dim": 128,
487
- "c_mults": [
488
- 1,
489
- 2,
490
- 2
491
- ],
492
- "strides": [
493
- [
494
- 1,
495
- 1
496
- ],
497
- [
498
- 2,
499
- 1
500
- ],
501
- [
502
- 2,
503
- 1
504
- ]
505
- ],
506
- "transformer_depths": [
507
- 2,
508
- 2,
509
- 2
510
- ],
511
- "num_heads": 8,
512
- "dim_heads": 64,
513
- "ffn_mult": 4.0,
514
- "use_layer_scale": true,
515
- "layer_scale_init": 0.01,
516
- "use_qk_norm": true,
517
- "rope_base": 10000,
518
- "use_block_dct": true,
519
- "block_dct_block_size": 8,
520
- "n_codebooks": 4,
521
- "codebook_size": 4096,
522
- "codebook_dim": 8,
523
- "quantizer_dropout": 0.5,
524
- "ema_decay": 0.95,
525
- "threshold_ema_dead": 2,
526
- "use_rotation_trick": false,
527
- "commitment_loss_weight": 0.25,
528
- "reconstruction_loss_weight": 1.0,
529
- "consistency_loss_weight": 0.0,
530
- "consistency_delta_max_begin": 1,
531
- "consistency_delta_max_end": 3,
532
- "consistency_eps_begin": 0.02,
533
- "consistency_eps_end": 0.1,
534
- "consistency_layer_weights_begin": [
535
- 0.0,
536
- 0.0,
537
- 0.0,
538
- 0.0
539
- ],
540
- "consistency_layer_weights_end": [
541
- 1.0,
542
- 0.7,
543
- 0.3,
544
- 0.1
545
  ],
546
- "consistency_schedule_begin_fraction": 0.2,
547
- "consistency_schedule_end_fraction": 0.8
548
- },
549
- "action_template": "{action}",
550
- "num_residuals": 2,
551
- "use_group_markers": true,
552
- "group_order_shuffle": false,
553
- "dropout_noop_parts": true,
554
- "absent_key_fill_value": -100.0
555
- },
556
- "vla_training_strategy": "vla-full-train",
557
- "backbone_lr_multiplier": 1.0,
558
- "max_text_tokens": 160,
559
- "num_input_images": 18,
560
- "camera_size_config": {
561
- "exterior": [
562
- 256,
563
- 256
564
- ],
565
- "wrist_left": [
566
- 256,
567
- 256
568
- ],
569
- "wrist_right": [
570
- 256,
571
- 256
572
- ]
573
- },
574
- "num_extra_image_tokens_per_camera": 0,
575
- "use_lm_head": true,
576
- "discrete_action": true,
577
- "continuous_action": true,
578
- "return_continuous_action": true,
579
- "action_attend_cot": false,
580
- "predict_cot": true,
581
- "max_chunk_token_length": 1024,
582
- "max_pad_token_length": null,
583
- "input_preprocessor": {
584
- "input_action_corruption": false,
585
- "pred_eov": true,
586
- "batchify_action": true,
587
- "pi05_ft_mode": false
588
- },
589
- "ae_vlm_condition_mode": "cross_attn_only",
590
- "mem_frame_drop_prob": 0.3,
591
- "vision_lr_multiplier": 1.0,
592
- "hf_processor_path": "checkpoints/qwen3_5_2b_base_processor"
593
- },
594
- "embodiment": "galaxea_r1lite",
595
- "predict_cot": true,
596
- "runtime_system": "system2",
597
- "prompt_template": "<chat_user_prefix><image0_image_!><image1_image_!><image2_image_!><bos>Embodiment: <embodiment_text_!>; Task: <command_text_!_200> State: <proprio_proprio_!>;<chat_user_suffix><chat_assistant_prefix><prompt_text_!>\n<EOC><atomic_task_text>|Action: <EOV><action_action>|<eos>",
598
- "num_prompt_images": 3,
599
- "num_input_images": 18,
600
- "camera_order": [
601
- "observation.images.head_rgb",
602
- "observation.images.left_wrist_rgb",
603
- "observation.images.right_wrist_rgb"
604
- ],
605
- "camera_sizes": {
606
- "observation.images.head_rgb": [
607
- 256,
608
- 256
609
- ],
610
- "observation.images.left_wrist_rgb": [
611
- 256,
612
- 256
613
- ],
614
- "observation.images.right_wrist_rgb": [
615
- 256,
616
- 256
617
- ]
618
- },
619
- "policy_state_dim": 27,
620
- "policy_action_dim": 27,
621
- "chunk_size": 32,
622
- "processor_metadata": {
623
- "shape_meta": {
624
- "action": [
625
- {
626
- "key": "left_arm",
627
- "lerobot_key": "observation.state.left_arm",
628
- "start_index": 0,
629
- "raw_shape": 6,
630
- "shape": 6,
631
- "time_offset": 1
632
- },
633
- {
634
- "key": "left_gripper",
635
- "lerobot_key": "observation.state.left_gripper",
636
- "start_index": 0,
637
- "raw_shape": 1,
638
- "shape": 1,
639
- "time_offset": 1
640
- },
641
- {
642
- "key": "right_arm",
643
- "lerobot_key": "observation.state.right_arm",
644
- "start_index": 0,
645
- "raw_shape": 6,
646
- "shape": 6,
647
- "time_offset": 1
648
- },
649
- {
650
- "key": "right_gripper",
651
- "lerobot_key": "observation.state.right_gripper",
652
- "start_index": 0,
653
- "raw_shape": 1,
654
- "shape": 1,
655
- "time_offset": 1
656
- }
657
- ],
658
- "state": [
659
- {
660
- "key": "left_arm",
661
- "lerobot_key": "observation.state.left_arm",
662
- "start_index": 0,
663
- "raw_shape": 6,
664
- "shape": 6,
665
- "time_offset": 0
666
- },
667
- {
668
- "key": "left_gripper",
669
- "lerobot_key": "observation.state.left_gripper",
670
- "start_index": 0,
671
- "raw_shape": 1,
672
- "shape": 1,
673
- "time_offset": 0
674
- },
675
- {
676
- "key": "right_arm",
677
- "lerobot_key": "observation.state.right_arm",
678
- "start_index": 0,
679
- "raw_shape": 6,
680
- "shape": 6,
681
- "time_offset": 0
682
- },
683
- {
684
- "key": "right_gripper",
685
- "lerobot_key": "observation.state.right_gripper",
686
- "start_index": 0,
687
- "raw_shape": 1,
688
- "shape": 1,
689
- "time_offset": 0
690
- }
691
- ],
692
- "images": [
693
- {
694
- "key": "head_rgb",
695
- "camera_type": "exterior",
696
- "lerobot_key": "observation.images.head_rgb",
697
- "start_index": 0,
698
- "raw_shape": [
699
- 3,
700
- 720,
701
- 1280
702
- ],
703
- "shape": [
704
- 3,
705
- 224,
706
- 224
707
- ],
708
- "time_offset": 0
709
- },
710
- {
711
- "key": "left_wrist_rgb",
712
- "camera_type": "wrist_left",
713
- "lerobot_key": "observation.images.left_wrist_rgb",
714
- "start_index": 0,
715
- "raw_shape": [
716
- 3,
717
- 720,
718
- 1280
719
- ],
720
- "shape": [
721
- 3,
722
- 224,
723
- 224
724
- ],
725
- "time_offset": 0
726
- },
727
- {
728
- "key": "right_wrist_rgb",
729
- "camera_type": "wrist_right",
730
- "lerobot_key": "observation.images.right_wrist_rgb",
731
- "start_index": 0,
732
- "raw_shape": [
733
- 3,
734
- 720,
735
- 1280
736
- ],
737
- "shape": [
738
- 3,
739
- 224,
740
- 224
741
- ],
742
- "time_offset": 0
743
- }
744
- ]
745
- },
746
- "action_state_transforms": [
747
- {
748
- "_target_": "g05.data_processor.transforms.relative_action.RelativeJointTransform",
749
- "keys": [
750
- "left_arm",
751
- "right_arm"
752
- ]
753
- }
754
- ],
755
- "train_transforms": {
756
- "head_rgb": [
757
- {
758
- "_target_": "g05.data_processor.transforms.image.ToTensor"
759
- },
760
- {
761
- "_target_": "torchvision.transforms.Normalize",
762
- "mean": [
763
- 0.5,
764
- 0.5,
765
- 0.5
766
- ],
767
- "std": [
768
- 0.5,
769
- 0.5,
770
- 0.5
771
- ]
772
- }
773
- ],
774
- "left_wrist_rgb": [
775
- {
776
- "_target_": "g05.data_processor.transforms.image.ToTensor"
777
- },
778
- {
779
- "_target_": "torchvision.transforms.Normalize",
780
- "mean": [
781
- 0.5,
782
- 0.5,
783
- 0.5
784
- ],
785
- "std": [
786
- 0.5,
787
- 0.5,
788
- 0.5
789
- ]
790
- }
791
- ],
792
- "right_wrist_rgb": [
793
- {
794
- "_target_": "g05.data_processor.transforms.image.ToTensor"
795
- },
796
- {
797
- "_target_": "torchvision.transforms.Normalize",
798
- "mean": [
799
- 0.5,
800
- 0.5,
801
- 0.5
802
- ],
803
- "std": [
804
- 0.5,
805
- 0.5,
806
- 0.5
807
- ]
808
- }
809
- ]
810
- },
811
- "val_transforms": {
812
- "head_rgb": [
813
- {
814
- "_target_": "g05.data_processor.transforms.image.ToTensor"
815
  },
816
- {
817
- "_target_": "torchvision.transforms.Normalize",
818
- "mean": [
819
  0.5,
820
  0.5,
821
  0.5
822
- ],
823
- "std": [
824
- 0.5,
825
- 0.5,
826
- 0.5
827
- ]
828
- }
829
- ],
830
- "left_wrist_rgb": [
831
- {
832
- "_target_": "g05.data_processor.transforms.image.ToTensor"
833
- },
834
- {
835
- "_target_": "torchvision.transforms.Normalize",
836
- "mean": [
837
- 0.5,
838
- 0.5,
839
- 0.5
840
- ],
841
- "std": [
842
- 0.5,
843
- 0.5,
844
- 0.5
845
- ]
846
- }
847
- ],
848
- "right_wrist_rgb": [
849
- {
850
- "_target_": "g05.data_processor.transforms.image.ToTensor"
851
- },
852
- {
853
- "_target_": "torchvision.transforms.Normalize",
854
- "mean": [
855
- 0.5,
856
- 0.5,
857
- 0.5
858
- ],
859
- "std": [
860
  0.5,
861
  0.5,
862
  0.5
863
- ]
864
- }
865
- ]
866
- },
867
- "norm_default_mode": "z-score-tail",
868
- "norm_exception_mode": {
869
- "action": {
870
- "left_gripper": "q01/q99",
871
- "right_gripper": "q01/q99"
872
- },
873
- "state": {
874
- "left_gripper": "q01/q99",
875
- "right_gripper": "q01/q99"
876
- }
877
- },
878
- "action_filter": {
879
- "_target_": "g05.data_processor.processor.galaxea_action_processor.R1LiteJointActionFilter",
880
- "velocity_threshold": 0.1,
881
- "dim_thresholds": {
882
- "left_arm": [
883
- 0.00756,
884
- 0.00662,
885
- 0.0432,
886
- 0.00732,
887
- 0.00493,
888
- 0.00612
889
  ],
890
- "right_arm": [
891
- 0.0115,
892
- 0.00717,
893
- 0.0619,
894
- 0.012,
895
- 0.00737,
896
- 0.00918
897
- ]
898
- }
899
- },
900
- "action_state_merger": {
901
- "_target_": "g05.data_processor.transforms.action_state_merger.GroupedPaddingMerger",
902
- "merge": true,
903
- "max_action_shape_meta": {
904
- "left_arm": 7,
905
- "right_arm": 7,
906
- "left_gripper": 1,
907
- "right_gripper": 1,
908
- "left_ee_pose": 9,
909
- "right_ee_pose": 9,
910
- "lower_body": 7
911
- },
912
- "max_state_shape_meta": {
913
- "left_arm": 7,
914
- "right_arm": 7,
915
- "left_gripper": 1,
916
- "right_gripper": 1,
917
- "left_ee_pose": 9,
918
- "right_ee_pose": 9,
919
- "lower_body": 7
920
- },
921
- "merge_spec": {
922
- "left_control": [
923
- "left_arm",
924
- "left_ee_pose"
925
- ],
926
- "left_gripper": [
927
- "left_gripper"
928
  ],
929
- "right_control": [
930
- "right_arm",
931
- "right_ee_pose"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
932
  ],
933
- "right_gripper": [
934
- "right_gripper"
 
 
 
 
 
 
 
 
 
 
 
 
 
935
  ],
936
- "lower_body": [
937
- "lower_body"
 
 
 
 
 
 
 
 
 
 
 
938
  ]
939
- }
940
- },
941
- "num_obs_steps": 6,
942
- "use_stepwise_action_norm": true,
943
- "drop_high_level_prob": 1.0,
944
- "use_zh_instruction": false,
945
- "_target_": "g05.data_processor.processor.galaxea_cot_processor.GalaxeaCoTProcessor",
946
- "discrete_action": true,
947
- "num_output_cameras": 18,
948
- "pad_token_id": 0,
949
- "image_token_index": 248056,
950
- "tokenizer_params": {
951
- "pretrained_model_name_or_path": null,
952
- "local_files_only": true,
953
- "token": null
954
- },
955
- "max_text_tokens": 160,
956
- "num_input_cameras": 3,
957
- "camera_size_config": {
958
- "exterior": [
959
- 256,
960
- 256
961
- ],
962
- "wrist_left": [
963
- 256,
964
- 256
965
- ],
966
- "wrist_right": [
967
- 256,
968
- 256
969
- ]
970
  },
971
- "samples_builder": {
972
- "_target_": "g05.data_processor.processor.samples_builder.MixedSamplesBuilder",
973
- "_partial_": true,
974
- "_recursive_": false,
975
- "candidates": [
976
- {
977
- "_target_": "g05.data_processor.processor.samples_builder.HighLevelAtomicTaskCoTBuilder",
978
- "weight": 1.0
979
- },
980
- {
981
- "_target_": "g05.data_processor.processor.samples_builder.SubtaskActionHintCoTBuilder",
982
- "weight": 1.0
983
- },
984
- {
985
- "_target_": "g05.data_processor.processor.samples_builder.AtomicTaskBaseSamplesBuilder",
986
- "weight": 1.0
987
- },
988
- {
989
- "_target_": "g05.data_processor.processor.samples_builder.Trace2DCoTBuilder",
990
- "weight": 1.0
991
- },
992
- {
993
- "_target_": "g05.data_processor.processor.samples_builder.BBoxSubtaskCoTBuilder",
994
- "weight": 1.0
995
- },
996
- {
997
- "_target_": "g05.data_processor.processor.samples_builder.BBoxCoTBuilder",
998
- "weight": 1.0
999
- },
1000
- {
1001
- "_target_": "g05.data_processor.processor.samples_builder.SubtaskCoTBuilder",
1002
- "weight": 2.0
 
 
 
 
 
 
 
 
 
 
 
1003
  },
1004
- {
1005
- "_target_": "g05.data_processor.processor.samples_builder.BaseSamplesBuilder",
1006
- "weight": 1.0
 
1007
  }
1008
- ],
1009
- "eval_builder": {
1010
- "_target_": "g05.data_processor.processor.samples_builder.BaseSamplesBuilder"
1011
- }
1012
  },
1013
- "vlm_input_action_norm_default_mode": null,
1014
- "vlm_input_action_norm_exception_mode": null
1015
- },
1016
- "cot_bbox_camera": null
1017
  },
1018
- "revision": null
1019
- }
1020
- }
1021
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1022
  }
 
1
  {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "render_runtime_messages_processor",
6
+ "config": {
7
+ "recipe": {
8
+ "messages": [
9
+ {
10
+ "role": "user",
11
+ "content": "${task}",
12
+ "stream": "low_level",
13
+ "target": false,
14
+ "if_present": null,
15
+ "tool_calls_from": null
16
+ },
17
+ {
18
+ "role": "assistant",
19
+ "content": "BBoxJSON: ${bbox}",
20
+ "stream": "low_level",
21
+ "target": true,
22
+ "if_present": "bbox",
23
+ "tool_calls_from": null
24
+ },
25
+ {
26
+ "role": "assistant",
27
+ "content": "Subtask: ${subtask}",
28
+ "stream": "low_level",
29
+ "target": true,
30
+ "if_present": "subtask",
31
+ "tool_calls_from": null
32
+ }
33
+ ],
34
+ "bindings": {
35
+ "bbox": "emitted_at(t, style=vqa, role=assistant, camera=observation.images.exterior)"
36
+ },
37
+ "blend": null,
38
+ "weight": null,
39
+ "route": null
40
+ }
41
  }
 
 
 
 
 
 
 
42
  },
43
+ {
44
+ "registry_name": "render_training_messages_processor",
45
+ "config": {
46
+ "recipe": null
47
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  },
49
+ {
50
+ "registry_name": "rename_observations_processor",
51
+ "config": {
52
+ "rename_map": {}
53
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  },
55
+ {
56
+ "registry_name": "to_batch_processor",
57
+ "config": {}
58
+ },
59
+ {
60
+ "registry_name": "g05_image_transform",
61
+ "config": {
62
+ "camera_order": [
63
+ "observation.images.head_rgb",
64
+ "observation.images.left_wrist_rgb",
65
+ "observation.images.right_wrist_rgb"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  ],
67
+ "camera_sizes": {
68
+ "observation.images.head_rgb": [
69
+ 256,
70
+ 256
71
+ ],
72
+ "observation.images.left_wrist_rgb": [
73
+ 256,
74
+ 256
75
+ ],
76
+ "observation.images.right_wrist_rgb": [
77
+ 256,
78
+ 256
79
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  },
81
+ "mean": [
 
 
82
  0.5,
83
  0.5,
84
  0.5
85
+ ],
86
+ "std": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  0.5,
88
  0.5,
89
  0.5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
  ],
91
+ "optional_camera_keys": []
92
+ }
93
+ },
94
+ {
95
+ "registry_name": "g05_action_operation_mask",
96
+ "config": {
97
+ "action_parts": [
98
+ [
99
+ "left_arm",
100
+ 6
101
+ ],
102
+ [
103
+ "left_gripper",
104
+ 1
105
+ ],
106
+ [
107
+ "right_arm",
108
+ 6
109
+ ],
110
+ [
111
+ "right_gripper",
112
+ 1
113
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  ],
115
+ "joint_threshold": null,
116
+ "gripper_threshold": null,
117
+ "velocity_threshold": 0.1,
118
+ "eef_threshold": 0.001,
119
+ "dim_thresholds": {
120
+ "left_arm": [
121
+ 0.00756,
122
+ 0.00662,
123
+ 0.0432,
124
+ 0.00732,
125
+ 0.00493,
126
+ 0.00612
127
+ ],
128
+ "right_arm": [
129
+ 0.0115,
130
+ 0.00717,
131
+ 0.0619,
132
+ 0.012,
133
+ 0.00737,
134
+ 0.00918
135
+ ]
136
+ }
137
+ }
138
+ },
139
+ {
140
+ "registry_name": "g05_relative_joint_actions",
141
+ "config": {
142
+ "enabled": true,
143
+ "exclude_joints": [
144
+ "gripper"
145
  ],
146
+ "action_names": [
147
+ "left_arm.0",
148
+ "left_arm.1",
149
+ "left_arm.2",
150
+ "left_arm.3",
151
+ "left_arm.4",
152
+ "left_arm.5",
153
+ "left_gripper.0",
154
+ "right_arm.0",
155
+ "right_arm.1",
156
+ "right_arm.2",
157
+ "right_arm.3",
158
+ "right_arm.4",
159
+ "right_arm.5",
160
+ "right_gripper.0"
161
  ],
162
+ "num_obs_steps": 6
163
+ }
164
+ },
165
+ {
166
+ "registry_name": "g05_embodiment_projection",
167
+ "config": {
168
+ "embodiment": "galaxea_r1lite",
169
+ "policy_state_dim": 27,
170
+ "policy_action_dim": 27,
171
+ "camera_order": [
172
+ "observation.images.head_rgb",
173
+ "observation.images.left_wrist_rgb",
174
+ "observation.images.right_wrist_rgb"
175
  ]
176
+ }
177
+ },
178
+ {
179
+ "registry_name": "g05_tail_normalization",
180
+ "config": {
181
+ "inverse": false,
182
+ "tail_scale": 0.075
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  },
184
+ "state_file": "policy_preprocessor_step_6_g05_tail_normalization.safetensors"
185
+ },
186
+ {
187
+ "registry_name": "normalizer_processor",
188
+ "config": {
189
+ "eps": 1e-08,
190
+ "features": {
191
+ "observation.state": {
192
+ "type": "STATE",
193
+ "shape": [
194
+ 27
195
+ ]
196
+ },
197
+ "observation.images.head_rgb": {
198
+ "type": "VISUAL",
199
+ "shape": [
200
+ 3,
201
+ 256,
202
+ 256
203
+ ]
204
+ },
205
+ "observation.images.left_wrist_rgb": {
206
+ "type": "VISUAL",
207
+ "shape": [
208
+ 3,
209
+ 256,
210
+ 256
211
+ ]
212
+ },
213
+ "observation.images.right_wrist_rgb": {
214
+ "type": "VISUAL",
215
+ "shape": [
216
+ 3,
217
+ 256,
218
+ 256
219
+ ]
220
+ },
221
+ "action": {
222
+ "type": "ACTION",
223
+ "shape": [
224
+ 27
225
+ ]
226
+ }
227
  },
228
+ "norm_map": {
229
+ "STATE": "MEAN_STD",
230
+ "ACTION": "MEAN_STD",
231
+ "VISUAL": "IDENTITY"
232
  }
 
 
 
 
233
  },
234
+ "state_file": "policy_preprocessor_step_7_normalizer_processor.safetensors"
 
 
 
235
  },
236
+ {
237
+ "registry_name": "g05_normalization_clamp",
238
+ "config": {
239
+ "minimum": -5.0,
240
+ "maximum": 5.0
241
+ }
242
+ },
243
+ {
244
+ "registry_name": "device_processor",
245
+ "config": {
246
+ "device": "cuda",
247
+ "float_dtype": null
248
+ }
249
+ },
250
+ {
251
+ "registry_name": "g05_tokenizer",
252
+ "config": {
253
+ "checkpoint_path": "lerobot/g05_base",
254
+ "revision": null
255
+ }
256
+ }
257
+ ]
258
  }