Robotics
Safetensors
world-action-models
robotwin
FreDERicia commited on
Commit
b33f9f2
·
verified ·
1 Parent(s): 476af01

Add verified posttrain artifacts

Browse files
posttrain/config.json ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_dim": 16,
3
+ "action_head_cfg": {
4
+ "_convert_": "object",
5
+ "_target_": "groot.vla.model.dreamzero.action_head.wan_flow_matching_action_tf.WANPolicyHead",
6
+ "config": {
7
+ "_recursive_": false,
8
+ "_target_": "groot.vla.model.dreamzero.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig",
9
+ "action_dim": 16,
10
+ "action_horizon": 48,
11
+ "action_inference_sigma_shift": null,
12
+ "action_loss_embodiment_ids": [
13
+ 26,
14
+ 17
15
+ ],
16
+ "action_train_sigma_shift": null,
17
+ "add_pos_embed": true,
18
+ "backbone_embedding_dim": 0,
19
+ "backbone_features_projector_cfg": null,
20
+ "decouple_inference_noise": false,
21
+ "decouple_video_action_noise": false,
22
+ "defer_lora_injection": false,
23
+ "diffusion_model_cfg": {
24
+ "_convert_": "object",
25
+ "_target_": "groot.vla.model.dreamzero.modules.wan_video_dit_action_casual_chunk.CausalWanModel",
26
+ "action_architecture": "full",
27
+ "action_condition_on_text": false,
28
+ "action_dim": 16,
29
+ "action_ffn_dim": 4096,
30
+ "action_hidden_size": 1024,
31
+ "concat_first_frame_latent": false,
32
+ "diffusion_model_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.2-TI2V-5B",
33
+ "dim": 3072,
34
+ "eps": 1e-06,
35
+ "ffn_dim": 14336,
36
+ "frame_seqlen": 80,
37
+ "freq_dim": 256,
38
+ "in_dim": 48,
39
+ "max_chunk_size": 4,
40
+ "max_state_dim": 16,
41
+ "model_type": "ti2v",
42
+ "num_action_per_block": 48,
43
+ "num_frame_per_block": 2,
44
+ "num_heads": 24,
45
+ "num_layers": 30,
46
+ "num_state_per_block": 1,
47
+ "out_dim": 48
48
+ },
49
+ "expand_batch": null,
50
+ "freeze_decode_layer": false,
51
+ "hidden_size": 64,
52
+ "high_noise_beta_alpha": 3.0,
53
+ "image_encoder_cfg": {
54
+ "_convert_": "object",
55
+ "_target_": "groot.vla.model.dreamzero.modules.wan_video_image_encoder.WanImageEncoder",
56
+ "image_encoder_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"
57
+ },
58
+ "init_lora_weights": "kaiming",
59
+ "input_embedding_dim": 1536,
60
+ "load_pretrained_det_decode_layer_path": null,
61
+ "lora_alpha": 4,
62
+ "lora_rank": 4,
63
+ "lora_target_modules": "q,k,v,o,ffn.0,ffn.2",
64
+ "max_action_dim": 16,
65
+ "max_state_dim": 16,
66
+ "model_dtype": "float32",
67
+ "noise_beta_alpha": 1.5,
68
+ "noise_beta_beta": 1.0,
69
+ "noise_s": 0.999,
70
+ "num_frame_per_block": 2,
71
+ "num_frames": 33,
72
+ "num_inference_timesteps": 16,
73
+ "num_timestep_buckets": 1000,
74
+ "repa_coeff": 1.0,
75
+ "repa_layer": 8,
76
+ "skip_component_loading": true,
77
+ "target_video_height": 256,
78
+ "target_video_width": 320,
79
+ "text_encoder_cfg": {
80
+ "_convert_": "object",
81
+ "_target_": "groot.vla.model.dreamzero.modules.wan_video_text_encoder.WanTextEncoder",
82
+ "text_encoder_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth"
83
+ },
84
+ "tile_size_height": 34,
85
+ "tile_size_width": 34,
86
+ "tile_stride_height": 18,
87
+ "tile_stride_width": 16,
88
+ "tiled": false,
89
+ "train_architecture": "full",
90
+ "tune_diffusion_model": true,
91
+ "tune_projector": true,
92
+ "use_clip_image_encoder": false,
93
+ "use_episode_first_frame_token": false,
94
+ "use_gradient_checkpointing": true,
95
+ "use_high_noise_emphasis": false,
96
+ "use_past_action_context": false,
97
+ "use_vlln": true,
98
+ "vae_cfg": {
99
+ "_convert_": "object",
100
+ "_target_": "groot.vla.model.dreamzero.modules.wan_video_vae.WanVideoVAE38",
101
+ "dim": 160,
102
+ "vae_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
103
+ "z_dim": 48
104
+ },
105
+ "video_condition_on_action_state": false,
106
+ "video_inference_final_noise": 0.8,
107
+ "video_noise_beta_alpha": 3.0,
108
+ "video_noise_beta_beta": 1.0,
109
+ "vl_self_attention_cfg": {
110
+ "_target_": "groot.vla.model.n1_5.modules.cross_attention_dit.SelfAttentionTransformer",
111
+ "attention_head_dim": 64,
112
+ "dropout": 0.2,
113
+ "final_dropout": true,
114
+ "num_attention_heads": 24,
115
+ "num_layers": 4,
116
+ "positional_embeddings": null
117
+ }
118
+ }
119
+ },
120
+ "action_horizon": 48,
121
+ "action_inference_sigma_shift": null,
122
+ "action_quaternion_mode": "temporal_continuous",
123
+ "action_train_sigma_shift": null,
124
+ "architectures": [
125
+ "VLA"
126
+ ],
127
+ "backbone_cfg": {
128
+ "_target_": "groot.vla.model.dreamzero.backbone.identity.IdentityBackbone"
129
+ },
130
+ "diffusion_model_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.2-TI2V-5B",
131
+ "dtype": "bfloat16",
132
+ "embodiment_tag": "robotwin",
133
+ "embodiment_tag_mapping": {
134
+ "robotwin": 33
135
+ },
136
+ "has_real_action": true,
137
+ "image_encoder_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth",
138
+ "image_resolution_height": 128,
139
+ "image_resolution_width": 160,
140
+ "is_lora": false,
141
+ "load_to_device": false,
142
+ "max_action_dim": 16,
143
+ "max_state_dim": 16,
144
+ "metadata_embodiment_tag": "robotwin",
145
+ "metadata_json_path": ".assets/robotwin_metadata_temporal_continuous.json",
146
+ "model_path": "/inspire/hdd3/project/embodied-model/public/zy/ckpts-rlinf-warm/25-4_8n64g_wo_img_robotics_umi_episode_relative_ego_1to1to1_from_video20k_pretrain80k_dmuon_r01_global_step_80000",
147
+ "model_type": "vla",
148
+ "num_action_per_block": 48,
149
+ "precision": "fp32",
150
+ "relative_action": false,
151
+ "relative_action_keys": [],
152
+ "relative_action_mode": "",
153
+ "relative_action_per_horizon": false,
154
+ "relative_action_pose_reference": "per_arm",
155
+ "state_horizon": 1,
156
+ "target_video_height": 256,
157
+ "target_video_width": 320,
158
+ "text_encoder_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth",
159
+ "tokenizer_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/umt5-xxl",
160
+ "transformers_version": "4.57.6",
161
+ "vae_pretrained_path": "/inspire/hdd/project/robot-dna/public/xuzijun/checkpoints_shared/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
162
+ "model_dtype": "float32"
163
+ }
posttrain/experiment_cfg/metadata.json ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "robotwin": {
3
+ "statistics": {
4
+ "state": {
5
+ "eef_position": {
6
+ "min": [
7
+ -0.7182875275611877,
8
+ -0.9362404942512512,
9
+ 0.5216488838195801,
10
+ 1.0730232133937534e-06,
11
+ -0.9643872976303101,
12
+ -0.5892208814620972,
13
+ -0.954880952835083,
14
+ 0.0,
15
+ -0.16835452616214752,
16
+ -0.45709583163261414,
17
+ 0.773105800151825,
18
+ 1.7880972791317618e-06,
19
+ -0.7881463766098022,
20
+ -0.9556690454483032,
21
+ -0.9836600422859192,
22
+ 0.0
23
+ ],
24
+ "max": [
25
+ 0.278211385011673,
26
+ 0.19547173380851746,
27
+ 1.4917669296264648,
28
+ 0.9999999403953552,
29
+ 0.8720628023147583,
30
+ 0.7767562866210938,
31
+ 0.9667533040046692,
32
+ 1.0,
33
+ 0.5140635371208191,
34
+ 0.18329843878746033,
35
+ 1.363722562789917,
36
+ 0.9694916009902954,
37
+ 0.7613260746002197,
38
+ 0.8820517063140869,
39
+ 0.9988598227500916,
40
+ 1.0
41
+ ],
42
+ "mean": [
43
+ -0.21393757445484884,
44
+ -0.18018024925430667,
45
+ 0.9591720669531342,
46
+ 0.6657184366512593,
47
+ -0.145775895275691,
48
+ 0.22337473917836473,
49
+ 0.5182260787068643,
50
+ 0.6569526437650256,
51
+ 0.22534439442337867,
52
+ -0.19138290441328104,
53
+ 0.9573524040307536,
54
+ 0.5515658075690234,
55
+ -0.17943925228058685,
56
+ 0.15273813794650004,
57
+ 0.6323696914892638,
58
+ 0.6856614049710095
59
+ ],
60
+ "std": [
61
+ 0.1077653936488303,
62
+ 0.12799604618198288,
63
+ 0.04921374601292238,
64
+ 0.14465284580849633,
65
+ 0.24838880209279898,
66
+ 0.26932902318395424,
67
+ 0.24890610618437933,
68
+ 0.45731486376258446,
69
+ 0.10688259629182566,
70
+ 0.12845018113033632,
71
+ 0.05033513427253471,
72
+ 0.191071209455465,
73
+ 0.2880768801710076,
74
+ 0.2423747114542121,
75
+ 0.2492274785300869,
76
+ 0.4475877923266417
77
+ ],
78
+ "q01": [
79
+ -0.3597920000553131,
80
+ -0.31383857131004333,
81
+ 0.8465567111968995,
82
+ 0.09784904420375824,
83
+ -0.6915426790714264,
84
+ -0.16546517461538315,
85
+ -0.5033752143383026,
86
+ 0.0,
87
+ -0.05245155692100525,
88
+ -0.3128008544445038,
89
+ 0.8273665070533752,
90
+ 0.0026048718951642512,
91
+ -0.7053335785865784,
92
+ -0.5005193710327148,
93
+ -0.7049345791339874,
94
+ 0.0
95
+ ],
96
+ "q99": [
97
+ 0.052491794526576596,
98
+ 0.08722782507538754,
99
+ 1.0893360376358032,
100
+ 0.988396817445755,
101
+ 0.5707527399063042,
102
+ 0.7072686314582824,
103
+ 0.7211236953735352,
104
+ 1.0,
105
+ 0.34580310285091354,
106
+ 0.07947802022099491,
107
+ 1.1244234442710872,
108
+ 0.7071852028369903,
109
+ 0.7017662584781647,
110
+ 0.7048980951309202,
111
+ 0.9434472620487213,
112
+ 1.0
113
+ ]
114
+ }
115
+ },
116
+ "action": {
117
+ "eef_position": {
118
+ "min": [
119
+ -0.7182875275611877,
120
+ -0.9362404942512512,
121
+ 0.5216488838195801,
122
+ -0.9375947117805481,
123
+ -0.9643872976303101,
124
+ -0.7177870869636536,
125
+ -0.8373916745185852,
126
+ 0.0,
127
+ -0.16845698654651642,
128
+ -0.45709583163261414,
129
+ 0.773105800151825,
130
+ -0.7287213206291199,
131
+ -0.7881523370742798,
132
+ -0.7602484226226807,
133
+ -0.998636782169342,
134
+ 0.0
135
+ ],
136
+ "max": [
137
+ 0.278211385011673,
138
+ 0.19547173380851746,
139
+ 1.4917669296264648,
140
+ 0.9999999403953552,
141
+ 0.9531664252281189,
142
+ 0.7767562866210938,
143
+ 0.9667533040046692,
144
+ 1.0,
145
+ 0.5140635371208191,
146
+ 0.18329843878746033,
147
+ 1.363722562789917,
148
+ 0.9694916009902954,
149
+ 0.7613260746002197,
150
+ 0.9556690454483032,
151
+ 0.9988598227500916,
152
+ 1.0
153
+ ],
154
+ "mean": [
155
+ -0.21353513675780292,
156
+ -0.1796377155976042,
157
+ 0.9592759076545144,
158
+ 0.6636497914620075,
159
+ -0.15162495071132723,
160
+ 0.22370221198429288,
161
+ 0.5225168700564277,
162
+ 0.6556916165714068,
163
+ 0.22484614369187772,
164
+ -0.19080697545613162,
165
+ 0.9574205329654474,
166
+ 0.5475672742883396,
167
+ -0.18815311070686913,
168
+ 0.15217908835634242,
169
+ 0.6406216851117243,
170
+ 0.6845670652799978
171
+ ],
172
+ "std": [
173
+ 0.10790538494159982,
174
+ 0.1279323874984017,
175
+ 0.04930913952922822,
176
+ 0.15326060276523396,
177
+ 0.24578360684907702,
178
+ 0.26990545065381244,
179
+ 0.2382539193246631,
180
+ 0.4577457127406035,
181
+ 0.10705815887162189,
182
+ 0.12846055296769895,
183
+ 0.05047166688744138,
184
+ 0.20050367247493628,
185
+ 0.2833043708682629,
186
+ 0.24375372667811027,
187
+ 0.22657202831804493,
188
+ 0.4480401628755786
189
+ ],
190
+ "q01": [
191
+ -0.3599411517381668,
192
+ -0.31383857131004333,
193
+ 0.8465381443500519,
194
+ 0.018337715230882175,
195
+ -0.6998488366603851,
196
+ -0.16579440236091614,
197
+ -0.17371345609426497,
198
+ 0.0,
199
+ -0.05254153162240982,
200
+ -0.3128008544445038,
201
+ 0.8259249150753021,
202
+ -0.0018913027015514672,
203
+ -0.706351363658905,
204
+ -0.5018746137619018,
205
+ -0.698279219865799,
206
+ 0.0
207
+ ],
208
+ "q99": [
209
+ 0.052819954231381326,
210
+ 0.08722782507538754,
211
+ 1.0894059300422667,
212
+ 0.988396942615509,
213
+ 0.5127807438373565,
214
+ 0.707279771566391,
215
+ 0.721123993396759,
216
+ 1.0,
217
+ 0.3462378948926921,
218
+ 0.07948291599750487,
219
+ 1.1245562195777892,
220
+ 0.7072111666202545,
221
+ 0.6956618428230285,
222
+ 0.7053033471107478,
223
+ 0.943844985961914,
224
+ 1.0
225
+ ]
226
+ }
227
+ }
228
+ },
229
+ "modalities": {
230
+ "video": {
231
+ "cam_high": {
232
+ "resolution": [
233
+ 640,
234
+ 480
235
+ ],
236
+ "channels": 3,
237
+ "fps": 50.0
238
+ },
239
+ "cam_left_wrist": {
240
+ "resolution": [
241
+ 640,
242
+ 480
243
+ ],
244
+ "channels": 3,
245
+ "fps": 50.0
246
+ },
247
+ "cam_right_wrist": {
248
+ "resolution": [
249
+ 640,
250
+ 480
251
+ ],
252
+ "channels": 3,
253
+ "fps": 50.0
254
+ }
255
+ },
256
+ "state": {
257
+ "eef_position": {
258
+ "absolute": true,
259
+ "rotation_type": null,
260
+ "shape": [
261
+ 16
262
+ ],
263
+ "continuous": true
264
+ }
265
+ },
266
+ "action": {
267
+ "eef_position": {
268
+ "absolute": true,
269
+ "rotation_type": null,
270
+ "shape": [
271
+ 16
272
+ ],
273
+ "continuous": true
274
+ }
275
+ }
276
+ },
277
+ "embodiment_tag": "robotwin",
278
+ "action_quaternion_mode": "temporal_continuous"
279
+ }
280
+ }
posttrain/manifest.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "path": "posttrain/model-00001-of-00007.safetensors",
4
+ "size": 4213157538,
5
+ "sha256": "65349cd83fbd24e4c369b4ea021e869a05e4a7e1ea542722fc56d975305c1f97"
6
+ },
7
+ {
8
+ "path": "posttrain/model-00002-of-00007.safetensors",
9
+ "size": 4293315672,
10
+ "sha256": "342902906f24694a34ddb8b0c86496876a12c046f48a3cbba1064ab2917b9c1a"
11
+ },
12
+ {
13
+ "path": "posttrain/model-00003-of-00007.safetensors",
14
+ "size": 4284998344,
15
+ "sha256": "4e8e1977b776c083e46796f1ca0120943b0ddea564b4b8ad24362d12ebfdb5f4"
16
+ },
17
+ {
18
+ "path": "posttrain/model-00004-of-00007.safetensors",
19
+ "size": 4259474896,
20
+ "sha256": "6a6e3ea537728fdef73b0ddf05f98949099444572107df371386d7dbfe7200ff"
21
+ },
22
+ {
23
+ "path": "posttrain/model-00005-of-00007.safetensors",
24
+ "size": 4244874968,
25
+ "sha256": "f48ff0ebba854fdb7d98dcc4f40ddfeecd010f2646d88b0510136b54285d9967"
26
+ },
27
+ {
28
+ "path": "posttrain/model-00006-of-00007.safetensors",
29
+ "size": 3171097104,
30
+ "sha256": "52b136b27a983ddf6649955a0a4d1185c7dbbe13e914ccb33e444ea942a59cf1"
31
+ },
32
+ {
33
+ "path": "posttrain/model-00007-of-00007.safetensors",
34
+ "size": 3509703568,
35
+ "sha256": "0d053921e4ef4915882e65d5088309a5b3081265f116b7d4ba159395e60853a2"
36
+ },
37
+ {
38
+ "path": "posttrain/model.safetensors.index.json",
39
+ "size": 254918,
40
+ "sha256": "a142765baa387372623f0dab67eeae79a03ecd84855d915ad8275be314f1bf01"
41
+ },
42
+ {
43
+ "path": "posttrain/config.json",
44
+ "size": 6420,
45
+ "sha256": "55792f7100f7cb5956a7c9d47ace897dd16efa007e637969d81fa5c92eff1e6a"
46
+ },
47
+ {
48
+ "path": "posttrain/experiment_cfg/metadata.json",
49
+ "size": 10632,
50
+ "sha256": "6dd94e976b4e689a320dd99443fb9a4f8254ed9ff1c326c8124977018d093247"
51
+ }
52
+ ]