File size: 12,432 Bytes
aa4d99c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
{
  "schema_version": 1,
  "values": {
    "model.model_id": "Wan-AI/Wan2.2-TI2V-5B",
    "model.tokenizer_model_id": "Wan-AI/Wan2.2-TI2V-5B",
    "model.tokenizer_max_len": 128,
    "model.redirect_common_files": true,
    "model.arch": "rgb_dep_sem",
    "model.action_conditioning": "predicted_depth_head_semantic",
    "model.attention_layout": "rgb_auxiliary_causal",
    "model.depth_projection": "independent",
    "model.semantic_projection": "independent",
    "model.input_paradigm": "t_concat",
    "model.view_layout": "robotwin_t",
    "model.proprio_dim": 14,
    "model.video_dit_config": {
      "in_dim": 48,
      "out_dim": 48,
      "text_dim": 4096,
      "freq_dim": 256,
      "patch_size": [
        1,
        2,
        2
      ],
      "num_layers": 30,
      "has_image_input": false,
      "has_image_pos_emb": false,
      "has_ref_conv": false,
      "add_control_adapter": false,
      "in_dim_control_adapter": 24,
      "seperated_timestep": true,
      "require_vae_embedding": false,
      "require_clip_embedding": false,
      "fuse_vae_embedding_in_latents": true,
      "action_conditioned": false,
      "action_dim": 14,
      "action_group_causal_mask_mode": "group_diagonal",
      "video_attention_mask_mode": "first_frame_causal",
      "moe": {
        "enabled": true,
        "routing": "per_sample",
        "num_experts": 4,
        "top_k": 0,
        "use_shared_expert": true,
        "moe_layers": [
          21,
          24,
          27
        ],
        "router_cond": "modality",
        "token_modality_bias": true,
        "token_dispatch": "dense_masked",
        "loss_lambda_aux": 0.01,
        "loss_lambda_entropy": 0.0,
        "aux_warmup_steps": 1611,
        "aux_warmup_ratio": 0.05,
        "aux_warmup_unit": "optimizer_step",
        "init_noise_std": 0.02
      },
      "attn_head_dim": 128,
      "eps": 1e-06,
      "ffn_dim": 14336,
      "hidden_dim": 3072,
      "num_heads": 24
    },
    "model.action_dit_config": {
      "action_dim": 14,
      "text_dim": 4096,
      "freq_dim": 256,
      "num_layers": 30,
      "moe": null,
      "attn_head_dim": 128,
      "eps": 1e-06,
      "ffn_dim": 4096,
      "hidden_dim": 1024,
      "num_heads": 24
    },
    "model.video_scheduler": {
      "train_shift": 5.0,
      "infer_shift": 5.0,
      "num_train_timesteps": 1000
    },
    "model.action_scheduler": {
      "train_shift": 5.0,
      "infer_shift": 5.0,
      "num_train_timesteps": 1000
    },
    "model.wrist": {
      "view_size": [
        128,
        160
      ],
      "frame": "first",
      "attends_video": false,
      "rope_offset": null,
      "cameras": [
        "left",
        "right"
      ]
    },
    "model.loss": {
      "lambda_video": 1.0,
      "lambda_action": 1.0,
      "lambda_semantic": 1.0,
      "lambda_depth": 1.0
    },
    "model.semantic.compression": {
      "type": "svae",
      "latent_dim": 48,
      "output_normalization": "layernorm",
      "model_config": {
        "input_dim": 768,
        "latent_dim": 48,
        "num_heads": 16,
        "num_layers": 3,
        "intermediate_size": 2048,
        "dropout": 0.0
      },
      "artifact_identity": {
        "schema_version": 1,
        "encoder": {
          "backbone": "dinov3_vitb16",
          "feat_dim": 768,
          "weights_sha256": "73cec8be7427c8655ceced13ce62f6e20a1fa90d1b4d4a550df17a1144081a7c",
          "source_sha256": "c5bb90cf2c71dd5700762b97da6a6d58404fdbc2d1303e7b70b40b43b13951fb",
          "wrapper_sha256": "3d83792f6472f88feff63b15eb69bdcb43532ee2e277b29c064476c229369c93",
          "pool_sha256": "d9ee695c8c9b48e3afc6f76e6a4e4f42880dfcee3d57c73fc991b339c35c40f1",
          "layout_sha256": "399ec692097a95e559406ddaebd5e5ca4bd39693ad915047e527655c23140204",
          "feature_space": "semwam_dino_normalized_patch_features_post_temporal_pool"
        },
        "features": {
          "geometry": {
            "num_frames": 33,
            "stride": 1,
            "action_video_freq_ratio": 4,
            "layout": "robotwin_t",
            "input_paradigm": "t_concat",
            "video_size": [
              384,
              320
            ],
            "wrist_view_size": [
              128,
              160
            ],
            "wrist_frame": "first"
          }
        },
        "fingerprint": "d3383726600435ddc50852e7e12db67c393b1a2f54568803214753980bf3597d"
      },
      "state_sha256": "e8760f44c242bb79232f2d3740dda71956cac2001ba8c00ceaec77ea80e36961"
    },
    "model.semantic.backbone": "dinov3_vitb16",
    "model.semantic.feat_dim": 768,
    "model.semantic.fusion_layers": [
      7,
      23
    ],
    "model.semantic.temporal_pool": "mean",
    "model.semantic.parametrization": "v_pred",
    "model.semantic.velocity_loss_form": "direct_error",
    "model.semantic.loss_diagnostics": true,
    "model.semantic.loss_lambda_semantic": 1.0,
    "model.depth.condition_source": "dataset",
    "model.depth.target_source": "dataset",
    "model.depth.condition_temporal_context": "current_frame",
    "model.depth.repr": "dvae",
    "model.depth.estimator": "vda",
    "model.depth.intrinsics": {
      "source": "auto",
      "sim": {
        "high": {
          "fovy_deg": 37.0,
          "ref_h": 480,
          "ref_w": 640
        },
        "left": {
          "fovy_deg": 37.0,
          "ref_h": 480,
          "ref_w": 640
        },
        "right": {
          "fovy_deg": 37.0,
          "ref_h": 480,
          "ref_w": 640
        },
        "image": {
          "fovy_deg": 45.0,
          "ref_h": 224,
          "ref_w": 224
        },
        "wrist_image": {
          "fovy_deg": 75.0,
          "ref_h": 224,
          "ref_w": 224
        }
      }
    },
    "model.depth.pointmap": {
      "norm_bounds": {
        "x": [
          -0.6,
          0.6
        ],
        "y": [
          -0.6,
          0.6
        ],
        "z": [
          0.0,
          1.5
        ]
      },
      "max_depth_m": 2.0
    },
    "model.depth.colormap": {
      "name": "turbo",
      "near_m": 0.1,
      "far_m": 2.0
    },
    "model.depth.vda": {
      "backbone": "vitb",
      "input_size": 518
    },
    "model.depth.unidepth": {
      "version": "v2",
      "backbone": "vitb14",
      "resolution_level": null
    },
    "model.depth.temporal_pool": "mean",
    "model.depth.loss_lambda_depth": 1.0,
    "model.depth.vae": {
      "expected_sha256": "c869ccb52c79b398e0f173f3054cfa229f620ef108ceeec82ad0670ca8bb1957",
      "identity": {
        "type": "prism_wan22_depth",
        "weights_sha256": "c869ccb52c79b398e0f173f3054cfa229f620ef108ceeec82ad0670ca8bb1957",
        "latent_channels": 48,
        "spatial_factor": 16,
        "temporal_factor": 4,
        "input_channels": 1,
        "latent_mean": [
          0.4887385368347168,
          -2.0717055797576904,
          -0.09361350536346436,
          -1.9305212497711182,
          0.6804881691932678,
          -0.18310534954071045,
          -0.017370181158185005,
          -0.05728627368807793,
          -0.7831002473831177,
          -1.8836477994918823,
          -0.2327776700258255,
          1.9035433530807495,
          0.46046552062034607,
          -0.6453059911727905,
          -1.4464999437332153,
          0.21215873956680298,
          1.3156590461730957,
          0.007310659624636173,
          0.745159924030304,
          -0.7342119216918945,
          -0.3538833260536194,
          2.9265501499176025,
          1.9496617317199707,
          0.3119705617427826,
          -1.0393187999725342,
          -0.45994818210601807,
          -0.13466505706310272,
          -1.2324262857437134,
          0.3329582214355469,
          -0.20562906563282013,
          1.6063544750213623,
          -0.37199077010154724,
          0.10520733147859573,
          -0.005950210150331259,
          -1.021066427230835,
          -0.16209781169891357,
          -0.8151541948318481,
          2.7302606105804443,
          -0.5384726524353027,
          -3.4191009998321533,
          -0.4929030239582062,
          2.472595691680908,
          0.09065636992454529,
          0.1566804200410843,
          0.6077839732170105,
          0.16436097025871277,
          0.574763834476471,
          -1.018074631690979
        ],
        "latent_std": [
          0.8759883046150208,
          4.3842878341674805,
          0.600670576095581,
          4.464785575866699,
          1.4707190990447998,
          0.7163799405097961,
          0.39080148935317993,
          0.521407425403595,
          2.459089994430542,
          4.432198524475098,
          0.9659329056739807,
          4.131554126739502,
          1.7189385890960693,
          1.1850491762161255,
          3.289055347442627,
          0.6653669476509094,
          2.821338176727295,
          0.44802847504615784,
          1.4034730195999146,
          1.3087146282196045,
          0.9007586240768433,
          6.145054340362549,
          3.761077404022217,
          1.2898753881454468,
          2.543177366256714,
          1.234912633895874,
          0.765889585018158,
          2.0544753074645996,
          0.9371235966682434,
          0.6772623062133789,
          3.1669251918792725,
          0.5857013463973999,
          0.5159927606582642,
          0.5009284615516663,
          2.351271152496338,
          0.6470572352409363,
          1.8693331480026245,
          5.487253189086914,
          1.4262751340866089,
          7.51733922958374,
          0.8003208637237549,
          4.9757843017578125,
          0.6112902760505676,
          0.5305010676383972,
          1.6170129776000977,
          0.4477066993713379,
          1.232177972793579,
          2.2924752235412598
        ],
        "preprocess": {
          "unit": "metres",
          "near_m": 0.0,
          "far_m": 1.5,
          "resize": "bilinear",
          "align_corners": false,
          "layout": "robotwin_t",
          "normalization": "linear_minus_one_one",
          "posterior": "mean"
        }
      }
    },
    "model.depth.defm.backbone": "defm_vit_s14",
    "model.depth.defm.feat_dim": 384,
    "model.depth.defm.patch_size": 14,
    "data.train.num_frames": 33,
    "data.train.action_video_freq_ratio": 4,
    "data.normalization_statistics_sha256": "82b5328e093511ec35dd52ba17a622d5357fa385903c49dd0a544fceb7b5e243",
    "data.train.processor.shape_meta": {
      "images": [
        {
          "key": "cam_high",
          "raw_shape": [
            3,
            240,
            320
          ],
          "shape": [
            3,
            240,
            320
          ]
        },
        {
          "key": "cam_left_wrist",
          "raw_shape": [
            3,
            240,
            320
          ],
          "shape": [
            3,
            240,
            320
          ]
        },
        {
          "key": "cam_right_wrist",
          "raw_shape": [
            3,
            240,
            320
          ],
          "shape": [
            3,
            240,
            320
          ]
        }
      ],
      "action": [
        {
          "key": "default",
          "raw_shape": 14,
          "shape": 14
        }
      ],
      "state": [
        {
          "key": "default",
          "raw_shape": 14,
          "shape": 14
        }
      ]
    },
    "data.train.processor.num_obs_steps": 33,
    "data.train.processor.num_output_cameras": 3,
    "data.train.processor.action_output_dim": 14,
    "data.train.processor.proprio_output_dim": 14,
    "data.train.processor.use_stepwise_action_norm": false,
    "data.train.processor.norm_default_mode": "z-score",
    "data.train.processor.action_state_merger": {
      "_target_": "wam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign"
    },
    "data.train.processor.train_transforms": [
      {
        "_target_": "wam.datasets.lerobot.transforms.image.ToTensor"
      },
      {
        "_target_": "torchvision.transforms.Resize",
        "size": [
          240,
          320
        ]
      }
    ],
    "data.train.processor.val_transforms": [
      {
        "_target_": "wam.datasets.lerobot.transforms.image.ToTensor"
      },
      {
        "_target_": "torchvision.transforms.Resize",
        "size": [
          240,
          320
        ]
      }
    ],
    "model.model_family": "wam",
    "model.depth.fusion_layers": null
  }
}