SemWAM-RoboTwin-Clean2Random-v1 / model_card_summary.json
RainyBot's picture
Publish validated final step 32220
aa4d99c verified
Raw History Blame Contribute Delete
21.9 kB
{
"schema_version": 2,
"identity_source": "saved inference_config",
"model": {
"model.arch": "rgb_dep_sem",
"model.attention_layout": "rgb_auxiliary_causal",
"model.action_conditioning": "predicted_depth_head_semantic",
"model.world_action_visibility": "none",
"model.input_paradigm": "t_concat",
"model.view_layout": "robotwin_t",
"model.depth.repr": "dvae",
"model.depth.condition_source": "dataset",
"model.depth.target_source": "dataset",
"model.depth.condition_temporal_context": "current_frame",
"model.depth_projection": "independent",
"model.semantic_projection": "independent",
"model.semantic.parametrization": "v_pred",
"model.semantic.compression": "svae",
"model.semantic.channels": 48,
"model.semantic.compression.output_normalization": "layernorm"
},
"historical_restoration_defaults": {
"model.world_action_visibility": {
"saved": null,
"effective": "none",
"reason": "historical checkpoint restoration rule"
}
},
"data": {
"releases": [],
"scene_filter": null,
"scene_order": null,
"episode_split": null,
"train_episode_count": null,
"train_episodes_sha256": null,
"validation_fraction": null,
"depth_codec": null,
"action_window_frames": 33,
"action_video_freq_ratio": 4,
"global_sample_stride": null,
"temporal_sampling": null,
"normalization_statistics_fingerprint": "82b5328e093511ec35dd52ba17a622d5357fa385903c49dd0a544fceb7b5e243"
},
"training": {
"num_epochs": null,
"learning_rate": null,
"batch_size": null,
"gradient_accumulation_steps": null,
"mixed_precision": null,
"seed": null,
"total_optimizer_steps": null,
"execution": {},
"world_size": null,
"global_batch_size": null,
"runtime_reference": {
"global_step": 32220,
"scheduler_type": null,
"sched_total_train_steps": null,
"sched_warmup_steps": null,
"lr_conv_ratio": null,
"enforce_grad_clipping": null,
"effective_grad_clip": null,
"video_moe_warmup": null
}
},
"augmentation": {
"enabled": null,
"patchwam": null,
"conditioning_latent_noise": null,
"color": {
"brightness": null,
"contrast": null,
"saturation": null,
"hue": null,
"probability": null
},
"geometry": {},
"coarse_dropout": null,
"effective_geometry_enabled": null,
"evaluation_random_augmentation": false
},
"frozen_assets": {
"semantic_compressor_state_sha256": "e8760f44c242bb79232f2d3740dda71956cac2001ba8c00ceaec77ea80e36961",
"semantic_artifact_fingerprint": "d3383726600435ddc50852e7e12db67c393b1a2f54568803214753980bf3597d",
"depth_vae_identity": {
"type": "prism_wan22_depth",
"weights_sha256": "c869ccb52c79b398e0f173f3054cfa229f620ef108ceeec82ad0670ca8bb1957",
"latent_channels": 48,
"spatial_factor": 16,
"temporal_factor": 4,
"input_channels": 1,
"latent_mean": [
0.4887385368347168,
-2.0717055797576904,
-0.09361350536346436,
-1.9305212497711182,
0.6804881691932678,
-0.18310534954071045,
-0.017370181158185005,
-0.05728627368807793,
-0.7831002473831177,
-1.8836477994918823,
-0.2327776700258255,
1.9035433530807495,
0.46046552062034607,
-0.6453059911727905,
-1.4464999437332153,
0.21215873956680298,
1.3156590461730957,
0.007310659624636173,
0.745159924030304,
-0.7342119216918945,
-0.3538833260536194,
2.9265501499176025,
1.9496617317199707,
0.3119705617427826,
-1.0393187999725342,
-0.45994818210601807,
-0.13466505706310272,
-1.2324262857437134,
0.3329582214355469,
-0.20562906563282013,
1.6063544750213623,
-0.37199077010154724,
0.10520733147859573,
-0.005950210150331259,
-1.021066427230835,
-0.16209781169891357,
-0.8151541948318481,
2.7302606105804443,
-0.5384726524353027,
-3.4191009998321533,
-0.4929030239582062,
2.472595691680908,
0.09065636992454529,
0.1566804200410843,
0.6077839732170105,
0.16436097025871277,
0.574763834476471,
-1.018074631690979
],
"latent_std": [
0.8759883046150208,
4.3842878341674805,
0.600670576095581,
4.464785575866699,
1.4707190990447998,
0.7163799405097961,
0.39080148935317993,
0.521407425403595,
2.459089994430542,
4.432198524475098,
0.9659329056739807,
4.131554126739502,
1.7189385890960693,
1.1850491762161255,
3.289055347442627,
0.6653669476509094,
2.821338176727295,
0.44802847504615784,
1.4034730195999146,
1.3087146282196045,
0.9007586240768433,
6.145054340362549,
3.761077404022217,
1.2898753881454468,
2.543177366256714,
1.234912633895874,
0.765889585018158,
2.0544753074645996,
0.9371235966682434,
0.6772623062133789,
3.1669251918792725,
0.5857013463973999,
0.5159927606582642,
0.5009284615516663,
2.351271152496338,
0.6470572352409363,
1.8693331480026245,
5.487253189086914,
1.4262751340866089,
7.51733922958374,
0.8003208637237549,
4.9757843017578125,
0.6112902760505676,
0.5305010676383972,
1.6170129776000977,
0.4477066993713379,
1.232177972793579,
2.2924752235412598
],
"preprocess": {
"unit": "metres",
"near_m": 0.0,
"far_m": 1.5,
"resize": "bilinear",
"align_corners": false,
"layout": "robotwin_t",
"normalization": "linear_minus_one_one",
"posterior": "mean"
}
}
},
"saved_construction_and_objectives": {
"model.video_dit_config": {
"in_dim": 48,
"out_dim": 48,
"text_dim": 4096,
"freq_dim": 256,
"patch_size": [
1,
2,
2
],
"num_layers": 30,
"has_image_input": false,
"has_image_pos_emb": false,
"has_ref_conv": false,
"add_control_adapter": false,
"in_dim_control_adapter": 24,
"seperated_timestep": true,
"require_vae_embedding": false,
"require_clip_embedding": false,
"fuse_vae_embedding_in_latents": true,
"action_conditioned": false,
"action_dim": 14,
"action_group_causal_mask_mode": "group_diagonal",
"video_attention_mask_mode": "first_frame_causal",
"moe": {
"enabled": true,
"routing": "per_sample",
"num_experts": 4,
"top_k": 0,
"use_shared_expert": true,
"moe_layers": [
21,
24,
27
],
"router_cond": "modality",
"token_modality_bias": true,
"token_dispatch": "dense_masked",
"loss_lambda_aux": 0.01,
"loss_lambda_entropy": 0.0,
"aux_warmup_steps": 1611,
"aux_warmup_ratio": 0.05,
"aux_warmup_unit": "optimizer_step",
"init_noise_std": 0.02
},
"attn_head_dim": 128,
"eps": 1e-06,
"ffn_dim": 14336,
"hidden_dim": 3072,
"num_heads": 24
},
"model.action_dit_config": {
"action_dim": 14,
"text_dim": 4096,
"freq_dim": 256,
"num_layers": 30,
"moe": null,
"attn_head_dim": 128,
"eps": 1e-06,
"ffn_dim": 4096,
"hidden_dim": 1024,
"num_heads": 24
},
"model.loss": {
"lambda_video": 1.0,
"lambda_action": 1.0,
"lambda_semantic": 1.0,
"lambda_depth": 1.0
},
"model.semantic.velocity_loss_form": "direct_error",
"model.video_scheduler": {
"train_shift": 5.0,
"infer_shift": 5.0,
"num_train_timesteps": 1000
},
"model.action_scheduler": {
"train_shift": 5.0,
"infer_shift": 5.0,
"num_train_timesteps": 1000
},
"model.semantic.backbone": "dinov3_vitb16",
"model.semantic.feature_normalization": null,
"model.semantic.temporal_pool": "mean",
"model.depth.pointmap": {
"norm_bounds": {
"x": [
-0.6,
0.6
],
"y": [
-0.6,
0.6
],
"z": [
0.0,
1.5
]
},
"max_depth_m": 2.0
},
"model.depth.intrinsics": {
"source": "auto",
"sim": {
"high": {
"fovy_deg": 37.0,
"ref_h": 480,
"ref_w": 640
},
"left": {
"fovy_deg": 37.0,
"ref_h": 480,
"ref_w": 640
},
"right": {
"fovy_deg": 37.0,
"ref_h": 480,
"ref_w": 640
},
"image": {
"fovy_deg": 45.0,
"ref_h": 224,
"ref_w": 224
},
"wrist_image": {
"fovy_deg": 75.0,
"ref_h": 224,
"ref_w": 224
}
}
}
},
"archived_run_configuration": {
"source": "original resolved run/config.yaml; never current recipe defaults",
"missing_fields": "unknown/not recorded; null is not false",
"inactive_fields": "recorded values may be inactive; use arch and enabled/mode gates",
"choices": {
"Architecture and Observation": {
"model.model_family": "wam",
"model.model_id": "Wan-AI/Wan2.2-TI2V-5B",
"model.arch": "rgb_dep_sem",
"model.attention_layout": "rgb_auxiliary_causal",
"model.action_conditioning": "predicted_depth_head_semantic",
"model.world_action_visibility": null,
"model.input_paradigm": "t_concat",
"model.view_layout": "robotwin_t",
"model.depth_projection": "independent",
"model.semantic_projection": "independent",
"model.wrist": {
"view_size": [
128,
160
],
"frame": "first",
"attends_video": false,
"rope_offset": null,
"cameras": [
"left",
"right"
]
},
"model.proprio_dim": 14,
"model.video_dit_config.video_attention_mask_mode": "first_frame_causal",
"model.video_dit_config.action_conditioned": false,
"model.semantic.fusion_layers": [
7,
23
],
"model.depth.fusion_layers": null
},
"Representations and Frozen Encoders": {
"model.semantic.backbone": "dinov3_vitb16",
"model.semantic.feat_dim": 768,
"model.semantic.feature_normalization": null,
"model.semantic.temporal_pool": "mean",
"model.semantic.clip_frames": null,
"model.semantic.image_size": null,
"model.semantic.compression.type": "svae",
"model.semantic.compression.latent_dim": 48,
"model.semantic.compression.output_normalization": "layernorm",
"model.semantic.compression.model_config": {
"input_dim": 768,
"latent_dim": 48,
"num_heads": 16,
"num_layers": 3,
"intermediate_size": 2048,
"dropout": 0.0
},
"model.depth.repr": "dvae",
"model.depth.condition_source": "dataset",
"model.depth.target_source": "dataset",
"model.depth.condition_temporal_context": "current_frame",
"model.depth.estimator": "vda",
"model.depth.vda": {
"backbone": "vitb",
"input_size": 518
},
"model.depth.unidepth": {
"version": "v2",
"backbone": "vitb14",
"resolution_level": null
},
"model.depth.pointmap": {
"norm_bounds": {
"x": [
-0.6,
0.6
],
"y": [
-0.6,
0.6
],
"z": [
0.0,
1.5
]
},
"max_depth_m": 2.0
},
"model.depth.intrinsics": {
"source": "auto",
"sim": {
"high": {
"fovy_deg": 37.0,
"ref_h": 480,
"ref_w": 640
},
"left": {
"fovy_deg": 37.0,
"ref_h": 480,
"ref_w": 640
},
"right": {
"fovy_deg": 37.0,
"ref_h": 480,
"ref_w": 640
},
"image": {
"fovy_deg": 45.0,
"ref_h": 224,
"ref_w": 224
},
"wrist_image": {
"fovy_deg": 75.0,
"ref_h": 224,
"ref_w": 224
}
}
},
"model.depth.vae.identity.type": "prism_wan22_depth",
"model.depth.vae.identity.latent_channels": 48,
"model.depth.vae.identity.input_channels": 1,
"model.depth.vae.identity.spatial_factor": 16,
"model.depth.vae.identity.temporal_factor": 4,
"model.depth.vae.identity.preprocess": {
"unit": "metres",
"near_m": 0.0,
"far_m": 1.5,
"resize": "bilinear",
"align_corners": false,
"layout": "robotwin_t",
"normalization": "linear_minus_one_one",
"posterior": "mean"
},
"model.depth.colormap": {
"name": "turbo",
"near_m": 0.1,
"far_m": 2.0
},
"model.depth.defm": {
"backbone": "defm_vit_s14",
"feat_dim": 384,
"patch_size": 14
}
},
"Backbone Capacity and MoE": {
"model.video_dit_config": {
"in_dim": 48,
"out_dim": 48,
"text_dim": 4096,
"freq_dim": 256,
"patch_size": [
1,
2,
2
],
"num_layers": 30,
"has_image_input": false,
"has_image_pos_emb": false,
"has_ref_conv": false,
"add_control_adapter": false,
"in_dim_control_adapter": 24,
"seperated_timestep": true,
"require_vae_embedding": false,
"require_clip_embedding": false,
"fuse_vae_embedding_in_latents": true,
"action_conditioned": false,
"action_dim": 14,
"action_group_causal_mask_mode": "group_diagonal",
"video_attention_mask_mode": "first_frame_causal",
"moe": {
"enabled": true,
"routing": "per_sample",
"num_experts": 4,
"top_k": 0,
"use_shared_expert": true,
"moe_layers": [
21,
24,
27
],
"router_cond": "modality",
"token_modality_bias": true,
"token_dispatch": "dense_masked",
"loss_lambda_aux": 0.01,
"loss_lambda_entropy": 0.0,
"aux_warmup_steps": 1611,
"aux_warmup_ratio": 0.05,
"aux_warmup_unit": "optimizer_step",
"init_noise_std": 0.02
},
"attn_head_dim": 128,
"eps": 1e-06,
"ffn_dim": 14336,
"hidden_dim": 3072,
"num_heads": 24
},
"model.action_dit_config": {
"action_dim": 14,
"text_dim": 4096,
"freq_dim": 256,
"num_layers": 30,
"moe": null,
"attn_head_dim": 128,
"eps": 1e-06,
"ffn_dim": 4096,
"hidden_dim": 1024,
"num_heads": 24
}
},
"Dataset Scope and Split": {
"data.train.expected_dataset_family": null,
"data.train.scene_filter": null,
"data.train.scene_order": null,
"data.train.selected_tasks": null,
"data.train.selected_episodes": null,
"data.train.selected_episode_ranges": null,
"data.train.episode_split": null,
"data.train.val_set_proportion": null,
"data.train.depth_codec": null,
"data.train.shape_meta": null,
"data.train.processor.action_output_dim": 14,
"data.train.processor.proprio_output_dim": 14,
"data.train.processor.use_stepwise_action_norm": false,
"data.train.processor.norm_default_mode": "z-score",
"data.train.processor.norm_exception_mode": null,
"data.train.processor.action_state_transforms": null,
"statistics_source": "null/not recorded in config; use saved statistics identity, not a guessed refit",
"data.releases": null
},
"Windows and Sample Selection": {
"data.train.num_frames": 33,
"data.train.action_video_freq_ratio": 4,
"data.train.global_sample_stride": null,
"data.train.video_size": null,
"data.train.skip_padding_as_possible": null,
"data.train.temporal_sampling": null,
"data.val.temporal_sampling": null,
"data.train.semantic_compute_mode": null,
"data.train.depth_compute_mode": null,
"model.semantic.compute_mode": null,
"model.depth.compute_mode": null,
"data.vae_cache.enabled": null,
"data.vae_cache.rgb": null,
"data.vae_cache.depth": null,
"data.vae_cache.dtype": null
},
"Image Augmentation": {
"data_augmentation.enabled": null,
"data_augmentation.brightness": null,
"data_augmentation.contrast": null,
"data_augmentation.saturation": null,
"data_augmentation.hue": null,
"data_augmentation.probability": null,
"data_augmentation.depth_source": null,
"data_augmentation.geometry": null,
"data_augmentation.coarse_dropout": null,
"data.train.color_jitter": null,
"data.train.geometry_augment": null,
"data.train.coarse_dropout": null,
"data_augmentation.patchwam": null,
"data.train.patchwam_augmentation": null,
"conditioning_latent_noise": null,
"data.train.augmented_depth_source": null,
"data.val.color_jitter": null,
"data.val.geometry_augment": null,
"data.val.coarse_dropout": null
},
"Objectives and Initialization": {
"model.loss": {
"lambda_video": 1.0,
"lambda_action": 1.0,
"lambda_semantic": 1.0,
"lambda_depth": 1.0
},
"model.semantic.parametrization": "v_pred",
"model.semantic.velocity_loss_form": "direct_error",
"model.semantic.loss_lambda_semantic": 1.0,
"model.depth.loss_lambda_depth": 1.0,
"model.video_scheduler": {
"train_shift": 5.0,
"infer_shift": 5.0,
"num_train_timesteps": 1000
},
"model.action_scheduler": {
"train_shift": 5.0,
"infer_shift": 5.0,
"num_train_timesteps": 1000
},
"model.skip_dit_load_from_pretrain": null,
"model.action_dit_initializer_filename": null
},
"Optimization and Execution": {
"num_epochs": null,
"max_steps": null,
"learning_rate": null,
"lr_scheduler_type": null,
"lr_conv_ratio": null,
"weight_decay": null,
"max_grad_norm": null,
"enforce_grad_clipping": null,
"batch_size": null,
"gradient_accumulation_steps": null,
"mixed_precision": null,
"seed": null,
"num_workers": null,
"model.mot_checkpoint_mixed_attn": null,
"compile_model": null,
"compile_mode": null,
"eval_every": null,
"eval_num_inference_steps": null,
"save_every": null,
"max_checkpoint_history": null,
"save_weights_only": null
}
},
"activation_gates": {
"Depth estimator": "inactive: native GT current and target",
"Pointmap bounds": "inactive",
"DVAE preprocessing": "active",
"Event resampling": "unknown",
"CoarseDropout": null,
"Video MoE": true,
"Action MoE": null,
"Separate wrist stream": "inactive"
}
},
"archived_attention_guide": {
"source": "WorldModelSpec applied to archived config, not checkpoint verification",
"world_streams": [
"rgb",
"depth",
"semantic"
],
"world_cross_reads": [
{
"query": "rgb",
"reads": "depth",
"key_frames": "none"
},
{
"query": "rgb",
"reads": "semantic",
"key_frames": "none"
},
{
"query": "depth",
"reads": "rgb",
"key_frames": "none"
},
{
"query": "depth",
"reads": "semantic",
"key_frames": "all"
},
{
"query": "semantic",
"reads": "rgb",
"key_frames": "none"
},
{
"query": "semantic",
"reads": "depth",
"key_frames": "all"
}
],
"action_reads": [
{
"modality": "depth",
"key_frames": "all",
"spatial_scope": "all"
},
{
"modality": "semantic",
"key_frames": "all",
"spatial_scope": "head positions only (top 2/3 of T token grid)"
}
],
"current_conditions_protected": true,
"within_stream_mask": "first_frame_causal",
"world_reads_noisy_action": "none",
"fusion_layers": "not used",
"separate_wrist_condition": false,
"wrist_reads_primary_condition": "not applicable",
"note": "Direct reads are not total information access: encoder mixing and world paths can carry indirect information. Protected current conditions do not imply strictly autoregressive future frames. World Video blocks are shared; Action DiT parameters are separate. MoE top_k masks weights, not expert computation."
}
}